829 files changed, 95528 insertions, 0 deletions
diff --git a/document/.gitignore b/document/.gitignore
new file mode 100644
index 00000000000..e23d1d9f4e9
--- /dev/null
+++ b/document/.gitignore
@@ -0,0 +1,14 @@
+.classpath
+.project
+bin
+document-lib.iml
+document.iml
+document.ipr
+document.iws
+include
+lib
+libexec
+target
+/pom.xml.build
+Makefile
+Testing
diff --git a/document/AUTHORS b/document/AUTHORS
new file mode 100644
index 00000000000..d88ca23dc13
--- /dev/null
+++ b/document/AUTHORS
@@ -0,0 +1,5 @@
+thomasg
+bratseth
+fledsbo
+borud
+larschri
diff --git a/document/CMakeLists.txt b/document/CMakeLists.txt
new file mode 100644
index 00000000000..72acb9399a9
--- /dev/null
+++ b/document/CMakeLists.txt
@@ -0,0 +1,44 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_define_module(
+    DEPENDS
+    fastos
+    vespalog
+    vespalib
+    staging_vespalib
+    config_cloudconfig
+
+    EXTERNAL_DEPENDS
+    lz4
+
+    LIBS
+    src/vespa/document
+    src/vespa/document/annotation
+    src/vespa/document/base
+    src/vespa/document/bucket
+    src/vespa/document/config
+    src/vespa/document/datatype
+    src/vespa/document/fieldset
+    src/vespa/document/fieldvalue
+    src/vespa/document/predicate
+    src/vespa/document/repo
+    src/vespa/document/select
+    src/vespa/document/serialization
+    src/vespa/document/update
+    src/vespa/document/util
+
+    TEST_DEPENDS
+    vdstestlib
+    cppunit
+
+    TESTS
+    src/tests
+    src/tests/annotation
+    src/tests/base
+    src/tests/datatype
+    src/tests/fieldvalue
+    src/tests/predicate
+    src/tests/repo
+    src/tests/serialization
+    src/tests/struct_anno
+    src/tests/tensor_fieldvalue
+)
diff --git a/document/OWNERS b/document/OWNERS
new file mode 100644
index 00000000000..1037590124e
--- /dev/null
+++ b/document/OWNERS
@@ -0,0 +1 @@
+balder
diff --git a/document/README b/document/README
new file mode 100644
index 00000000000..a7fa52dd062
--- /dev/null
+++ b/document/README
@@ -0,0 +1 @@
+Document module
diff --git a/document/doc/.gitignore b/document/doc/.gitignore
new file mode 100644
index 00000000000..72ec51b65a0
--- /dev/null
+++ b/document/doc/.gitignore
@@ -0,0 +1,4 @@
+document-api.html
+html
+javadoc
+latex
diff --git a/document/doc/document-format.html b/document/doc/document-format.html
new file mode 100644
index 00000000000..214447d6d51
--- /dev/null
+++ b/document/doc/document-format.html
@@ -0,0 +1,564 @@
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<html>
+<head>
+<title>Developers guide to the serialized document format</title>
+</head>
+<body>
+<h1>Developers guide to the serialized document format</h1>
+
+<p>When a Vespa document is stored or transferred from one application to
+another, it is serialized. The serialization format tries to achieve
+serialization robustness and speed. The most important fields are kept in a
+header that is accessible at low cost. The other fields are located by table
+look-ups.</p>
+
+<h2>Purpose</h2>
+
+<p>The purpose of the serialized format is
+<ul>
+<li><b>Robustness</b>. The format shall detect errors gracefully.</li>
+<li><b>Speed</b>. Deserialization shall be fast, especially for basic fields like <b>DocumentId</b>.</li> 
+<li><b>Size</b>. The serialized format shall be compact and allow for efficient storage and transfer.
+That is partly achieved by allowing different kinds of compression. As of now <b>lz4</b> are supported.</li> 
+</ul>
+</p>
+
+<p><strong>All fields are in network byte order.</strong></p>
+
+<h2>Changelog</h2>
+
+<h3>Current version: 8</h3>
+
+<ul>
+<li>CRC removed from document format. There used to be a 4 byte CRC in the end
+of a header or header + body serialization, calculated as a crc32 of all the
+other data in the serialization. This CRC was included in the document length.
+</ul>
+
+
+<h3>Version: 7</h3>
+
+<ul>
+<li>The document length is now a static sized 4 byte value, instead of a variable 2,4,8 byte value.
+<li>(Anything else? I wrote this changelog when bopping from 7 to 8. Dunno if more was changed in 7.)
+</ul>
+
+<h3>Version: 6</h3>
+
+This is the oldest version that we currently support. No known installation stores documents with a version smaller than this.
+
+<h2>Document Format</h2>
+
+<p>This is the description of the serialized document format.</p>
+
+<table  border="1" cellspacing="0" cellpadding="1%" width="100%">
+<caption><em>Document serialization format</em></caption>
+<tr>
+<td width="10%"><b>Field</td>
+<td width="10%"><b>Type</td>
+<td width="10%"><b>Length</td>
+<td><b>Description</td>
+</tr>
+<tr><td>Version</td>
+<td>Short integer</td>
+<td>2</td>
+<td>Version number. Current is 6.</td>
+</tr>
+<tr><td>Length</td>
+<td>Integer</td>
+<td>4 bytes</td>
+<td>Total length of object (excluding this field and version).</td>
+</tr>
+<tr><td>Document ID</td>
+<td>Bytes</td>
+<td>&nbsp;</td>
+<td>Unique ID for document. 0-terminated string, UTF-8 encoding.</td>
+</tr>
+<tr><td>Field Map</td>
+<td>Bytes</td>
+<td>See below</td><td>Placeholder for fields. (Note: Fieldmaps may contain other fieldmaps)</td>
+</tr>
+</table>
+
+<p>Field maps are serialized like this</p></p><p>
+
+<table  border="1" cellspacing="0" cellpadding="1%" width="100%">
+<caption><em>Fieldmap serialization format</em></caption>
+<tr>
+<td width="10%"><b>Field</td>
+<td width="10%"><b>Type</td>
+<td width="10%"><b>Length (bytes)</td>
+<td><b>Description</td>
+</tr>
+<tr><td>Inventory bit mask</td>
+<td>Byte</td>
+<td>1</td>
+<td>
+Inventory bits describing the FieldMap element with data:<br>
+&nbsp;Bit 0 set: FieldMap has document type <br>
+&nbsp;Bit 1 set: FieldMap has header fields <br>
+&nbsp;Bit 2 set: FieldMap has body fields <br>
+&nbsp;Bit 3 set: FieldMap has external body fields<br>
+</tr>
+<tr><td colspan = "4"><b>Below section is present when bit 0 of inventory is set</b></td></tr>
+<tr><td>Document Type</td>
+<td>Bytes</td>
+<td>&nbsp;</td>
+<td>Document type. (0-terminated string, UTF-8 encoding.)</td>
+</tr>
+<tr><td>Version</td>
+<td>Short integer</td>
+<td>2</td>
+<td>Document type version number.</td></tr>
+<tr><td colspan = "4"><b>Below section is present when bit 1 of inventory is set</b></td></tr>
+<tr><td>Header data</td>
+<td>Data array</td>
+<td>See below</td>
+<td>Header data packed in data array</td></tr>
+<tr><td colspan = "4"><b>Below section is present when bit 2 of inventory is set</b></td></tr>
+<tr><td>Body data</td>
+<td>Data array</td>
+<td>See below</td>
+<td>Body data packed in data array</td></tr>
+</table>
+
+
+<p>
+<table  border="1" cellspacing="0" cellpadding="1%" width="100%">
+<caption><em>Data array serialization</em></caption>
+<tr>
+<td width="10%"><b>Field</td>
+<td width="10%"><b>Type</td>
+<td width="10%"><b>Length (bytes)</td>
+<td><b>Description</b></td>
+</tr>
+<tr><td>Data length</td>
+<td>Integer_2_4_8</td>
+<td>2, 4 or 8</td>
+<td>Length of data block (see below). NOTE THAT THIS LENGTH INCLUDE ITSELF.</td>
+</tr>
+<tr><td>Compression</td>
+<td>Byte</td>
+<td>1</td>
+<td>Compression method
+<br>
+&nbsp;0: No compression<br>
+&nbsp;5: Uncompressable<br>
+&nbsp;6: lz4 <br>
+<p>Note that the uncompressable flag is not a configurable option. Rather it
+will be used in document instances who are configured for compression, but
+where compression yields negative results, to avoid later serializations to
+retry compression.</p>
+</td>
+<tr><td>Number of fields<td>Integer_1_4</td>
+<td>1 or 4</td>
+<td>Number of fields in data array</td>
+<tr><td colspan = "4"><b>Below item is present if compression method is not uncompressed or uncompressable</b></td></tr>
+<tr><td>Uncompressed data length<td>Integer_2_4_8</td>
+<td>2, 4 or 8</td>
+<td>Length of data block after decompression</td>
+<tr><td colspan = "4"><b>Below block is repeated "Number of fields" times</b></td></tr>
+<tr><td>Field ID<td>Integer_1_4</td>
+<td>1 or 4</td>
+<td>ID of field.</td>
+<tr><td>Field Size<td>Integer_1_2_4</td>
+<td>1, 2 or 4</td>
+<td>Length of field.</td>
+</td>
+<tr><td colspan = "4"><b>End of repeated block </b></td></tr>
+<tr><td>Data block<td>Bytes</td>
+<td>&nbsp;</td>
+<td>The data block.<br> 
+&nbsp; - Items are ordered the same way field array is sorted.<br> 
+&nbsp; - Use lengths from field array above to find item offset and length.<br>
+&nbsp; - If the block is compressed, lengths refer to decompressed version</td>
+</table>
+
+
+<table  border="1" cellspacing="0" cellpadding="1%" width="100%">
+<caption><em>Data type serialization</em></caption>
+<tr>
+<td width="15%"><b>Data type</td>
+<td width="10%"><b>Length</td>
+<td><b>Serialization</td>
+</tr>
+<tr><td>Integer (ID 0)</td>
+<td>4</td>
+<td>Signed integer, two's complement notation, network byte order.</td>
+</tr>
+<tr><td>Floating point number (ID 1)</td>
+<td>4</td>
+<td>IEEE 754, single precision, network byte order.</td>
+</tr>
+<tr><td>String (ID 2)</td>
+<td>1 + (1 or 4) + length + 1</td>
+<td>Strings are serialization format:<br />
+&nbsp;- First byte represents coding. This has traditionally denoted the maximum number of bits
+        per character in the UTF-8 encoded string, but has never been used in deserialization code.
+<ul>
+  <li>Set to 32 if not used.</li>
+  <li>Set to &lt;32 if you know the UTF-8 string uses less bits per character; e.g. ASCII could use 8.</li>
+  <li>Set bit 6 (decimal 64) if the string has an <a href="#annotations">annotation tree</a>.</li>
+</ul>
+<br />
+&nbsp;- Integer_1_4 with length of string. <br />
+&nbsp;- The string (UTF-8 encoding), including 0-terminating byte.<br>
+&nbsp;- An annotation tree, if bit 6 (decimal 64) of coding byte is set:
+<ul>
+  <li>total length of all span trees excl. itself: <strong>uint32</strong></li>
+  <li>number of span trees <strong>int_1_2_4</strong></li>
+  <li>for each root node:
+  <ol>
+    <li>tree name serialized as String</li>
+    <li>serialized SpanNode as given below, see <a href="#annotations">annotation serialization</a></li>
+  </ol>
+</ul>
+</td>
+</tr>
+<tr><td>Raw bytes (ID 3)</td>
+<td>Length of buffer</td>
+<td>Byte for byte copy</td>
+</tr>
+<tr><td>Long integer (ID 4)</td>
+<td>8</td>
+<td>Signed integer, two's complement notation, network byte order.</td>
+</tr>
+<tr><td>Double floating point number (ID 5)</td>
+<td>8</td>
+<td>IEEE 754, double precision, network byte order.</td>
+</tr>
+<tr><td>Array (ID 6)</td>
+<td>At least 8 bytes</td>
+<td>Arrays of any fields are serialized like this:<br>
+&nbsp;- 4 bytes: Data type array consists of <br>
+&nbsp;- 4 bytes: Number of elements in array<br>
+&nbsp;Below sequence is repeated "number of element" times<br>
+&nbsp;- 4 bytes: Length of element<br>
+&nbsp;- Serialized element<br>
+</td></tr>
+<tr><td>Fieldmap (ID 7)</td>
+<td>&nbsp;</td>
+<td>Field maps (embedded or not) are defined above</td>
+</tr>
+<tr><td>Document (ID 8)</td>
+<td>&nbsp;</td>
+<td>Document objects (embedded or not) are defined above</td>
+</tr>
+<tr><td>Timestamp (ID 9)</td>
+<td>&nbsp;</td>
+<td>Same as long integer</td>
+</tr>
+<tr><td>Uri (ID 10)</td>
+<td>&nbsp;</td>
+<td>Same as string</td>
+</tr>
+<tr><td>Exact string (ID 11)</td>
+<td>&nbsp;</td>
+<td>Same as string</td>
+</tr>
+<tr><td>Content (ID 12)</td>
+<td>At least 11 bytes</td>
+<td>Content fields are serialized like this:<br>
+&nbsp;- Content type length (1 byte)<br>
+&nbsp;- Content type (0 terminated string, UTF-8 encoding.)<br>
+&nbsp;- Content encoding length (1 byte)<br>
+&nbsp;- Content encoding (0 terminated string, UTF-8 encoding.)<br>
+&nbsp;- Content language length (1 byte)<br>
+&nbsp;- Content language (0 terminated string, UTF-8 encoding.)<br>
+&nbsp;- Content length (Integer, 4 bytes)<br>
+&nbsp;- Content (including 0-terminating char)<br>
+</td>
+</tr>
+<tr><td>Content meta (ID 13)</td>
+<td>At least 12 bytes</td>
+<td>Content (attachment) meta data are serialized like this:<br>
+&nbsp;- Attachment size (Integer, 4 bytes)<br>
+&nbsp;- Attachment name  (0 terminated string, UTF-8 encoding.)<br>
+&nbsp;- Attachment encoding (0 terminated string, UTF-8 encoding.)<br>
+&nbsp;- Attachment content type (0 terminated string, UTF-8 encoding.)<br>
+&nbsp;- Attachment part (0 terminated string, UTF-8 encoding.)<br>
+&nbsp;- Attachment flag (Integer, 4 bytes)<br>
+</td>
+</tr>
+<tr><td>Term boost (ID 15)</td>
+<td>&nbsp;</td>
+<td>Same as string</td>
+</tr>
+<tr><td>Byte (ID 16)</td>
+<td>1</td>
+<td>One single byte</td>
+</tr>
+<tr><td>Set (ID 17)</td>
+<td>At least 8 bytes</td>
+<td>Set of any fields are serialized like this:<br>
+&nbsp;- Integer (4 bytes): Data type set is made up of<br>
+&nbsp;- Integer (4 bytes): Number of elements in set<br>
+&nbsp;Below sequence is repeated "number of element" times<br>
+&nbsp;- Serialized element<br>
+</td></tr>
+</table>
+
+
+<a name="annotations"><table  border="1" cellspacing="0" cellpadding="1%" width="100%">
+<caption><em>Annotation tree serialization</em></caption>
+<tr>
+<td width="15%"><b>Data type</td>
+<td width="10%"><b>Length</td>
+<td><b>Serialization</td>
+</tr>
+<tr>
+<td>SpanNode (base class)</td>
+<td>1 + (1, 2 or 4) + Annotation serialization + subclass payload</td>
+<td>
+ <ul>
+<li> type <strong>byte</strong> (1: Span, 2: SpanList, 4: AlternateSpanList)
+</li> <li> number of annotations <strong>int_1_2_4</strong>
+</li> <li> each annotation as given below
+</li> <li> (remaining payload serialized as given below by subclasses Span, SpanList and AlternateSpanList)
+</li></ul>
+</td>
+</tr>
+<tr>
+<td>Annotation</td>
+<td>4 + (1, 2 or 4) + (possibly 4 + FieldValue serialization)</td>
+<td>
+<ul>
+<li>MD5 name hash (4 LSBytes) <strong>uint32</strong> (NOTE: 0-127 reserved for internal Vespa usage.)
+</li> <li> length <strong>int_1_2_4</strong>
+</li> <li> the following fields are <em>only</em> present if length &gt; 0: <ul>
+<li> data type id <strong>uint32</strong>
+</li> <li> NOTE: no sequence id, as we will rely on annotations being serialized/deserialized in particular order, so we don't need to write this explicitly
+</li> <li> FieldValue as given by its own serialization
+</li></ul> 
+</li></ul> 
+</td>
+</tr>
+<tr>
+<td>Span</td>
+<td>SpanNode serialization + (1, 2 or 4) + (1, 2 or 4)</td>
+<td><ul>
+<li>serialization from SpanNode base class</li>
+<li> from index, as given by Java String (UTF-16) <strong>int_1_2_4</strong>
+</li> <li> length, as given by Java String (UTF-16) <strong>int_1_2_4</strong>
+</li></ul>
+</td>
+</tr>
+<tr>
+<td>SpanList</td>
+<td>SpanNode serialization + (1, 2 or 4) + n times SpanNode serialization</td>
+<td>
+<ul>
+<li>serialization from SpanNode base class</li>
+<li> number of children <strong>int_1_2_4</strong>
+</li> <li> each child node serialized as SpanNode (Span, SpanList, AlternateSpanList)
+</li></ul>
+</td>
+</tr>
+<tr>
+<td>AlternateSpanList</td>
+<td>SpanNode serialization + (1, 2 or 4) + n times (8 + SpanList serialization)</td>
+<td><ul>
+<li>serialization from SpanNode base class</li>
+<li> number of child trees <strong>int_1_2_4</strong>
+</li> <li> for each child tree: <ul>
+<li> probability <strong>double</strong>
+</li> <li> serialization as given by SpanList above
+</li></ul> 
+</li></ul>
+</td>
+</tr>
+<tr>
+<td>AnnotationRef</td>
+<td>1, 2 or 4</td>
+<td>AnnotationRef serialization <ul>
+<li> unique sequence id of annotation being referred to <strong>int1_2_4</strong>
+</li></ul>
+</td>
+</tr>
+</table>
+
+<table  border="1" cellspacing="0" cellpadding="1%" width="100%">
+<caption><em>Data types used in serialized format</em></caption>
+<tr>
+<td width="15%"><b>Data type</td>
+<td><b>Serialization</b></td>
+</tr>
+<tr><td>Integer_1_4</td>
+<td>If bit 7 of first byte is unset, coded using 1 byte.<br />
+    If bit 7 of first byte is set, coded using 4 bytes (bit 7 of first byte must be masked away).<br />
+    <em>Range: 0  -  2**31-1.</em></td>
+</tr>
+<tr><td>Integer_1_2_4</td>
+<td>If bit 7 of first byte is unset, coded using 1 byte.<br />
+    If bit 7 of first byte is set and bit 6 of first byte is unset, coded using 2 bytes (bit 7 and 6 of first byte must be masked away).<br />
+    If bit 7 and 6 of first byte are set, coded using 4 bytes (bit 7 and 6 of first byte must be masked away).<br />
+    <em>Range: 0  -  2**30-1.</em></td>
+</td>
+</tr>
+<tr><td>Integer_2_4_8</td>
+<td>If bit 7 of first byte is unset, coded using 2 byte.<br />
+    If bit 7 of first byte is set and bit 6 of first byte is unset, coded using 4 bytes (bit 7 and 6 of first byte must be masked away).<br />
+    If bit 7 and 6 of first byte are set, coded using 8 bytes (bit 7 and 6 of first byte must be masked away).<br />
+    <em>Range: 0  -  2**62-1.</em></td>
+</td>
+</tr>
+</table>
+
+
+<h2>Document Update Format</h2>
+
+<p>This is the description of the serialized document update format.</p>
+
+<table  border="1" cellspacing="0" cellpadding="1%" width="100%">
+<caption><em>Document update serialization format</em></caption>
+<tr>
+<td width="10%"><b>Field</td>
+<td width="10%"><b>Type</td>
+<td width="10%"><b>Length</td>
+<td><b>Description</td>
+</tr>
+<tr><td>Document ID</td>
+<td>Bytes</td>
+<td>&nbsp;</td>
+<td>Unique ID for document. 0-terminated string, UTF-8 encoding.</td>
+</tr>
+<tr><td>Content byte</td>
+<td>Byte</td>
+<td>1 byte</td>
+<td>Always set to 1</td>
+</tr>
+<tr><td>Document Type</td>
+<td>Bytes</td>
+<td>&nbsp;</td>
+<td>Document type. (0-terminated string, UTF-8 encoding.)</td>
+</tr>
+<tr><td>Number of fields to update</td>
+<td>Integer</td>
+<td>4 bytes</td>
+<td>The number of fields to update</td>
+</tr>
+<tr><td>Serialized field updates</td>
+<td>Field Update</td>
+<td>&nbsp;</td>
+<td>The serialized field updates. See below.</td>
+</tr>
+</table>
+
+<table  border="1" cellspacing="0" cellpadding="1%" width="100%">
+<caption><em>Document update serialization format</em></caption>
+<tr>
+<td width="10%"><b>Field</td>
+<td width="10%"><b>Type</td>
+<td width="10%"><b>Length</td>
+<td><b>Description</td>
+</tr>
+<tr><td>Field Id</td>
+<td>Integer</td>
+<td>4 bytes</td>
+<td>Field id within document type.</td>
+</tr>
+<tr><td>Number of value updates</td>
+<td>Integer</td>
+<td>4 bytes</td>
+<td>Numer of value updates to this field.</td>
+</tr>
+<tr><td>Serialized field update values</td>
+<td>Bytes</td>
+<td>&nbsp;</td>
+<td>The serialized field update values. See below.</td>
+</tr>
+</table>
+
+<table  border="1" cellspacing="0" cellpadding="1%" width="100%">
+<caption><em>Document update value serialization format</em></caption>
+<tr>
+<td width="10%"><b>Field</td>
+<td width="10%"><b>Type</td>
+<td width="10%"><b>Length</td>
+<td><b>Description</td>
+</tr>
+<tr><th colspan="4">Add Value Update</th></tr>
+<tr><td>Add Value Update ID</td>
+<td>Integer</td>
+<td>4 bytes</td>
+<td>25 + 0x1000 for value updates.</td>
+</tr>
+<tr><td>Field serialization</td>
+<td>FieldValue</td>
+<td>&nbsp;</td>
+<td>Serialization of the field to add.</td>
+</tr>
+<tr><td>Weight</td>
+<td>Integer</td>
+<td>4 bytes</td>
+<td>Weight. Used if update applies to weighted set.</td>
+</tr>
+<tr><th colspan="4">Arithmetic Update</th></tr>
+<tr><td>Arithmetic Update ID</td>
+<td>Integer</td>
+<td>4 bytes</td>
+<td>26 + 0x1000 for arithmetic updates.</td>
+</tr>
+<tr><td>Operator ID</td>
+<td>Integer</td>
+<td>4 bytes</td>
+<td>Identifies whether this does add, subtract, multiply or divide.</td>
+</tr>
+<tr><td>Operand</td>
+<td>Double</td>
+<td>8 bytes</td>
+<td>The right operand to use in the arithmetic operation.</td>
+</tr>
+<tr><th colspan="4">Assign Update</th></tr>
+<tr><td>Assign Update ID</td>
+<td>Integer</td>
+<td>4 bytes</td>
+<td>27 + 0x1000 for assign updates.</td>
+</tr>
+<tr><td>Content flag</td>
+<td>Byte</td>
+<td>1 bytes</td>
+<td>Contains 1 if we have content, 0 if not.</td>
+</tr>
+<tr><td>Field serialization</td>
+<td>FieldValue</td>
+<td>&nbsp;</td>
+<td>Serialization of the field to assign.</td>
+</tr>
+<tr><th colspan="4">Clear Update</th></tr>
+<tr><td>Clear Update ID</td>
+<td>Integer</td>
+<td>4 bytes</td>
+<td>28 + 0x1000 for clear updates.</td>
+</tr>
+<tr><th colspan="4">Map Value Update</th></tr>
+<tr><td>Map Value Update ID</td>
+<td>Integer</td>
+<td>4 bytes</td>
+<td>29 + 0x1000 for map value updates.</td>
+</tr>
+<tr><td>Field serialization</td>
+<td>FieldValue</td>
+<td>4 bytes</td>
+<td>The field indicating what entry to update.</td>
+</tr>
+<tr><td>Value Update</td>
+<td>Document Value Update</td>
+<td>&nbsp;</td>
+<td>The update operation to apply to the field indicated above.</td>
+</tr>
+<tr><th colspan="4">Remove Value Update</th></tr>
+<tr><td>Remove Update ID</td>
+<td>Integer</td>
+<td>4 bytes</td>
+<td>30 + 0x1000 for remove updates.</td>
+</tr>
+<tr><td>Field serialization</td>
+<td>FieldValue</td>
+<td>&nbsp;</td>
+<td>The field indicating what entry to update.</td>
+</tr>
+</table>
+
+</body>
+</html>
diff --git a/document/pom.xml b/document/pom.xml
new file mode 100644
index 00000000000..5d882b17f25
--- /dev/null
+++ b/document/pom.xml
@@ -0,0 +1,142 @@
+<?xml version="1.0"?>
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
+                             http://maven.apache.org/maven-v4_0_0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>com.yahoo.vespa</groupId>
+        <artifactId>parent</artifactId>
+        <version>6-SNAPSHOT</version>
+        <relativePath>../parent/pom.xml</relativePath>
+    </parent>
+    <artifactId>document</artifactId>
+    <version>6-SNAPSHOT</version>
+    <packaging>jar</packaging>
+    <name>${project.artifactId}</name>
+    <dependencies>
+        <dependency>
+          <groupId>com.fasterxml.jackson.core</groupId>
+          <artifactId>jackson-core</artifactId>
+          <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-databind</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>commons-codec</groupId>
+            <artifactId>commons-codec</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>commons-collections</groupId>
+            <artifactId>commons-collections</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>net.jpountz.lz4</groupId>
+            <artifactId>lz4</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.mockito</groupId>
+            <artifactId>mockito-core</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.yahoo.vespa</groupId>
+            <artifactId>config</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.yahoo.vespa</groupId>
+            <artifactId>predicate-search-core</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.yahoo.vespa</groupId>
+            <artifactId>vespajlib</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.yahoo.vespa</groupId>
+            <artifactId>annotations</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+          <groupId>com.google.guava</groupId>
+          <artifactId>guava</artifactId>
+          <scope>provided</scope>
+        </dependency>
+      <dependency>
+        <groupId>com.google.inject</groupId>
+        <artifactId>guice</artifactId>
+        <classifier>no_aop</classifier>
+        <scope>provided</scope>
+      </dependency>
+    </dependencies>
+    <build>
+        <plugins>
+            <plugin>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <configuration>
+                    <archive>
+                        <manifest>
+                            <mainClass>com.yahoo.document.foo</mainClass>
+                        </manifest>
+                    </archive>
+                    <descriptorRefs>
+                        <descriptorRef>jar-with-dependencies</descriptorRef>
+                    </descriptorRefs>
+                </configuration>
+                <executions>
+                    <execution>
+                        <id>make-assembly</id>
+                        <phase>package</phase>
+                        <!-- append to the packaging phase. -->
+                        <goals>
+                            <goal>single</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.codehaus.mojo</groupId>
+                <artifactId>javacc-maven-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>javacc</id>
+                        <goals>
+                            <goal>javacc</goal>
+                        </goals>
+                        <configuration>
+                            <lookAhead>1</lookAhead>
+                            <isStatic>false</isStatic>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>com.yahoo.vespa</groupId>
+                <artifactId>config-class-plugin</artifactId>
+                <version>${project.version}</version>
+                <configuration>
+                    <defFilesDirectories>src/vespa/document/config</defFilesDirectories>
+                </configuration>
+                <executions>
+                    <execution>
+                        <id>config-gen</id>
+                        <goals>
+                            <goal>config-gen</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+</project>
diff --git a/document/src/.gitignore b/document/src/.gitignore
new file mode 100644
index 00000000000..12cd838c514
--- /dev/null
+++ b/document/src/.gitignore
@@ -0,0 +1,4 @@
+Makefile.ini
+config_command.sh
+document.mak
+project.dsw
diff --git a/document/src/main/java/com/yahoo/document/ArrayDataType.java b/document/src/main/java/com/yahoo/document/ArrayDataType.java
new file mode 100644
index 00000000000..640bd94bd1c
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/ArrayDataType.java
@@ -0,0 +1,62 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import com.yahoo.document.datatypes.Array;
+import com.yahoo.vespa.objects.Ids;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class ArrayDataType extends CollectionDataType {
+    // The global class identifier shared with C++.
+    public static int classId = registerClass(Ids.document + 54, ArrayDataType.class);
+
+    public ArrayDataType(DataType nestedType) {
+        super("Array<"+nestedType.getName()+">", 0, nestedType);
+        setId(getName().toLowerCase().hashCode());
+    }
+
+    public ArrayDataType(DataType nestedType, int code) {
+        super("Array<"+nestedType.getName()+">", code, nestedType);
+    }
+
+    public ArrayDataType clone() {
+        return (ArrayDataType) super.clone();
+    }
+
+    public Array createFieldValue() {
+        return new Array(this);
+    }
+
+    @Override
+    public Class getValueClass() {
+        return Array.class;
+    }
+
+    @Override
+    public FieldPath buildFieldPath(String remainFieldName)
+    {
+        if (remainFieldName.length() > 0 && remainFieldName.charAt(0) == '[') {
+            int endPos = remainFieldName.indexOf(']');
+            if (endPos == -1) {
+                throw new IllegalArgumentException("Array subscript must be closed with ]");
+            } else {
+                FieldPath path = getNestedType().buildFieldPath(skipDotInString(remainFieldName, endPos));
+                List<FieldPathEntry> tmpPath = new ArrayList<FieldPathEntry>(path.getList());
+                if (remainFieldName.charAt(1) == '$') {
+                    tmpPath.add(0, FieldPathEntry.newVariableLookupEntry(remainFieldName.substring(2, endPos), getNestedType()));
+                } else {
+                    tmpPath.add(0, FieldPathEntry.newArrayLookupEntry(Integer.parseInt(remainFieldName.substring(1, endPos)), getNestedType()));
+                }
+
+                return new FieldPath(tmpPath);
+            }
+        }
+
+        return getNestedType().buildFieldPath(remainFieldName);
+    }
+
+}
diff --git a/document/src/main/java/com/yahoo/document/BaseStructDataType.java b/document/src/main/java/com/yahoo/document/BaseStructDataType.java
new file mode 100755
index 00000000000..76ee98331ab
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/BaseStructDataType.java
@@ -0,0 +1,152 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import com.yahoo.compress.CompressionType;
+import com.yahoo.compress.Compressor;
+
+import java.util.Collection;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+/**
+ * Date: Apr 15, 2008
+ *
+ * @author humbe
+ */
+public abstract class BaseStructDataType extends StructuredDataType {
+
+    protected Map<Integer, Field> fieldIds = new LinkedHashMap<>();
+    protected Map<Integer, Field> fieldIdsV6 = new LinkedHashMap<>();
+    protected Map<String, Field> fields = new LinkedHashMap<>();
+
+    protected Compressor compressor = new Compressor(CompressionType.NONE);
+
+    BaseStructDataType(String name) {
+        super(name);
+    }
+
+    BaseStructDataType(int id, String name) {
+        super(id, name);
+    }
+
+    protected void assign(BaseStructDataType type) {
+        BaseStructDataType stype = type.clone();
+
+        fieldIds = stype.fieldIds;
+        fieldIdsV6 = stype.fieldIdsV6;
+        fields = stype.fields;
+    }
+
+    @Override
+    public BaseStructDataType clone() {
+        BaseStructDataType type = (BaseStructDataType) super.clone();
+        type.fieldIds = new LinkedHashMap<>();
+        type.fieldIdsV6 = new LinkedHashMap<>();
+
+        type.fields = new LinkedHashMap<>();
+        for (Field field : fieldIds.values()) {
+            type.fields.put(field.getName(), field);
+            type.fieldIds.put(field.getId(Document.SERIALIZED_VERSION), field);
+            type.fieldIdsV6.put(field.getId(6), field);
+        }
+        return type;
+    }
+
+    public void addField(Field field) {
+        if (fields.containsKey(field.getName())) {
+            throw new IllegalArgumentException("Struct " + getName() + " already contains field with name " + field.getName());
+        }
+        if (fieldIds.containsKey(field.getId(Document.SERIALIZED_VERSION))) {
+            throw new IllegalArgumentException("Struct " + getName() + " already contains field with id " + field.getId(Document.SERIALIZED_VERSION));
+        }
+        if (fieldIdsV6.containsKey(field.getId(6))) {
+            throw new IllegalArgumentException("Struct " + getName() + " already contains a field with deprecated document serialization id " + field.getId(6));
+        }
+
+        fields.put(field.getName(), field);
+        fieldIds.put(field.getId(Document.SERIALIZED_VERSION), field);
+        fieldIdsV6.put(field.getId(6), field);
+    }
+
+    public Field removeField(String fieldName) {
+        Field old = fields.remove(fieldName);
+        if (old != null) {
+            fieldIds.remove(old.getId(Document.SERIALIZED_VERSION));
+            fieldIdsV6.remove(old.getId(6));
+        }
+        return old;
+    }
+
+    public void clearFields() {
+        fieldIds.clear();
+        fieldIdsV6.clear();
+        fields.clear();
+    }
+
+    public Field getField(Integer fieldId, int version) {
+        if (version > 6) {
+            return fieldIds.get(fieldId);
+        } else {
+            return fieldIdsV6.get(fieldId);
+        }
+    }
+
+    @Override
+    public Field getField(String fieldName) {
+        return fields.get(fieldName);
+    }
+
+    @Override
+    public Field getField(int id) {
+        return fieldIds.get(id);
+    }
+
+    public boolean hasField(Field field, int version) {
+        Field f = getField(field.getId(version), version);
+        return f != null && f.equals(field);
+    }
+
+    public boolean hasField(String name) {
+        return fields.containsKey(name);
+    }
+
+    public boolean hasField(Field f) {
+        if (hasField(f, 6) || hasField(f, Document.SERIALIZED_VERSION)) {
+            return true;
+        }
+        return false;
+    }
+
+    @Override
+    public Collection<Field> getFields() {
+        return fields.values();
+    }
+
+    public int getFieldCount() {
+        return fields.size();
+    }
+
+    /** Returns the compressor to use to compress data of this type */
+    public Compressor getCompressor() { return compressor; }
+
+    /** Returns a view of the configuration of the compressor used to compress this type */
+    public CompressionConfig getCompressionConfig() {
+        // CompressionConfig accepts a percentage (but exposes a factor) ...
+        float compressionThresholdPercentage = (float)compressor.compressionThresholdFactor() * 100;
+
+        return new CompressionConfig(compressor.type(),
+                                     compressor.level(),
+                                     compressionThresholdPercentage,
+                                     compressor.compressMinSizeBytes());
+    }
+
+    /** Set the config to the compressor used to compress data of this type */
+    public void setCompressionConfig(CompressionConfig config) {
+        CompressionType type = config.type;
+        compressor = new Compressor(type,
+                                    config.compressionLevel,
+                                    config.thresholdFactor(),
+                                    (int)config.minsize);
+    }
+
+}
diff --git a/document/src/main/java/com/yahoo/document/BucketDistribution.java b/document/src/main/java/com/yahoo/document/BucketDistribution.java
new file mode 100644
index 00000000000..bb4792b5982
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/BucketDistribution.java
@@ -0,0 +1,205 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import com.yahoo.document.BucketId;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class BucketDistribution {
+
+    // A logger object to enable proper logging.
+    private static Logger log = Logger.getLogger(BucketDistribution.class.getName());
+
+    // A map from bucket id to column index.
+    private int[] bucketToColumn;
+
+    // The number of columns to distribute to.
+    private int numColumns;
+
+    // The number of bits to use for bucket identification.
+    private int numBucketBits;
+
+    /**
+     * Constructs a new bucket distribution object with a given number of columns and buckets.
+     *
+     * @param numColumns    The number of columns to distribute to.
+     * @param numBucketBits The number of bits to use for bucket id.
+     */
+    public BucketDistribution(int numColumns, int numBucketBits) {
+        this.numBucketBits = numBucketBits;
+        bucketToColumn = new int[getNumBuckets()];
+        reset();
+        setNumColumns(numColumns);
+    }
+
+    /**
+     * Constructs a new bucket distribution object as a copy of another.
+     *
+     * @param other The distribution object to copy.
+     */
+    public BucketDistribution(BucketDistribution other) {
+        bucketToColumn = other.bucketToColumn.clone();
+        numColumns = other.numColumns;
+        numBucketBits = other.numBucketBits;
+    }
+
+    /**
+     * Returns the number of buckets that the given number of bucket bits will allow.
+     *
+     * @param numBucketBits The number of bits to use for bucket id.
+     * @return The number of buckets allowed.
+     */
+    private static int getNumBuckets(int numBucketBits) {
+        return 1 << numBucketBits;
+    }
+
+    /**
+     * This method returns a list that contains the distributions of the given number of buckets over the given number
+     * of columns.
+     *
+     * @param numColumns    The number of columns to distribute to.
+     * @param numBucketBits The number of bits to use for bucket id.
+     * @return The bucket distribution.
+     */
+    private static List<Integer> getBucketCount(int numColumns, int numBucketBits) {
+        List<Integer> ret = new ArrayList<Integer>(numColumns);
+        int cnt = getNumBuckets(numBucketBits) / numColumns;
+        int rst = getNumBuckets(numBucketBits) % numColumns;
+        for (int i = 0; i < numColumns; ++i) {
+            ret.add(cnt + (i < rst ? 1 : 0));
+        }
+        return ret;
+    }
+
+    /**
+     * This method returns a list similar to {@link BucketDistribution#getBucketCount(int, int)}, except that the returned list
+     * contains the number of buckets that will have to be migrated from each column if an additional column was added.
+     *
+     * @param numColumns    The original number of columns.
+     * @param numBucketBits The number of bits to use for bucket id.
+     * @return The number of buckets to migrate, one value per column.
+     */
+    private static List<Integer> getBucketMigrateCount(int numColumns, int numBucketBits) {
+        List<Integer> ret = getBucketCount(numColumns++, numBucketBits);
+        int cnt = getNumBuckets(numBucketBits) / numColumns;
+        int rst = getNumBuckets(numBucketBits) % numColumns;
+        for (int i = 0; i < numColumns - 1; ++i) {
+            ret.set(i, ret.get(i) - (cnt + (i < rst ? 1 : 0)));
+        }
+        return ret;
+    }
+
+    /**
+     * Sets the number of columns to distribute to to 1, and resets the content of the internal bucket-to-column map so
+     * that it all buckets point to that single column.
+     */
+    public void reset() {
+        for (int i = 0; i < bucketToColumn.length; ++i) {
+            bucketToColumn[i] = 0;
+        }
+        numColumns = 1;
+    }
+
+    /**
+     * Adds a single column to this bucket distribution object. This will modify the internal bucket-to-column map so
+     * that it takes into account the new column.
+     */
+    private void addColumn() {
+        int newColumns = numColumns + 1;
+        List<Integer> migrate = getBucketMigrateCount(numColumns, numBucketBits);
+        int numBuckets = getNumBuckets(numBucketBits);
+        for (int i = 0; i < numBuckets; ++i) {
+            int old = bucketToColumn[i];
+            if (migrate.get(old) > 0) {
+                bucketToColumn[i] = numColumns; // move this bucket to the new column
+                migrate.set(old, migrate.get(old) - 1);
+            }
+        }
+        numColumns = newColumns;
+    }
+
+    /**
+     * Sets the number of columns to use for this document distribution object. This will reset and setup this object
+     * from scratch. The original number of buckets is maintained.
+     *
+     * @param numColumns The new number of columns to distribute to.
+     */
+    public synchronized void setNumColumns(int numColumns) {
+        if (numColumns < this.numColumns) {
+            reset();
+        }
+        if (numColumns == this.numColumns) {
+            return;
+        }
+        for (int i = numColumns - this.numColumns; --i >= 0; ) {
+            addColumn();
+        }
+    }
+
+    /**
+     * Returns the number of columns to distribute to.
+     *
+     * @return The number of columns.
+     */
+    public int getNumColumns() {
+        return numColumns;
+    }
+
+    /**
+     * Sets the number of buckets to use for this document distribution object. This will reset and setup this object
+     * from scratch. The original number of columns is maintained.
+     *
+     * @param numBucketBits The new number of bits to use for bucket id.
+     */
+    public synchronized void setNumBucketBits(int numBucketBits) {
+        if (numBucketBits == this.numBucketBits) {
+            return;
+        }
+        this.numBucketBits = numBucketBits;
+        bucketToColumn = new int[getNumBuckets(numBucketBits)];
+        int numColumns = this.numColumns;
+        reset();
+        setNumColumns(numColumns);
+    }
+
+    /**
+     * Returns the number of bits used for bucket identifiers.
+     *
+     * @return The number of bits.
+     */
+    public int getNumBucketBits() {
+        return numBucketBits;
+    }
+
+    /**
+     * Returns the number of buckets available using the configured number of bucket bits.
+     *
+     * @return The number of buckets.
+     */
+    public int getNumBuckets() {
+        return getNumBuckets(numBucketBits);
+    }
+
+    /**
+     * This method maps the given bucket id to its corresponding column.
+     *
+     * @param bucketId The bucket whose column to lookup.
+     * @return The column to distribute the bucket to.
+     */
+    public int getColumn(BucketId bucketId) {
+        int ret = (int)(bucketId.getId() & (getNumBuckets(numBucketBits) - 1));
+        if (ret >= bucketToColumn.length) {
+            log.log(Level.SEVERE,
+                    "The bucket distribution map is not in sync with the number of bucket bits. " +
+                    "This should never happen! Distribution is broken!!");
+            return 0;
+        }
+        return bucketToColumn[ret];
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/BucketId.java b/document/src/main/java/com/yahoo/document/BucketId.java
new file mode 100755
index 00000000000..d0e360ddb2d
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/BucketId.java
@@ -0,0 +1,131 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+/**
+ * Representation of a bucket identifier.
+ */
+public class BucketId implements Comparable<BucketId> {
+    public static final int COUNT_BITS = 6;
+    private static final long STRIP_MASK = 0xFC000000FFFFFFFFl;
+    private long id = 0;
+    private static long[] usedMask;
+
+    static {
+        usedMask = new long[59];
+        long val = 0;
+        for (int i=0; i<usedMask.length; ++i) {
+            usedMask[i] = val;
+            val = (val << 1) | 1;
+        }
+    }
+
+    /**
+     * Default-constructed BucketId signifies an invalid bucket ID.
+     */
+    public BucketId() {
+    }
+
+    /**
+     * Creates a bucket id with the given raw bucket id. This is a 64 bit mask
+     * where the first 6 MSB bits set how many LSB bits should actually be used.
+     * Right now it only have simple functionality. More will be added for it
+     * to be configurable.
+     */
+    public BucketId(long id) {
+        this.id = id;
+    }
+
+    public BucketId(int usedBits, long id) {
+        long usedMask = ((long) usedBits) << (64 - COUNT_BITS);
+        id <<= COUNT_BITS;
+        id >>>= COUNT_BITS;
+        this.id = id | usedMask;
+    }
+
+    public BucketId(String serialized) {
+        if (!serialized.startsWith("BucketId(0x")) {
+            throw new IllegalArgumentException("Serialized bucket id must start with 'BucketId(0x'");
+        }
+        if (!serialized.endsWith(")")) {
+            throw new IllegalArgumentException("Serialized bucket id must end with ')'");
+        }
+
+        // Parse hex string after "0x"
+        int index;
+        char c;
+        long id = 0;
+        for (index = 11; index < serialized.length()-1; index++) {
+            c = serialized.charAt(index);
+            if (!((c>=48 && c<=57) || // digit
+                  (c>=97 && c<=102))) { // a-f
+                throw new IllegalArgumentException("Serialized bucket id (" + serialized + ") contains illegal character at position " + index);
+            }
+            id <<= 4;
+            id += Integer.parseInt(String.valueOf(c),16);
+        }
+        this.id = id;
+        if (getUsedBits() == 0) {
+            throw new IllegalArgumentException("Created bucket id "+id+", but no countbits are set");
+        }
+    }
+
+    public boolean equals(Object o) {
+        return (o instanceof BucketId && ((BucketId) o).getId() == this.getId());
+    }
+
+    public int compareTo(BucketId other) {
+        if (id >>> 32 == other.id >>> 32) {
+            if ((id & 0xFFFFFFFFl) > (other.id & 0xFFFFFFFFl)) {
+                return 1;
+            } else if ((id & 0xFFFFFFFFl) < (other.id & 0xFFFFFFFFl)) {
+                return -1;
+            }
+            return 0;
+        } else if ((id >>> 32) > (other.id >>> 32)) {
+            return 1;
+        } else {
+            return -1;
+        }
+    }
+
+    public int hashCode() {
+        return (int) id;
+    }
+
+    public int getUsedBits() { return (int) (id >>> (64 - COUNT_BITS)); }
+
+    public long getRawId() { return id; }
+
+    public long getId() {
+        int notUsed = 64 - getUsedBits();
+        long usedMask = (0xFFFFFFFFFFFFFFFFl << notUsed) >>> notUsed;
+        long countMask = (0xFFFFFFFFFFFFFFFFl >>> (64 - COUNT_BITS)) << (64 - COUNT_BITS);
+        return id & (usedMask | countMask);
+    }
+
+    public long withoutCountBits() {
+        return id & usedMask[getUsedBits()];
+    }
+
+    public String toString() {
+        StringBuilder sb = new StringBuilder().append("BucketId(0x");
+        String number = Long.toHexString(getId());
+        for (int i=number.length(); i<16; ++i) {
+            sb.append('0');
+        }
+        sb.append(number).append(')');
+        return sb.toString();
+    }
+
+    public boolean contains(BucketId id) {
+        if (id.getUsedBits() < getUsedBits()) {
+            return false;
+        }
+        BucketId copy = new BucketId(getUsedBits(), id.getRawId());
+        return (copy.getId() == getId());
+    }
+
+    public boolean contains(DocumentId docId, BucketIdFactory factory) {
+        return contains(factory.getBucketId(docId));
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/BucketIdFactory.java b/document/src/main/java/com/yahoo/document/BucketIdFactory.java
new file mode 100644
index 00000000000..f327d907448
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/BucketIdFactory.java
@@ -0,0 +1,104 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import com.yahoo.document.idstring.*;
+
+/**
+ * A bucket id contains bit used for various purposes. In most use cases, these can use the default
+ * settings, but the number of bits used for the different purposes is configurable, to allow for
+ * special uses.
+ *
+ * Because of this, bucket ids cannot be generated without knowing how the bucket id is configured to
+ * be put together, so all bucket ids must be generated by this factory class.
+ *
+ * For more information about what the sub parts of a bucket id actually is, read the bucket splitting documentation.
+ *
+ * @author <a href="mailto:humbe@yahoo-inc.com">H&aring;kon Humberset</a>
+ */
+public class BucketIdFactory {
+
+    private final int gidBits;
+    private final int locationBits;
+    private final int countBits;
+
+    private final long initialCount;
+    private final long locationMask;
+    private final long gidMask;
+
+    /** Create a factory, using the default configuration. */
+    public BucketIdFactory() {
+        this(32, 26, 6);
+    }
+
+    /**
+     * Create a factory, using the provided configuration.
+     * @param gidBits How many bits that are used to specify gidbits.
+     */
+    public BucketIdFactory(int locationBits, int gidBits, int countBits) {
+        this.locationBits = locationBits;
+        this.gidBits = gidBits;
+        this.countBits = countBits;
+        initialCount = 58l << (64 - countBits);
+        locationMask = 0xFFFFFFFFFFFFFFFFl >>> (64 - getLocationBitCount());
+        gidMask = ((0xFFFFFFFFFFFFFFFFl >>> getLocationBitCount()) << (getLocationBitCount() + BucketId.COUNT_BITS)) >>> BucketId.COUNT_BITS;
+
+    }
+
+    /**
+     * Create a factory, with parameters gotten from configuration.
+     * TODO: Not implemented yet
+     * @param configId The config id from where to get config.
+     */
+    public BucketIdFactory(String configId) {
+        this(32, 26, 6);
+    }
+
+    /** @return Get number of bits used for storing of LSB part of location.*/
+    public int getLocationBitCount() { return locationBits; }
+
+    /** @return Get number of bits used to specify gid. */
+    public int getGidBitCount() { return gidBits; }
+
+    /** @return Get number of bits used to store bit count used. */
+    public int getCountBitCount() { return countBits; }
+
+    /**
+     * Get the gid bit contribution in the bucket id, shifted to the correct
+     * position in the id.
+     *
+     * @param gid The gid we need to calculate contribution from.
+     * @return A mask to or with the bucket id to get the bit set.
+     */
+    private long getGidContribution(byte[] gid) {
+        long gidbits = 0;
+        for (int i=4; i<12; ++i) {
+            gidbits <<= 8;
+            long tall = gid[15 - i] & 0xFFl;
+            assert(tall >= 0 && tall <= 255);
+            gidbits |= tall;
+        }
+        return gidbits & gidMask;
+    }
+
+    /**
+     * Get the bucket id for a given document.
+     *
+     * @param doc The doc.
+     * @return The bucket id.
+     */
+    public BucketId getBucketId(DocumentId doc) {
+        long location = doc.getScheme().getLocation();
+        byte[] gid = doc.getGlobalId();
+
+        long gidContribution = getGidContribution(gid);
+
+        IdString.GidModifier gm = doc.getScheme().getGidModifier();
+        if (gm != null && gm.usedBits != 0) {
+            gidContribution &= (0xFFFFFFFFFFFFFFFFl << (gm.usedBits + getLocationBitCount()));
+            gidContribution |= (gm.value << getLocationBitCount());
+        }
+
+        return new BucketId(64 - BucketId.COUNT_BITS, initialCount | (gidMask & gidContribution) | (locationMask & location));
+    }
+
+}
diff --git a/document/src/main/java/com/yahoo/document/CollectionDataType.java b/document/src/main/java/com/yahoo/document/CollectionDataType.java
new file mode 100644
index 00000000000..87f1f2947bf
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/CollectionDataType.java
@@ -0,0 +1,87 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import com.yahoo.document.datatypes.CollectionFieldValue;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.vespa.objects.Ids;
+import com.yahoo.vespa.objects.ObjectVisitor;
+
+import java.util.List;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public abstract class CollectionDataType extends DataType {
+    // The global class identifier shared with C++.
+    public static int classId = registerClass(Ids.document + 53, CollectionDataType.class);
+
+    private DataType nestedType;
+
+    protected CollectionDataType(String name, int code, DataType nestedType) {
+        super(name, code);
+        this.nestedType = nestedType;
+    }
+
+    @Override
+    public abstract CollectionFieldValue createFieldValue();
+
+    @Override
+    public CollectionDataType clone() {
+        CollectionDataType type = (CollectionDataType) super.clone();
+        type.nestedType = nestedType.clone();
+        return type;
+    }
+
+    @SuppressWarnings("deprecation")
+    public DataType getNestedType() {
+        return nestedType;
+    }
+
+    @Override
+    protected FieldValue createByReflection(Object arg) { return null; }
+
+    /**
+     * Sets the nested type of this CollectionDataType.&nbsp;WARNING! Do not use! Only to be used by config system!
+     */
+    public void setNestedType(DataType nestedType) {
+        this.nestedType = nestedType;
+    }
+
+    @Override
+    public PrimitiveDataType getPrimitiveType() {
+        return nestedType.getPrimitiveType();
+    }
+
+    @Override
+    public boolean isValueCompatible(FieldValue value) {
+        if (!(value instanceof CollectionFieldValue)) {
+            return false;
+        }
+        CollectionFieldValue cfv = (CollectionFieldValue) value;
+        if (equals(cfv.getDataType())) {
+            //the field value if of this type:
+            return true;
+        }
+        return false;
+    }
+
+    @Override
+    protected void register(DocumentTypeManager manager, List<DataType> seenTypes) {
+        seenTypes.add(this);
+        if (!seenTypes.contains(getNestedType())) {
+            //we haven't seen this one before, register it:
+            getNestedType().register(manager, seenTypes);
+        }
+        super.register(manager, seenTypes);
+    }
+
+    @Override
+    public void visitMembers(ObjectVisitor visitor) {
+        super.visitMembers(visitor);
+        visitor.visit("nestedType", nestedType);
+    }
+
+    @Override
+    public boolean isMultivalue() { return true; }
+
+}
diff --git a/document/src/main/java/com/yahoo/document/CompressionConfig.java b/document/src/main/java/com/yahoo/document/CompressionConfig.java
new file mode 100644
index 00000000000..c827ea23b03
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/CompressionConfig.java
@@ -0,0 +1,40 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import com.yahoo.compress.CompressionType;
+
+import java.io.Serializable;
+
+public class CompressionConfig implements Serializable {
+
+    public CompressionConfig(CompressionType type,
+                             int level,
+                             float threshold,
+                             long minSize)
+    {
+        this.type = type;
+        this.compressionLevel = level;
+        this.threshold = threshold;
+        this.minsize = minSize;
+    }
+
+    public CompressionConfig() {
+        this(CompressionType.NONE, 9, 95, 0);
+    }
+
+    public CompressionConfig(CompressionType type) {
+        this(type, 9, 95, 0);
+    }
+
+    public CompressionConfig(CompressionType type, int level, float threshold) {
+        this(type, level, threshold, 0);
+    }
+
+    public final CompressionType type;
+    public int compressionLevel;
+    public float threshold;
+    public final long minsize;
+
+    /** get a multiplier for comparing compressed and original size */
+    public float thresholdFactor() { return 0.01f * threshold; }
+}
diff --git a/document/src/main/java/com/yahoo/document/DataType.java b/document/src/main/java/com/yahoo/document/DataType.java
new file mode 100644
index 00000000000..6e6103c61fd
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/DataType.java
@@ -0,0 +1,325 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import com.yahoo.collections.Pair;
+import com.yahoo.concurrent.CopyOnWriteHashMap;
+import com.yahoo.document.datatypes.ByteFieldValue;
+import com.yahoo.document.datatypes.DoubleFieldValue;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.FloatFieldValue;
+import com.yahoo.document.datatypes.IntegerFieldValue;
+import com.yahoo.document.datatypes.LongFieldValue;
+import com.yahoo.document.datatypes.PredicateFieldValue;
+import com.yahoo.document.datatypes.Raw;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.document.datatypes.TensorFieldValue;
+import com.yahoo.document.datatypes.UriFieldValue;
+import com.yahoo.vespa.objects.Identifiable;
+import com.yahoo.vespa.objects.Ids;
+import com.yahoo.vespa.objects.ObjectVisitor;
+
+import java.io.Serializable;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+import java.util.LinkedList;
+import java.util.List;
+
+/**
+ * Enumeration of the possible types of fields. Since arrays and weighted sets may be defined for any types, including
+ * themselves, this enumeration is open ended.
+ *
+ * @author bratseth
+ */
+public abstract class DataType extends Identifiable implements Serializable, Comparable<DataType> {
+
+    // The global class identifier shared with C++.
+    public static int classId = registerClass(Ids.document + 50, DataType.class);
+
+    // NOTE: These types are also defined in
+    // document/src/vespa/document/datatype/datatype.h
+    // Changes here must also be done there
+
+    public final static NumericDataType NONE = new NumericDataType("none", -1, IntegerFieldValue.class, IntegerFieldValue.getFactory());
+    public final static NumericDataType INT = new NumericDataType("int", 0, IntegerFieldValue.class, IntegerFieldValue.getFactory());
+    public final static NumericDataType FLOAT = new NumericDataType("float", 1, FloatFieldValue.class, FloatFieldValue.getFactory());
+    public final static PrimitiveDataType STRING = new PrimitiveDataType("string", 2, StringFieldValue.class, StringFieldValue.getFactory());
+    public final static PrimitiveDataType RAW = new PrimitiveDataType("raw", 3, Raw.class, Raw.getFactory());
+    public final static NumericDataType LONG = new NumericDataType("long", 4, LongFieldValue.class, LongFieldValue.getFactory());
+    public final static NumericDataType DOUBLE = new NumericDataType("double", 5, DoubleFieldValue.class, DoubleFieldValue.getFactory());
+    // ARRAY is type 6, but never used, array IDs are generated
+    // public final static PrimitiveDataType FIELDMAP = new PrimitiveDataType("FieldMap", 7, FieldMap.class);
+    public final static DocumentType DOCUMENT = new DocumentType("document");
+    // Not used anymore : public final static NumericDataType TIMESTAMP = new NumericDataType("Timestamp", 9, LongFieldValue.class);
+    public final static PrimitiveDataType URI = new PrimitiveDataType("uri", 10, UriFieldValue.class, new UriFieldValue.Factory());
+    // Not used anymore : public final static PrimitiveDataType EXACTSTRING = new PrimitiveDataType("ExactString", 11, StringFieldValue.class);
+    // Not used anymore: public final static PrimitiveDataType CONTENT = new PrimitiveDataType("content", 12, Content.class, new Content.Factory());
+    public final static NumericDataType BYTE = new NumericDataType("byte", 16, ByteFieldValue.class, ByteFieldValue.getFactory());
+    // WEIGHTEDSET is type 17, but never used, weighted set IDs are generated
+    // Tags are converted to weightedset&lt;string&gt; when reading the search definition
+    public final static WeightedSetDataType TAG = new WeightedSetDataType(DataType.STRING, true, true);
+    // Not yet, just reserve id 19. public final static NumericDataType SHORT = new NumericDataType("Int", 19, ShortFieldValue.class);
+    // Guess I'll say STRUCT is 19 though, although I never intend to use it for anything as it has to be autogenerated now..
+    // Let's say that AnnotationReference is 20, but those types will be generated from AnnotationReferenceDataType
+    public final static PrimitiveDataType PREDICATE = new PrimitiveDataType("predicate", 20, PredicateFieldValue.class, PredicateFieldValue.getFactory());
+    public final static PrimitiveDataType TENSOR = new PrimitiveDataType("tensor", 21, TensorFieldValue.class, TensorFieldValue.getFactory());
+
+    public static int lastPredefinedDataTypeId() {
+        return 21;
+    }
+
+    /**
+     * Set to true when this type is registered in a type manager. From that time we should refuse changes.
+     */
+    private boolean registered = false;
+
+    private String name;
+
+    /**
+     * The id of this type
+     */
+    private int dataTypeId;
+
+    static final private CopyOnWriteHashMap<Pair, Constructor> constructorCache = new CopyOnWriteHashMap<>();
+    /**
+     * Creates a datatype
+     *
+     * @param name       the name of the type
+     * @param dataTypeId the id of the type
+     */
+    protected DataType(java.lang.String name, int dataTypeId) {
+        this.name = name;
+        this.dataTypeId = dataTypeId;
+    }
+
+    @SuppressWarnings("CloneDoesntDeclareCloneNotSupportedException")
+    public DataType clone() {
+        return (DataType)super.clone();
+    }
+
+    public void setRegistered() {
+        registered = true;
+    }
+
+    public boolean isRegistered() {
+        return registered;
+    }
+
+    /**
+     * Creates a new, empty FieldValue of this type.
+     *
+     * @return a new, empty FieldValue of this type.
+     */
+    public abstract FieldValue createFieldValue();
+
+    /**
+     * This will try to create the object by reflection. This can be very expensive
+     * so some might discourage that.
+     * @param arg The constructor argument.
+     * @return Fully constructed value.
+     */
+    protected FieldValue createByReflection(Object arg) {
+        Class<?> valClass = getValueClass();
+        if (valClass != null) {
+            Pair<Class<?>, Class<?>> key = new Pair<>(valClass, arg.getClass());
+            Constructor<?> cstr = constructorCache.get(key);
+            try {
+                if (cstr == null) {
+                    cstr = valClass.getConstructor(key.getSecond());
+                    constructorCache.put(key, cstr);
+                }
+                return (FieldValue)cstr.newInstance(arg);
+            } catch (ReflectiveOperationException e) {
+                // Only rethrow exceptions coming from the underlying FieldValue constructor.
+                if (e instanceof InvocationTargetException) {
+                    throw new IllegalArgumentException(e.getCause().getMessage(), e.getCause());
+                }
+            }
+        }
+        return null;
+    }
+
+    /**
+     * Creates a new FieldValue of this type, with the given value.
+     *
+     * @param arg the value that the new FieldValue shall have.
+     * @return A new FieldValue of this type, with the given value.
+     */
+    public FieldValue createFieldValue(Object arg) {
+        if (arg == null) {
+            return createFieldValue();
+        }
+        FieldValue val = createByReflection(arg);
+        if (val == null) {
+            val = createFieldValue();
+            if (val != null) {
+                val.assign(arg);
+            }
+        }
+        return val;
+    }
+
+    public abstract Class getValueClass();
+
+    public abstract boolean isValueCompatible(FieldValue value);
+
+    public final boolean isAssignableFrom(DataType dataType) {
+        // TODO: Reverse this so that isValueCompatible() uses this instead.
+        return isValueCompatible(dataType.createFieldValue());
+    }
+
+    /**
+     * Returns an array datatype, where the array elements are of the given type
+     *
+     * @param type the type to create an array of
+     * @return the array data type
+     */
+    public static ArrayDataType getArray(DataType type) {
+        return new ArrayDataType(type);
+    }
+
+    /**
+     * Returns a map datatype
+     *
+     * @param key   the key type
+     * @param value the value type
+     * @return the map data type
+     */
+    public static MapDataType getMap(DataType key, DataType value) {
+        return new MapDataType(key, value);
+    }
+
+    /**
+     * Returns a weighted set datatype, where the elements are of the given type
+     *
+     * @param type the type to create a weighted set of
+     * @return the weighted set data type
+     */
+    public static WeightedSetDataType getWeightedSet(DataType type) {
+        return getWeightedSet(type, false, false);
+    }
+
+    /**
+     * Returns a weighted set datatype, where the elements are of the given type, and which supports the properties
+     * createIfNonExistent and removeIfZero
+     *
+     * @param type                the type to create a weighted set of
+     * @param createIfNonExistent whether the type has createIfNonExistent
+     * @param removeIfZero        whether the type has removeIfZero
+     * @return the weighted set data type
+     */
+    public static WeightedSetDataType getWeightedSet(DataType type, boolean createIfNonExistent, boolean removeIfZero) {
+        return new WeightedSetDataType(type, createIfNonExistent, removeIfZero);
+    }
+
+    public java.lang.String getName() {
+        return name;
+    }
+
+    /**
+     * Sets the name of this data type.&nbsp;WARNING! Do not use!
+     *
+     * @param name the name of this data type.
+     */
+    protected void setName(String name) {
+        this.name = name;
+    }
+
+    public int getId() {
+        return dataTypeId;
+    }
+
+    /**
+     * Sets the ID of this data type.&nbsp;WARNING! Do not use!
+     *
+     * @param id the ID of this data type.
+     */
+    public void setId(int id) {
+        dataTypeId = id;
+    }
+
+    /**
+     * Registeres this type in the given document manager.
+     *
+     * @param manager the DocumentTypeManager to register in.
+     */
+    public final void register(DocumentTypeManager manager) {
+        register(manager, new LinkedList<>());
+    }
+
+    protected void register(DocumentTypeManager manager, List<DataType> seenTypes) {
+        manager.registerSingleType(this);
+    }
+
+    public int hashCode() {
+        return name.hashCode();
+    }
+
+    public boolean equals(Object other) {
+        if (!(other instanceof DataType)) {
+            return false;
+        }
+        DataType type = (DataType)other;
+        return (name.equals(type.name) && dataTypeId == type.dataTypeId);
+    }
+
+    public java.lang.String toString() {
+        return "datatype " + name + " (code: " + dataTypeId + ")";
+    }
+
+    public int getCode() {
+        return dataTypeId;
+    }
+
+    /**
+     * Creates a field path from the given field path string.
+     *
+     * @param fieldPathString a string containing the field path
+     * @return Returns a valid field path, parsed from the string
+     */
+    public FieldPath buildFieldPath(String fieldPathString) {
+        if (fieldPathString.length() > 0) {
+            throw new IllegalArgumentException(
+                    "Datatype " + toString() + " does not support further recursive structure: " + fieldPathString);
+        }
+        return new FieldPath();
+    }
+
+    /**
+     * Returns the primitive datatype associated with this datatype, i.e. the type itself if this is a
+     * PrimitiveDataType, the nested type if this is a CollectionDataType or null for all other cases
+     *
+     * @return primitive data type, or null
+     */
+    public PrimitiveDataType getPrimitiveType() {
+        return null;
+    }
+
+    @Override
+    public void visitMembers(ObjectVisitor visitor) {
+        super.visitMembers(visitor);
+        visitor.visit("name", name);
+        visitor.visit("id", dataTypeId);
+    }
+
+    /**
+     * Utility function for parsing field paths.
+     */
+    static String skipDotInString(String remaining, int endPos) {
+        if (remaining.length() < endPos + 2) {
+            return "";
+        } else if (remaining.charAt(endPos + 1) == '.') {
+            return remaining.substring(endPos + 2);
+        } else {
+            return remaining.substring(endPos + 1);
+        }
+    }
+
+    @Override
+    public int compareTo(DataType dataType) {
+        return Integer.valueOf(dataTypeId).compareTo(dataType.dataTypeId);
+    }
+
+    /** Returns whether this is a multivalue type, i.e either a CollectionDataType or a MapDataType */
+    public boolean isMultivalue() { return false; }
+
+}
diff --git a/document/src/main/java/com/yahoo/document/DataTypeName.java b/document/src/main/java/com/yahoo/document/DataTypeName.java
new file mode 100644
index 00000000000..b7d5be5c5c0
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/DataTypeName.java
@@ -0,0 +1,51 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import com.yahoo.text.Lowercase;
+import com.yahoo.text.Utf8Array;
+import com.yahoo.text.Utf8String;
+
+import java.io.Serializable;
+
+/**
+ * A full document type name, consisting of a <i>name</i> and a <i>version</i>. The name is case insensitive, and the
+ * version must be a positive integer or 0. This is a <i>value object</i>.
+ *
+ * @author bratseth
+ */
+public final class DataTypeName implements Serializable {
+
+    private final Utf8String name;
+
+    /**
+     * Creates a document name from a string of the form "name"
+     *
+     * @param name The name string to parse.
+     * @throws NumberFormatException if the version part of the name is present but is not a number
+     */
+    public DataTypeName(String name) {
+        this.name = new Utf8String(name);
+    }
+    public DataTypeName(Utf8Array name) {
+        this.name = new Utf8String(name);
+    }
+    public DataTypeName(Utf8String name) {
+        this.name = new Utf8String(name);
+    }
+
+    public String getName() { return name.toString(); }
+
+    @Override
+    public String toString() { return name.toString(); }
+
+    @Override
+    public int hashCode() { return name.hashCode(); }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (!(obj instanceof DataTypeName)) return false;
+        DataTypeName datatype = (DataTypeName)obj;
+        return this.name.equals(datatype.name);
+    }
+
+}
diff --git a/document/src/main/java/com/yahoo/document/Document.java b/document/src/main/java/com/yahoo/document/Document.java
new file mode 100644
index 00000000000..740c91c5c1b
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/Document.java
@@ -0,0 +1,397 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.Struct;
+import com.yahoo.document.datatypes.StructuredFieldValue;
+import com.yahoo.document.serialization.*;
+import com.yahoo.io.GrowableByteBuffer;
+import com.yahoo.vespa.objects.BufferSerializer;
+import com.yahoo.vespa.objects.Ids;
+import com.yahoo.vespa.objects.Serializer;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.Iterator;
+import java.util.Map;
+
+/**
+ * A document is an identifiable
+ * set of value bindings of a {@link DocumentType document type}.
+ * A document represents an instance of some entity of interest
+ * in an application, like an article, a web document, a product, etc.
+ *
+ * Deprecation: Try to use document set and get methods only with FieldValue types,
+ * not with primitive types. Support for direct access to primitive types will
+ * be removed soon.
+ *
+ * @author <a href="bratseth@yahoo-inc.com">Jon S Bratseth</a>
+ * @author <a href="einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class Document extends StructuredFieldValue {
+
+    public static final int classId = registerClass(Ids.document + 3, Document.class);
+    public static final short SERIALIZED_VERSION = 8;
+    private DocumentId docId;
+    private Struct header;
+    private Struct body;
+    private Long lastModified = null;
+
+    /**
+     * Create a document with the given document type and identifier.
+     * @param docType DocumentType to use for creation
+     * @param id The id for this document
+     */
+    public Document(DocumentType docType, String id) {
+        this(docType, new DocumentId(id));
+    }
+
+    /**
+     * Create a document with the given document type and identifier.
+     * @param docType DocumentType to use for creation
+     * @param id The id for this document
+     */
+    public Document(DocumentType docType, DocumentId id) {
+        super(docType);
+        setNewType(docType);
+        internalSetId(id, docType);
+    }
+
+    /**
+     * Creates a document that is a shallow copy of another.
+     *
+     * @param doc The document to copy.
+     */
+    public Document(Document doc) {
+        this(doc.getDataType(), doc.getId());
+        header = doc.header;
+        body = doc.body;
+        lastModified = doc.lastModified;
+    }
+
+    /**
+     *
+     * @param reader The deserializer to use for creating this document
+     */
+    public Document(DocumentReader reader) {
+        super(null);
+        reader.read(this);
+    }
+
+    public DocumentId getId() { return docId; }
+    public void setId(DocumentId id) { internalSetId(id, getDataType()); }
+    private void internalSetId(DocumentId id, DocumentType docType) {
+        if (id != null && id.hasDocType() && docType != null && !id.getDocType().equals(docType.getName())) {
+            throw new IllegalArgumentException("Trying to set a document id (type " + id.getDocType() +
+                                               ") that don't match the document type (" + getDataType().getName() + ").");
+        }
+        docId = id;
+    }
+
+    public Struct getHeader() { return header; }
+    public Struct getBody() { return body; }
+
+    @Override
+    public void assign(Object o) {
+        throw new IllegalArgumentException("Assign not implemented for " + getClass() + " objects");
+    }
+
+    @Override
+    public Document clone() {
+        Document doc = (Document) super.clone();
+        doc.docId = docId.clone();
+        doc.header = header.clone();
+        doc.body = body.clone();
+        return doc;
+    }
+
+    private void setNewType(DocumentType type) {
+        header = type.getHeaderType().createFieldValue();
+        body = type.getBodyType().createFieldValue();
+    }
+
+    public void setDataType(DataType type) {
+        if (docId != null && docId.hasDocType() && !docId.getDocType().equals(type.getName())) {
+            throw new IllegalArgumentException("Trying to set a document type (" + type.getName() +
+                                               ") that don't match the document id (" + docId + ").");
+        }
+        super.setDataType(type);
+        setNewType((DocumentType)type);
+    }
+
+    public int getSerializedSize() throws SerializationException {
+        DocumentSerializer data = DocumentSerializerFactory.create42(new GrowableByteBuffer(64 * 1024, 2.0f));
+        data.write(this);
+        return data.getBuf().position();
+    }
+
+    /**
+     *  This is an approximation of serialized size. We just set it to 4096 as a definition of a medium document.
+     * @return Approximate size of document (4096)
+     */
+    public final int getApproxSize() { return 4096; }
+
+    public void serialize(OutputStream out) throws SerializationException {
+        DocumentSerializer writer = DocumentSerializerFactory.create42(new GrowableByteBuffer(64 * 1024, 2.0f));
+        writer.write(this);
+        GrowableByteBuffer data = writer.getBuf();
+        byte[] array;
+        if (data.hasArray()) {
+            //just get the array
+            array = data.array();
+        } else {
+            //copy the bytebuffer into the array
+            array = new byte[data.position()];
+            int endPos = data.position();
+            data.position(0);
+            data.get(array);
+            data.position(endPos);
+        }
+        try {
+            out.write(array, 0, data.position());
+        } catch (IOException ioe) {
+            throw new SerializationException(ioe);
+        }
+    }
+
+    public static Document createDocument(DocumentReader buffer) {
+        return new Document(buffer);
+    }
+
+    @Override
+    public Field getField(String fieldName) {
+        Field field = header.getField(fieldName);
+        if (field == null) {
+            field = body.getField(fieldName);
+        }
+        if (field == null) {
+            for(DocumentType parent : getDataType().getInheritedTypes()) {
+                field = parent.getField(fieldName);
+                if (field != null) {
+                    break;
+                }
+            }
+        }
+        return field;
+    }
+
+    @Override
+    public FieldValue getFieldValue(Field field) {
+        if (field.isHeader()) {
+            return header.getFieldValue(field);
+        } else {
+            return body.getFieldValue(field);
+        }
+    }
+
+    @Override
+    protected void doSetFieldValue(Field field, FieldValue value) {
+        if (field.isHeader()) {
+            header.setFieldValue(field, value);
+        } else {
+            body.setFieldValue(field, value);
+        }
+    }
+
+    @Override
+    public FieldValue removeFieldValue(Field field) {
+        if (field.isHeader()) {
+            return header.removeFieldValue(field);
+        } else {
+            return body.removeFieldValue(field);
+        }
+    }
+
+    @Override
+    public void clear() {
+        header.clear();
+        body.clear();
+    }
+
+    @Override
+    public Iterator<Map.Entry<Field, FieldValue>> iterator() {
+        return new Iterator<Map.Entry<Field, FieldValue>>() {
+
+            private Iterator<Map.Entry<Field, FieldValue>> headerIt = header.iterator();
+            private Iterator<Map.Entry<Field, FieldValue>> bodyIt = body.iterator();
+
+            public boolean hasNext() {
+                if (headerIt != null) {
+                    if (headerIt.hasNext()) {
+                        return true;
+                    } else {
+                        headerIt = null;
+                    }
+                }
+                return bodyIt.hasNext();
+            }
+
+            public Map.Entry<Field, FieldValue> next() {
+                return (headerIt == null ? bodyIt.next() : headerIt.next());
+            }
+
+            public void remove() {
+                if (headerIt == null) {
+                    bodyIt.remove();
+                } else {
+                    headerIt.remove();
+                }
+            }
+        };
+    }
+
+    public String toString() {
+        return "document '" + String.valueOf(docId) + "' of type '" + getDataType().getName() + "'";
+    }
+
+    public String toXML(String indent) {
+        XmlStream xml = new XmlStream();
+        xml.setIndent(indent);
+        xml.beginTag("document");
+        printXml(xml);
+        xml.endTag();
+        return xml.toString();
+    }
+
+    /**
+     * Get XML representation of the document root and its children, contained
+     * within a &lt;document&gt;&lt;/document&gt; tag.
+     * @return XML representation of document
+     */
+    public String toXml() {
+        return toXML("  ");
+    }
+
+    public void printXml(XmlStream xml) {
+        XmlSerializationHelper.printDocumentXml(this, xml);
+    }
+
+    /** Returns true if the argument is a document which has the same set of values */
+    public boolean equals(Object o) {
+        if (!(o instanceof Document)) return false;
+        Document other = (Document) o;
+        return (super.equals(o) && docId.equals(other.docId) &&
+                header.equals(other.header) && body.equals(other.body));
+    }
+
+    @Override
+    public int hashCode() {
+        return 31 * super.hashCode() + (docId != null ? docId.hashCode() : 0);
+    }
+
+    /**
+     * Returns the last modified time of this Document, when stored in persistent storage. This is typically set by the
+     * library that retrieves the Document from persistent storage.
+     *
+     * This variable doesn't really belong in document. It is used when retrieving docblocks of documents to be able to
+     * see when documents was last modified in VDS, without having to add modified times separate in the API.
+     *
+     * NOTE: This is a transient field, and will not be serialized with a Document (will be null after deserialization).
+     *
+     * @return the last modified time of this Document (in milliseconds), or null if unset
+     */
+    public Long getLastModified() {
+        return lastModified;
+    }
+
+    /**
+     * Sets the last modified time of this Document. This is typically set by the library that retrieves the
+     * Document from persistent storage, and should not be set by arbitrary clients. NOTE: This is a
+     * transient field, and will not be serialized with a Document (will be null after deserialization).
+     *
+     * @param lastModified the last modified time of this Document (in milliseconds)
+     */
+    public void setLastModified(Long lastModified) {
+        this.lastModified = lastModified;
+    }
+
+    public void onSerialize(Serializer data) throws SerializationException {
+        serialize((DocumentWriter)data);
+    }
+
+    public void serializeHeader(Serializer data) throws SerializationException {
+        if (data instanceof DocumentWriter) {
+            if (data instanceof VespaDocumentSerializer42) {
+                ((VespaDocumentSerializer42)data).setHeaderOnly(true);
+            }
+            serialize((DocumentWriter)data);
+        } else if (data instanceof BufferSerializer) {
+            serialize(DocumentSerializerFactory.create42(((BufferSerializer) data).getBuf(), true));
+        } else {
+            DocumentSerializer fw = DocumentSerializerFactory.create42(new GrowableByteBuffer(), true);
+            serialize(fw);
+            data.put(null, fw.getBuf().getByteBuffer());
+        }
+    }
+
+    public void serializeBody(Serializer data) throws SerializationException {
+        if (getBody().getFieldCount() > 0) {
+            if (data instanceof FieldWriter) {
+                getBody().serialize(new Field("body", getBody().getDataType()), (FieldWriter) data);
+            } else if (data instanceof BufferSerializer) {
+                getBody().serialize(new Field("body", getBody().getDataType()), DocumentSerializerFactory.create42(((BufferSerializer) data).getBuf()));
+            } else {
+                DocumentSerializer fw = DocumentSerializerFactory.create42(new GrowableByteBuffer());
+                getBody().serialize(new Field("body", getBody().getDataType()), fw);
+                data.put(null, fw.getBuf().getByteBuffer());
+            }
+        }
+    }
+
+    @Override
+    public DocumentType getDataType() {
+        return (DocumentType)super.getDataType();
+    }
+
+    @Override
+    public int getFieldCount() {
+        return header.getFieldCount() + body.getFieldCount();
+    }
+
+    public void serialize(DocumentWriter writer) {
+        writer.write(this);
+    }
+
+    public void deserialize(DocumentReader reader) {
+        reader.read(this);
+    }
+
+    @Override
+    public void serialize(Field field, FieldWriter writer) {
+        writer.write(field, this);
+    }
+
+    /* (non-Javadoc)
+     * @see com.yahoo.document.datatypes.FieldValue#deserialize(com.yahoo.document.Field, com.yahoo.document.serialization.FieldReader)
+     */
+    @Override
+    public void deserialize(Field field, FieldReader reader) {
+        reader.read(field, this);
+    }
+
+    @Override
+    public int compareTo(FieldValue fieldValue) {
+        int comp = super.compareTo(fieldValue);
+
+        if (comp != 0) {
+            return comp;
+        }
+
+        //types are equal, this must be of this type
+        Document otherValue = (Document) fieldValue;
+        comp = getId().compareTo(otherValue.getId());
+
+        if (comp != 0) {
+            return comp;
+        }
+
+        comp = header.compareTo(otherValue.header);
+
+        if (comp != 0) {
+            return comp;
+        }
+
+        comp = body.compareTo(otherValue.body);
+        return comp;
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/DocumentCalculator.java b/document/src/main/java/com/yahoo/document/DocumentCalculator.java
new file mode 100644
index 00000000000..312f72e432d
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/DocumentCalculator.java
@@ -0,0 +1,39 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import com.yahoo.document.select.Context;
+import com.yahoo.document.select.parser.ParseException;
+import com.yahoo.document.select.parser.SelectInput;
+import com.yahoo.document.select.parser.SelectParser;
+import com.yahoo.document.select.rule.ComparisonNode;
+
+import java.util.Map;
+
+/**
+ * @author <a href="mailto:thomasg@yahoo-inc.com">Thomas Gundersen</a>
+ */
+public class DocumentCalculator {
+
+    private ComparisonNode comparison;
+
+    public DocumentCalculator(String expression) throws ParseException {
+        SelectParser parser = new SelectParser(new SelectInput(expression + " == 0"));
+        comparison = (ComparisonNode)parser.expression();
+    }
+
+    public Number evaluate(Document doc, Map<String, Object> variables) {
+        Context context = new Context(new DocumentPut(doc));
+        context.setVariables(variables);
+
+        try {
+            Object o = comparison.getLHS().evaluate(context);
+
+            if (Double.isInfinite(((Number)o).doubleValue())) {
+                throw new IllegalArgumentException("Expression evaluated to an infinite number");
+            }
+            return ((Number)o).doubleValue();
+        } catch (ArithmeticException e) {
+            throw new IllegalArgumentException("Arithmetic exception " + e.getMessage(), e);
+        }
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/DocumentId.java b/document/src/main/java/com/yahoo/document/DocumentId.java
new file mode 100644
index 00000000000..59650ea23f0
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/DocumentId.java
@@ -0,0 +1,109 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import com.yahoo.document.idstring.IdString;
+import com.yahoo.document.serialization.*;
+import com.yahoo.vespa.objects.Deserializer;
+import com.yahoo.vespa.objects.Identifiable;
+import com.yahoo.vespa.objects.Serializer;
+
+import java.io.Serializable;
+
+/**
+ * The id of a document
+ */
+public class DocumentId extends Identifiable implements Serializable {
+
+    private IdString id;
+    private GlobalId globalId;
+
+    /**
+     * Constructor used for deserialization.
+     */
+    public DocumentId(Deserializer buf) {
+        deserialize(buf);
+    }
+
+    /**
+     * Constructor. This constructor is used if the DocumentId is used outside of a Document object, but we have the
+     * URI.
+     *
+     * @param id Associate with this URI, storage address etc. is not applicable.
+     */
+    public DocumentId(String id) {
+        if (id == null) {
+            throw new IllegalArgumentException("Cannot create DocumentId from null id.");
+        }
+        this.id = IdString.createIdString(id);
+        globalId = null;
+    }
+
+    public DocumentId(IdString id) {
+        this.id = id;
+        globalId = null;
+    }
+
+    @Override
+    public DocumentId clone() {
+        DocumentId docId =  (DocumentId)super.clone();
+        return docId;
+    }
+
+    public void setId(IdString id) {
+        this.id = id;
+    }
+
+    public IdString getScheme() {
+        return id;
+    }
+
+    public byte[] getGlobalId() {
+        if (globalId == null) {
+            globalId = new GlobalId(id);
+        }
+        return globalId.getRawId();
+    }
+
+    public int compareTo(Object o) {
+        DocumentId cmp = (DocumentId)o;
+        return id.toString().compareTo(cmp.id.toString());
+    }
+
+    public boolean equals(Object o) {
+        return o instanceof DocumentId && id.equals(((DocumentId)o).id);
+    }
+
+    public int hashCode() {
+        return id.hashCode();
+    }
+
+    public String toString() {
+        return id.toString();
+    }
+
+    @Override
+    public void onSerialize(Serializer target) throws SerializationException {
+        if (target instanceof DocumentWriter) {
+            ((DocumentWriter)target).write(this);
+        } else {
+            target.put(null, id.toString());
+        }
+    }
+
+
+    public void onDeserialize(Deserializer data) throws DeserializationException {
+        if (data instanceof DocumentReader) {
+            id = ((DocumentReader)data).readDocumentId().getScheme();
+        } else {
+            id = IdString.createIdString(data.getString(null));
+        }
+    }
+
+    public boolean hasDocType() {
+        return id.hasDocType();
+    }
+
+    public String getDocType() {
+        return id.getDocType();
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/DocumentOperation.java b/document/src/main/java/com/yahoo/document/DocumentOperation.java
new file mode 100644
index 00000000000..250e780e65a
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/DocumentOperation.java
@@ -0,0 +1,39 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import com.google.common.annotations.Beta;
+
+/**
+ * Base class for "document operations".
+ * These include "put" (DocumentPut), "update" (DocumentUpdate), and "remove" (DocumentRemove).
+ * Historically, put operations were represented by the Document class alone,
+ * but since it doesn't make much sense to put a *test and set* condition in Document,
+ * a more uniform interface for document operations was needed.
+ *
+ * @author Vegard Sjonfjell
+ */
+public abstract class DocumentOperation {
+
+    private TestAndSetCondition condition = TestAndSetCondition.NOT_PRESENT_CONDITION;
+
+    public abstract DocumentId getId();
+
+    public void setCondition(TestAndSetCondition condition) {
+        this.condition = condition;
+    }
+
+    public TestAndSetCondition getCondition() {
+        return condition;
+    }
+
+    protected DocumentOperation() {}
+
+    /**
+     * Copy constructor
+     * @param other DocumentOperation to copy
+     */
+    protected DocumentOperation(DocumentOperation other) {
+        this.condition = other.condition;
+    }
+
+}
diff --git a/document/src/main/java/com/yahoo/document/DocumentPut.java b/document/src/main/java/com/yahoo/document/DocumentPut.java
new file mode 100644
index 00000000000..f02d1e6d6d8
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/DocumentPut.java
@@ -0,0 +1,48 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+/**
+ * @author Vegard Sjonfjell
+ */
+public class DocumentPut extends DocumentOperation {
+
+    private final Document document;
+
+    public DocumentPut(Document document) {
+        this.document = document;
+    }
+
+    public DocumentPut(DocumentType docType, DocumentId docId) {
+        this.document = new Document(docType, docId);
+    }
+
+    public DocumentPut(DocumentType docType, String docId) {
+        this.document = new Document(docType, docId);
+    }
+
+    public Document getDocument() {
+        return document;
+    }
+
+    public DocumentId getId() {
+        return document.getId();
+    }
+
+    /**
+     * Copy constructor
+     * @param other DocumentPut to copy
+     */
+    public DocumentPut(DocumentPut other) {
+        super(other);
+        this.document = new Document(other.getDocument());
+    }
+
+    /**
+     * Base this DocumentPut on another, but use newDocument as the Document.
+     */
+    public DocumentPut(DocumentPut other, Document newDocument) {
+        super(other);
+        this.document = newDocument;
+    }
+
+}
diff --git a/document/src/main/java/com/yahoo/document/DocumentRemove.java b/document/src/main/java/com/yahoo/document/DocumentRemove.java
new file mode 100644
index 00000000000..8d4f37d5583
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/DocumentRemove.java
@@ -0,0 +1,35 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+/**
+ * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a>
+ */
+public class DocumentRemove extends DocumentOperation {
+
+    private final DocumentId docId;
+
+    public DocumentRemove(DocumentId docId) { this.docId = docId; }
+
+    @Override
+    public DocumentId getId() { return docId; }
+
+    @Override
+    public String toString() {
+        return "DocumentRemove '" + docId + "'";
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (!(o instanceof DocumentRemove)) return false;
+        DocumentRemove that = (DocumentRemove) o;
+        if (!docId.equals(that.docId)) return false;
+        return true;
+    }
+
+    @Override
+    public int hashCode() {
+        return docId.hashCode();
+    }
+
+}
diff --git a/document/src/main/java/com/yahoo/document/DocumentType.java b/document/src/main/java/com/yahoo/document/DocumentType.java
new file mode 100755
index 00000000000..db38228489d
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/DocumentType.java
@@ -0,0 +1,476 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.serialization.DocumentWriter;
+import com.yahoo.vespa.objects.Ids;
+import com.yahoo.vespa.objects.ObjectVisitor;
+import com.yahoo.vespa.objects.Serializer;
+
+import java.util.*;
+
+/**
+ * <p>A document definition is a list of fields. Documents may inherit other documents,
+ * implicitly acquiring their fields as it's own. If a document is not set to inherit
+ * any document, it will always inherit the document "document.0".</p>
+ *
+ * @author <a href="mailto:thomasg@yahoo-inc.com">Thomas Gundersen</a>
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a>
+ */
+public class DocumentType extends StructuredDataType {
+
+    public static final int classId = registerClass(Ids.document + 58, DocumentType.class);
+    private StructDataType headerType;
+    private StructDataType bodyType;
+    private List<DocumentType> inherits = new ArrayList<DocumentType>(1);
+
+    /**
+     * Creates a new document type and registers it with the document type manager.
+     * This will be created as version 0 of this document type.
+     * Implicitly registers this with the document type manager.
+     * The document type id will be generated as a hash from the document type name.
+     *
+     * @param name The name of the new document type
+     */
+    public DocumentType(String name) {
+        this(name, new StructDataType(name + ".header"),
+                   new StructDataType(name + ".body"));
+    }
+
+    /**
+     * Creates a new document type and registers it with the document type manager.
+     * Implicitly registers this with the document type manager.
+     * The document type id will be generated as a hash from the document type name.
+     *
+     * @param name       The name of the new document type
+     * @param headerType The type of the header struct
+     * @param bodyType   The type of the body struct
+     */
+    public DocumentType(String name, StructDataType headerType, StructDataType bodyType) {
+        super(name);
+        this.headerType = headerType;
+        this.bodyType = bodyType;
+    }
+
+    @Override
+    public DocumentType clone() {
+        DocumentType type = (DocumentType) super.clone();
+        type.headerType = headerType.clone();
+        type.bodyType = bodyType.clone();
+        type.inherits = new ArrayList<>(inherits.size());
+        for (DocumentType inherited : inherits) {
+            type.inherits.add(inherited);
+        }
+        return type;
+    }
+
+    @Override
+    public Document createFieldValue() {
+        return new Document(this, (DocumentId) null);
+    }
+
+    @Override
+    public Class getValueClass() {
+        return Document.class;
+    }
+
+    @Override
+    public boolean isValueCompatible(FieldValue value) {
+        if (!(value instanceof Document)) {
+            return false;
+        }
+        Document doc = (Document) value;
+        if (doc.getDataType().inherits(this)) {
+            //the value is of this type; or the supertype of the value is of this type, etc....
+            return true;
+        }
+        return false;
+    }
+
+    public StructDataType getHeaderType() {
+        return headerType;
+    }
+
+    public StructDataType getBodyType() {
+        return bodyType;
+    }
+
+    @Override
+    protected void register(DocumentTypeManager manager, List<DataType> seenTypes) {
+        seenTypes.add(this);
+        for (DocumentType type : getInheritedTypes()) {
+            if (!seenTypes.contains(type)) {
+                type.register(manager, seenTypes);
+            }
+        }
+        // Get parent fields into fields specified in this type
+        StructDataType header = headerType.clone();
+        StructDataType body = bodyType.clone();
+
+        header.clearFields();
+        body.clearFields();
+
+        for (Field field : fieldSet()) {
+            (field.isHeader() ? header : body).addField(field);
+        }
+        headerType.assign(header);
+        bodyType.assign(body);
+
+        if (!seenTypes.contains(headerType)) {
+            headerType.register(manager, seenTypes);
+        }
+        if (!seenTypes.contains(bodyType)) {
+            bodyType.register(manager, seenTypes);
+        }
+        manager.registerSingleType(this);
+    }
+
+    /**
+     * Check if this document type has the given name,
+     * or inherits from a type with that name.
+     */
+    public boolean isA(String docTypeName) {
+        if (getName().equalsIgnoreCase(docTypeName)) {
+            return true;
+        }
+        for (DocumentType parent : inherits) {
+            if (parent.isA(docTypeName)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    /**
+     * Adds an field that can be used with this document type.
+     *
+     * @param field the field to add
+     */
+    public void addField(Field field) {
+        if (isRegistered()) {
+            throw new IllegalStateException("You cannot add fields to a document type that is already registered.");
+        }
+        StructDataType struct = (field.isHeader() ? headerType : bodyType);
+        struct.addField(field);
+    }
+
+    /**
+     * Adds a new body field to this document type and returns the new field object
+     *
+     * @param name The name of the field to add
+     * @param type The datatype of the field to add
+     * @return The field created
+     *         TODO Fix searchdefinition so that exception can be thrown if filed is already registerd.
+     */
+    public Field addField(String name, DataType type) {
+        if (isRegistered()) {
+            throw new IllegalStateException("You cannot add fields to a document type that is already registered.");
+        }
+        Field field = new Field(name, type, false);
+        bodyType.addField(field);
+        return field;
+    }
+
+    /**
+     * Adds a new header field to this document type and returns the new field object
+     *
+     * @param name The name of the field to add
+     * @param type The datatype of the field to add
+     * @return The field created
+     *         TODO Fix searchdefinition so that exception can be thrown if filed is already registerd
+     */
+    public Field addHeaderField(String name, DataType type) {
+        if (isRegistered()) {
+            throw new IllegalStateException("You cannot add fields to a document type that is already registered.");
+        }
+        Field field = new Field(name, type, true);
+        headerType.addField(field);
+        return field;
+    }
+
+    /**
+     * Adds a document to the inherited document types of this.
+     * If this type is already directly inherited, nothing is done
+     *
+     * @param type An already DocumentType object.
+     */
+    public void inherit(DocumentType type) {
+    	//TODO: There is also a check like the following in SDDocumentType addField(), try to move that to this class' addField() to get it proper,
+    	// as this method is called only when the doc types are exported.
+        verifyTypeConsistency(type);
+        if (isRegistered()) {
+            throw new IllegalStateException("You cannot add inheritance to a document type that is already registered.");
+        }
+        if (type == null) {
+            throw new IllegalArgumentException("The document type cannot be null in inherit()");
+        }
+
+        // If it inherits the exact same type
+        if (inherits.contains(type)) return;
+
+        // If we inherit a type, don't inherit the supertype
+        if (inherits.size() == 1 && inherits.get(0).getDataTypeName().equals(new DataTypeName("document"))) {
+            inherits.clear();
+        }
+
+        inherits.add(type);
+    }
+
+    /**
+     * Fail if the subtype changes the type of any equally named field.
+     *
+     * @param superType The supertype to verify against
+     *                  TODO Add strict type checking no duplicate fields are allowed
+     */
+    private void verifyTypeConsistency(DocumentType superType) {
+        for (Field f : fieldSet()) {
+            Field supField = superType.getField(f.getName());
+            if (supField != null) {
+                if (!f.getDataType().equals(supField.getDataType())) {
+                    throw new IllegalArgumentException("Inheritance type mismatch: field \"" + f.getName() +
+                                                       "\" in datatype \"" + getName() + "\"" +
+                                                       " must have same datatype as in parent document type \"" + superType.getName() + "\"");
+                }
+            }
+        }
+    }
+
+    /**
+     * Returns the DocumentNames which are directly inherited by this
+     * as a read-only collection.
+     * If this document type does not explicitly inherit anything, the list will
+     * contain the root type 'Document'
+     *
+     * @return a read-only list iterator containing the name Strings of the directly
+     *         inherited document types of this
+     */
+    public Collection<DocumentType> getInheritedTypes() {
+        return Collections.unmodifiableCollection(inherits);
+    }
+
+    public ListIterator<DataTypeName> inheritedIterator() {
+        List<DataTypeName> names = new ArrayList<>(inherits.size());
+        for (DocumentType type : inherits) {
+            names.add(type.getDataTypeName());
+        }
+        return ImmutableList.copyOf(names).listIterator();
+    }
+
+    /**
+     * Return whether this document type inherits the given document type.
+     *
+     * @param superType The documenttype to check if it inherits.
+     * @return true if it inherits the superType, false if not
+     */
+    public boolean inherits(DocumentType superType) {
+        if (equals(superType)) return true;
+        for (DocumentType type : inherits) {
+            if (type.inherits(superType)) return true;
+        }
+        return false;
+    }
+
+    /**
+     * Gets the field matching a given name.
+     *
+     * @param name The name of a field.
+     * @return Returns the matching field, or null if not found.
+     */
+    public Field getField(String name) {
+        Field field = headerType.getField(name);
+        if (field == null) {
+            field = bodyType.getField(name);
+        }
+        if (field == null && !isRegistered()) {
+            for (DocumentType inheritedType : inherits) {
+                field = inheritedType.getField(name);
+                if (field != null) break;
+            }
+        }
+        return field;
+    }
+
+    @Override
+    public Field getField(int id) {
+        Field field = headerType.getField(id);
+        if (field == null) {
+            field = bodyType.getField(id);
+        }
+        if (field == null && !isRegistered()) {
+            for (DocumentType inheritedType : inherits) {
+                field = inheritedType.getField(id);
+                if (field != null) break;
+            }
+        }
+        return field;
+    }
+
+    /**
+     * Returns whether this type defines the given field name
+     *
+     * @param name The name of the field to check if it has
+     * @return True if there is a field with the given name.
+     */
+    public boolean hasField(String name) {
+        return getField(name) != null;
+    }
+
+    //@Override
+
+
+    public int getFieldCount() {
+        return headerType.getFieldCount() + bodyType.getFieldCount();
+    }
+
+    /**
+     * Gets the field matching a given ID.
+     *
+     * @param id      The ID of a field.
+     * @param version The serialization version of the document.
+     * @return Returns the matching field, or null if not found.
+     */
+    public Field getField(Integer id, int version) {
+        Field field = headerType.getField(id, version);
+        if (field == null) {
+            field = bodyType.getField(id, version);
+        }
+        if (field == null && !isRegistered()) {
+            for (DocumentType inheritedType : inherits) {
+                field = inheritedType.getField(id, version);
+                if (field != null) break;
+            }
+        }
+        return field;
+    }
+
+    /**
+     * Removes an field from the DocumentType.
+     *
+     * @param name The name of the field.
+     * @return The field that was removed or null if it did not exist.
+     */
+    public Field removeField(String name) {
+        if (isRegistered()) {
+            throw new IllegalStateException("You cannot remove fields from a document type that is already registered.");
+        }
+        Field field = headerType.removeField(name);
+        if (field == null) {
+            field = bodyType.removeField(name);
+        }
+        if (field == null) {
+            for (DocumentType inheritedType : inherits) {
+                field = inheritedType.removeField(name);
+                if (field != null) break;
+            }
+        }
+        return field;
+    }
+
+    public Collection<Field> getFields() {
+        Collection<Field> collection = new LinkedList<>();
+
+        for (DocumentType type : inherits) {
+            collection.addAll(type.getFields());
+        }
+
+        collection.addAll(headerType.getFields());
+        collection.addAll(bodyType.getFields());
+        return ImmutableList.copyOf(collection);
+    }
+
+    /**
+     * <p>Returns an ordered set snapshot of all fields of this documenttype,
+     * <i>except the fields of Document</i>.
+     * Only the overridden version will be returned for overridden fields.</p>
+     *
+     * <p>The fields of a document type has a well-defined order which is
+     * exhibited in this set:
+     * - Fields come in the order defined in the document type definition.
+     * - The fields defined in inherited types come before those in
+     * the document type itself.
+     * - When a field in an inherited type is overridden, the value is overridden,
+     * but not the ordering.
+     * </p>
+     *
+     * @return an unmodifiable snapshot of the fields in this type
+     */
+    public Set<Field> fieldSet() {
+        Map<String, Field> map = new LinkedHashMap<String, Field>();
+        for (Field field : getFields()) { // Uniqify on field name
+            map.put(field.getName(), field);
+        }
+        return ImmutableSet.copyOf(map.values());
+    }
+
+    /**
+     * Returns an iterator over all fields in this documenttype
+     *
+     * @return An iterator for iterating the fields in this documenttype.
+     */
+    public Iterator<Field> fieldIteratorThisTypeOnly() {
+        return new Iterator<Field>() {
+            Iterator<Field> headerIt = headerType.getFields().iterator();
+            Iterator<Field> bodyIt = bodyType.getFields().iterator();
+
+            public boolean hasNext() {
+                if (headerIt != null) {
+                    if (headerIt.hasNext()) return true;
+                    headerIt = null;
+                }
+                return bodyIt.hasNext();
+            }
+
+            public Field next() {
+                return (headerIt != null ? headerIt.next() : bodyIt.next());
+            }
+
+
+            public void remove() {
+                if (headerIt != null) {
+                    headerIt.remove();
+                } else {
+                    bodyIt.remove();
+                }
+            }
+        };
+    }
+
+    public boolean equals(Object o) {
+        if (!(o instanceof DocumentType)) return false;
+        DocumentType other = (DocumentType) o;
+        // Ignore whether one of them have added inheritance to super Document.0 type
+        if (super.equals(o) && headerType.equals(other.headerType) &&
+            bodyType.equals(other.bodyType)) {
+            if ((inherits.size() > 1 || other.inherits.size() > 1) ||
+                (inherits.size() == 1 && other.inherits.size() == 1)) {
+                return inherits.equals(other.inherits);
+            }
+            return !(((inherits.size() == 1) && !inherits.get(0).getDataTypeName().equals(new DataTypeName("document")))
+                     || ((other.inherits.size() == 1) && !other.inherits.get(0).getDataTypeName().equals(new DataTypeName("document"))));
+        }
+        return false;
+    }
+
+    public int hashCode() {
+        return super.hashCode() + headerType.hashCode() + bodyType.hashCode() + inherits.hashCode();
+    }
+
+    @Override
+    public void onSerialize(Serializer target) {
+        if (target instanceof DocumentWriter) {
+            ((DocumentWriter) target).write(this);
+        }
+        // TODO: what if it's not a DocumentWriter?
+    }
+
+
+    @Override
+    public void visitMembers(ObjectVisitor visitor) {
+        super.visitMembers(visitor);
+        visitor.visit("headertype", headerType);
+        visitor.visit("bodytype", bodyType);
+        visitor.visit("inherits", inherits);
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/DocumentTypeId.java b/document/src/main/java/com/yahoo/document/DocumentTypeId.java
new file mode 100644
index 00000000000..46e5040c998
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/DocumentTypeId.java
@@ -0,0 +1,33 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+/**
+ * The id of a document type.
+ *
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class DocumentTypeId {
+    private int id;
+
+    public DocumentTypeId(int id) {
+        this.id = id;
+    }
+
+    public int getId() {
+        return id;
+    }
+
+    public boolean equals(Object o) {
+        if (!(o instanceof DocumentTypeId)) return false;
+        DocumentTypeId other = (DocumentTypeId) o;
+        return other.id == this.id;
+    }
+
+    public int hashCode() {
+        return id;
+    }
+
+    public String toString() {
+        return "" + id;
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/DocumentTypeManager.java b/document/src/main/java/com/yahoo/document/DocumentTypeManager.java
new file mode 100644
index 00000000000..0de7fe60500
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/DocumentTypeManager.java
@@ -0,0 +1,363 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import com.google.inject.Inject;
+import com.yahoo.config.subscription.ConfigSubscriber;
+import com.yahoo.document.annotation.AnnotationReferenceDataType;
+import com.yahoo.document.annotation.AnnotationType;
+import com.yahoo.document.annotation.AnnotationTypeRegistry;
+import com.yahoo.document.annotation.AnnotationTypes;
+import com.yahoo.document.config.DocumentmanagerConfig;
+import com.yahoo.document.serialization.DocumentDeserializer;
+import com.yahoo.document.serialization.DocumentDeserializerFactory;
+import com.yahoo.document.serialization.VespaDocumentDeserializer42;
+import com.yahoo.io.GrowableByteBuffer;
+
+import java.lang.reflect.Modifier;
+import java.util.*;
+import java.util.logging.Logger;
+
+/**
+ * The DocumentTypeManager keeps track of the document types registered in
+ * the Vespa common repository.
+ * <p>
+ * The DocumentTypeManager is also responsible for registering a FieldValue
+ * factory for each data type a field can have. The Document object
+ * uses this factory to serialize and deserialize the various datatypes.
+ * The factory could also be used to expand the functionality of various
+ * datatypes, for instance displaying the data type in human-readable form
+ * or as XML.
+ *
+ * @author <a href="mailto:thomasg@yahoo-inc.com">Thomas Gundersen</a>
+ */
+public class DocumentTypeManager {
+
+    private final static Logger log = Logger.getLogger(DocumentTypeManager.class.getName());
+    private ConfigSubscriber subscriber;
+
+    private Map<Integer, DataType> dataTypes = new LinkedHashMap<>();
+    private Map<DataTypeName, DocumentType> documentTypes = new LinkedHashMap<>();
+    private AnnotationTypeRegistry annotationTypeRegistry = new AnnotationTypeRegistry();
+
+    public DocumentTypeManager() {
+        registerDefaultDataTypes();
+    }
+
+    @Inject
+    public DocumentTypeManager(DocumentmanagerConfig config) {
+        this();
+        DocumentTypeManagerConfigurer.configureNewManager(config, this);
+    }
+
+    public void assign(DocumentTypeManager other) {
+        dataTypes = other.dataTypes;
+        documentTypes = other.documentTypes;
+        annotationTypeRegistry = other.annotationTypeRegistry;
+    }
+
+    public DocumentTypeManager configure(String configId) {
+        subscriber = DocumentTypeManagerConfigurer.configure(this, configId);
+        return this;
+    }
+
+    private void registerDefaultDataTypes() {
+        DocumentType superDocType = DataType.DOCUMENT;
+        dataTypes.put(superDocType.getId(), superDocType);
+        documentTypes.put(superDocType.getDataTypeName(), superDocType);
+
+        Class<? extends DataType> dataTypeClass = DataType.class;
+        for (java.lang.reflect.Field field : dataTypeClass.getFields()) {
+            if (Modifier.isStatic(field.getModifiers()) && Modifier.isFinal(field.getModifiers())) {
+                if (DataType.class.isAssignableFrom(field.getType())) {
+                    try {
+                        //these are all static final DataTypes listed in DataType:
+                        DataType type = (DataType) field.get(null);
+                        register(type);
+                    } catch (IllegalAccessException e) {
+                        //ignore
+                    }
+                }
+            }
+        }
+        for (AnnotationType type : AnnotationTypes.ALL_TYPES) {
+            annotationTypeRegistry.register(type);
+        }
+    }
+
+    public boolean hasDataType(String name) {
+        for (DataType type : dataTypes.values()) {
+            if (type.getName().equalsIgnoreCase(name)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    public boolean hasDataType(int code) {
+        return dataTypes.containsKey(code);
+    }
+
+    public DataType getDataType(String name) {
+        List<DataType> foundTypes = new ArrayList<>();
+        for (DataType type : dataTypes.values()) {
+            if (type.getName().equalsIgnoreCase(name)) {
+                foundTypes.add(type);
+            }
+        }
+        if (foundTypes.isEmpty()) {
+            throw new IllegalArgumentException("No datatype named " + name);
+        } else if (foundTypes.size() == 1) {
+            return foundTypes.get(0);
+        } else {
+            //the found types are probably documents or structs, sort them by type
+            Collections.sort(foundTypes, new Comparator<DataType>() {
+                public int compare(DataType first, DataType second) {
+                    if (first instanceof StructuredDataType && !(second instanceof StructuredDataType)) {
+                        return 1;
+                    } else if (!(first instanceof StructuredDataType) && second instanceof StructuredDataType) {
+                        return -1;
+                    } else {
+                        return 0;
+                    }
+                }
+            });
+        }
+        return foundTypes.get(0);
+    }
+
+    public DataType getDataType(int code) {
+        DataType type = dataTypes.get(code);
+        if (type == null) {
+            StringBuilder types=new StringBuilder();
+            for (Integer key : dataTypes.keySet()) {
+                types.append(key).append(" ");
+            }
+            throw new IllegalArgumentException("No datatype with code " + code + ". Registered type ids: " + types);
+        } else {
+            return type;
+        }
+    }
+
+    DataType getDataTypeAndReturnTemporary(int code) {
+        if (hasDataType(code)) {
+            return getDataType(code);
+        }
+        return new TemporaryDataType(code);
+    }
+
+    /**
+     * Register a data type of any sort, including document types.
+     * @param type The datatype to register
+     * TODO Give unique ids to document types
+     */
+    public void register(DataType type) {
+        type.register(this); // Recursively walk through all nested types and call registerSingleType for each one
+    }
+
+    /**
+     * Register a single datatype. Re-registering an existing, but equal, datatype is ok.
+     * @param type The datatype to register
+     */
+    void registerSingleType(DataType type) {
+        if (dataTypes.containsKey(type.getId())) {
+            DataType existingType = dataTypes.get(type.getId());
+            if (((type instanceof TemporaryDataType) || (type instanceof TemporaryStructuredDataType))
+                && !((existingType instanceof TemporaryDataType) || (existingType instanceof TemporaryStructuredDataType))) {
+                //we're trying to register a temporary type over a permanent one, don't do that:
+                return;
+            } else if ((existingType == type || existingType.equals(type))
+                    && !(existingType instanceof TemporaryDataType)
+                    && !(type instanceof TemporaryDataType)
+                    && !(existingType instanceof TemporaryStructuredDataType)
+                    && !(type instanceof TemporaryStructuredDataType)) { // Shortcut to improve speed
+                // Oki. Already registered.
+                return;
+            } else if (type instanceof DocumentType && dataTypes.get(type.getId()) instanceof DocumentType) {
+                /*
+                DocumentType newInstance = (DocumentType) type;
+                DocumentType oldInstance = (DocumentType) dataTypes.get(type.getId());
+                TODO fix tests
+                */
+                log.warning("Document type " + existingType + " is not equal to document type attempted registered " + type
+                        + ", but have same name. OVERWRITING TYPE as many tests currently does this. "
+                        + "Fix tests so we can throw exception here.");
+                //throw new IllegalStateException("Datatype " + existingType + " is not equal to datatype attempted registered "
+                //        + type + ", but already uses id " + type.getId());
+            } else if ((existingType instanceof TemporaryDataType) || (existingType instanceof TemporaryStructuredDataType)) {
+                //removing temporary type to be able to register correct type
+                dataTypes.remove(existingType.getId());
+            } else {
+                throw new IllegalStateException("Datatype " + existingType + " is not equal to datatype attempted registered "
+                        + type + ", but already uses id " + type.getId());
+            }
+        }
+
+        if (type instanceof DocumentType) {
+            DocumentType docType = (DocumentType) type;
+            if (docType.getInheritedTypes().size() == 0) {
+                docType.inherit(documentTypes.get(new DataTypeName("document")));
+            }
+            documentTypes.put(docType.getDataTypeName(), docType);
+        }
+        dataTypes.put(type.getId(), type);
+        type.setRegistered();
+    }
+
+    /**
+     * Registers a document type. Typically called by someone
+     * importing the document types from the common Vespa repository.
+     *
+     * @param docType The document type to register.
+     * @return the previously registered type, or null if none was registered
+     */
+    public DocumentType registerDocumentType(DocumentType docType) {
+        register(docType);
+        return docType;
+    }
+
+    /**
+     * Gets a registered document.
+     *
+     * @param name the document name of the type
+     * @return returns the document type found,
+     *         or null if there is no type with this name
+     */
+    public DocumentType getDocumentType(DataTypeName name) {
+        return documentTypes.get(name);
+    }
+
+    /**
+     * Returns a registered document type
+     *
+     * @param name    the type name of the document type
+     * @return returns the document type having this name, or null if none
+     */
+    public DocumentType getDocumentType(String name) {
+        return documentTypes.get(new DataTypeName(name));
+    }
+
+    final public Document createDocument(GrowableByteBuffer buf) {
+        DocumentDeserializer data = DocumentDeserializerFactory.create42(this, buf);
+        return new Document(data);
+    }
+    public Document createDocument(DocumentDeserializer data) {
+        return new Document(data);
+    }
+
+    public Document createDocument(GrowableByteBuffer header, GrowableByteBuffer body) {
+        DocumentDeserializer data = DocumentDeserializerFactory.create42(this, header, body);
+        return new Document(data);
+    }
+
+    /**
+     * A read only view of the registered data types
+     * @return collection of types
+     */
+    public Collection<DataType> getDataTypes() {
+        return Collections.unmodifiableCollection(dataTypes.values());
+    }
+
+    /**
+     * A read only view of the registered document types
+     * @return map of types
+     */
+    public Map<DataTypeName, DocumentType> getDocumentTypes() {
+        return Collections.unmodifiableMap(documentTypes);
+    }
+
+    public Iterator<DocumentType> documentTypeIterator() {
+        return documentTypes.values().iterator();
+    }
+
+    /**
+     * Clears the DocumentTypeManager. After this operation,
+     * only the default document type and data types are available.
+     */
+    public void clear() {
+        documentTypes.clear();
+        dataTypes.clear();
+        registerDefaultDataTypes();
+    }
+
+    public AnnotationTypeRegistry getAnnotationTypeRegistry() {
+        return annotationTypeRegistry;
+    }
+
+    void replaceTemporaryTypes() {
+        for (DataType type : dataTypes.values()) {
+            List<DataType> seenStructs = new LinkedList<>();
+            replaceTemporaryTypes(type, seenStructs);
+        }
+    }
+
+    private void replaceTemporaryTypes(DataType type, List<DataType> seenStructs) {
+        if (type instanceof WeightedSetDataType) {
+            replaceTemporaryTypesInWeightedSet((WeightedSetDataType) type, seenStructs);
+        } else if (type instanceof MapDataType) {
+            replaceTemporaryTypesInMap((MapDataType) type, seenStructs);
+        } else if (type instanceof CollectionDataType) {
+            replaceTemporaryTypesInCollection((CollectionDataType) type, seenStructs);
+        } else if (type instanceof StructDataType) {
+            replaceTemporaryTypesInStruct((StructDataType) type, seenStructs);
+        } else if (type instanceof PrimitiveDataType) {
+            //OK
+        } else if (type instanceof AnnotationReferenceDataType) {
+            //OK
+        } else if (type instanceof DocumentType) {
+            //OK
+        } else if (type instanceof TemporaryDataType) {
+            throw new IllegalStateException("TemporaryDataType registered in DocumentTypeManager, BUG!!");
+        } else {
+            log.warning("Don't know how to replace temporary data types in " + type);
+        }
+    }
+
+    @SuppressWarnings("deprecation")
+    private void replaceTemporaryTypesInStruct(StructDataType structDataType, List<DataType> seenStructs) {
+        seenStructs.add(structDataType);
+        for (Field field : structDataType.getFieldsThisTypeOnly()) {
+            DataType fieldType = field.getDataType();
+            if (fieldType instanceof TemporaryDataType) {
+                field.setDataType(getDataType(fieldType.getCode()));
+            } else {
+                if (!seenStructs.contains(fieldType)) {
+                    replaceTemporaryTypes(fieldType, seenStructs);
+                }
+            }
+        }
+    }
+
+    private void replaceTemporaryTypesInCollection(CollectionDataType collectionDataType, List<DataType> seenStructs) {
+        if (collectionDataType.getNestedType() instanceof TemporaryDataType) {
+            collectionDataType.setNestedType(getDataType(collectionDataType.getNestedType().getCode()));
+        } else {
+            replaceTemporaryTypes(collectionDataType.getNestedType(), seenStructs);
+        }
+    }
+
+    private void replaceTemporaryTypesInMap(MapDataType mapDataType, List<DataType> seenStructs) {
+        if (mapDataType.getValueType() instanceof TemporaryDataType) {
+            mapDataType.setValueType(getDataType(mapDataType.getValueType().getCode()));
+        } else {
+            replaceTemporaryTypes(mapDataType.getValueType(), seenStructs);
+        }
+
+        if (mapDataType.getKeyType() instanceof TemporaryDataType) {
+            mapDataType.setKeyType(getDataType(mapDataType.getKeyType().getCode()));
+        } else {
+            replaceTemporaryTypes(mapDataType.getKeyType(), seenStructs);
+        }
+    }
+
+    private void replaceTemporaryTypesInWeightedSet(WeightedSetDataType weightedSetDataType, List<DataType> seenStructs) {
+        if (weightedSetDataType.getNestedType() instanceof TemporaryDataType) {
+            weightedSetDataType.setNestedType(getDataType(weightedSetDataType.getNestedType().getCode()));
+        } else {
+            replaceTemporaryTypes(weightedSetDataType.getNestedType(), seenStructs);
+        }
+    }
+
+    public void shutdown() {
+        if (subscriber!=null) subscriber.close();
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java b/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java
new file mode 100644
index 00000000000..a575fbfba2a
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java
@@ -0,0 +1,246 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import com.yahoo.compress.CompressionType;
+import com.yahoo.config.subscription.ConfigSubscriber;
+import com.yahoo.document.config.DocumentmanagerConfig;
+import com.yahoo.document.annotation.AnnotationReferenceDataType;
+import com.yahoo.document.annotation.AnnotationType;
+import com.yahoo.log.LogLevel;
+import java.util.ArrayList;
+import java.util.logging.Logger;
+
+/**
+ * Configures the Vepsa document manager from a document id.
+ *
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSubscriber<DocumentmanagerConfig>{
+
+    private final static Logger log = Logger.getLogger(DocumentTypeManagerConfigurer.class.getName());
+
+    private DocumentTypeManager managerToConfigure;
+
+    public DocumentTypeManagerConfigurer(DocumentTypeManager manager) {
+        this.managerToConfigure = manager;
+    }
+
+    private static CompressionConfig makeCompressionConfig(DocumentmanagerConfig.Datatype.Structtype cfg) {
+        return new CompressionConfig(toCompressorType(cfg.compresstype()), cfg.compresslevel(),
+                                     cfg.compressthreshold(), cfg.compressminsize());
+    }
+
+
+    public static CompressionType toCompressorType(DocumentmanagerConfig.Datatype.Structtype.Compresstype.Enum value) {
+        switch (value) {
+            case NONE: return CompressionType.NONE;
+            case LZ4: return CompressionType.LZ4;
+            case UNCOMPRESSABLE: return CompressionType.INCOMPRESSIBLE;
+        }
+        throw new IllegalArgumentException("Compression type " + value + " is not supported");
+    }
+
+    /**
+     * <p>Makes the DocumentTypeManager subscribe on its config.</p>
+     *
+     * <p>Proper Vespa setups will use a config id which looks up the document manager config
+     * at the document server, but it is also possible to read config from a file containing
+     * a document manager configuration by using
+     * <code>file:path-to-document-manager.cfg</code>.</p>
+     *
+     * @param configId the config ID to use
+     */
+    public static ConfigSubscriber configure(DocumentTypeManager manager, String configId) {
+        return new DocumentTypeManagerConfigurer(manager).configure(configId);
+    }
+
+    public ConfigSubscriber configure(String configId) {
+        ConfigSubscriber subscriber = new ConfigSubscriber();
+        subscriber.subscribe(this, DocumentmanagerConfig.class, configId);
+        return subscriber;
+    }
+
+    static void configureNewManager(DocumentmanagerConfig config, DocumentTypeManager manager) {
+        if (config == null) {
+            return;
+        }
+
+        setupAnnotationTypesWithoutPayloads(config, manager);
+        setupAnnotationRefTypes(config, manager);
+
+        log.log(LogLevel.DEBUG, "Configuring document manager with " + config.datatype().size() + " data types.");
+        ArrayList<DocumentmanagerConfig.Datatype> failed = new ArrayList<>();
+        failed.addAll(config.datatype());
+        int failCounter = 30;
+        while (!failed.isEmpty()) {
+            --failCounter;
+            ArrayList<DocumentmanagerConfig.Datatype> tmp = failed;
+            failed = new ArrayList<>();
+            for (int i = 0; i < tmp.size(); i++) {
+                DocumentmanagerConfig.Datatype thisDataType = tmp.get(i);
+                int id = thisDataType.id();
+                try {
+                    for (Object o : thisDataType.arraytype()) {
+                        DocumentmanagerConfig.Datatype.Arraytype array = (DocumentmanagerConfig.Datatype.Arraytype) o;
+                        DataType nestedType = manager.getDataType(array.datatype());
+                        ArrayDataType type = new ArrayDataType(nestedType, id);
+                        manager.register(type);
+                    }
+                    for (Object o : thisDataType.maptype()) {
+                        DocumentmanagerConfig.Datatype.Maptype map = (DocumentmanagerConfig.Datatype.Maptype) o;
+                        DataType keyType = manager.getDataType(map.keytype());
+                        DataType valType = manager.getDataType(map.valtype());
+                        MapDataType type = new MapDataType(keyType, valType, id);
+                        manager.register(type);
+                    }
+                    for (Object o : thisDataType.weightedsettype()) {
+                        DocumentmanagerConfig.Datatype.Weightedsettype wset =
+                                    (DocumentmanagerConfig.Datatype.Weightedsettype) o;
+                        DataType nestedType = manager.getDataType(wset.datatype());
+                        WeightedSetDataType type = new WeightedSetDataType(
+                                nestedType, wset.createifnonexistant(), wset.removeifzero(), id);
+                        manager.register(type);
+                    }
+                    for (Object o : thisDataType.structtype()) {
+                        DocumentmanagerConfig.Datatype.Structtype struct = (DocumentmanagerConfig.Datatype.Structtype) o;
+                        StructDataType type = new StructDataType(id, struct.name());
+
+                        if (config.enablecompression()) {
+                            CompressionConfig comp = makeCompressionConfig(struct);
+                            type.setCompressionConfig(comp);
+                        }
+
+                        for (Object j : struct.field()) {
+                            DocumentmanagerConfig.Datatype.Structtype.Field field =
+                                    (DocumentmanagerConfig.Datatype.Structtype.Field) j;
+                            DataType fieldType = (field.datatype() == id)
+                                               ? manager.getDataTypeAndReturnTemporary(field.datatype())
+                                               : manager.getDataType(field.datatype());
+
+                            if (field.id().size() == 1) {
+                                type.addField(new Field(field.name(), field.id().get(0).id(), fieldType, true));
+                            } else {
+                                type.addField(new Field(field.name(), fieldType, true));
+                            }
+                        }
+                        manager.register(type);
+                    }
+                    for (Object o : thisDataType.documenttype()) {
+                        DocumentmanagerConfig.Datatype.Documenttype doc = (DocumentmanagerConfig.Datatype.Documenttype) o;
+                        StructDataType header = (StructDataType) manager.getDataType(doc.headerstruct());
+                        StructDataType body = (StructDataType) manager.getDataType(doc.bodystruct());
+                        for (Field field : body.getFields()) {
+                            field.setHeader(false);
+                        }
+                        DocumentType type = new DocumentType(doc.name(), header, body);
+                        for (Object j : doc.inherits()) {
+                            DocumentmanagerConfig.Datatype.Documenttype.Inherits parent =
+                                    (DocumentmanagerConfig.Datatype.Documenttype.Inherits) j;
+                            DataTypeName name = new DataTypeName(parent.name());
+                            DocumentType parentType = manager.getDocumentType(name);
+                            if (parentType == null) {
+                                throw new IllegalArgumentException("Could not find document type '" + name.toString() + "'.");
+                            }
+                            type.inherit(parentType);
+                        }
+                        manager.register(type);
+                    }
+                } catch (IllegalArgumentException e) {
+                    failed.add(thisDataType);
+                    if (failCounter < 0) {
+                        throw e;
+                    }
+                }
+            }
+        }
+        addStructInheritance(config, manager);
+        addAnnotationTypePayloads(config, manager);
+        addAnnotationTypeInheritance(config, manager);
+
+        manager.replaceTemporaryTypes();
+    }
+
+    public static DocumentTypeManager configureNewManager(DocumentmanagerConfig config) {
+        DocumentTypeManager manager = new DocumentTypeManager();
+        if (config == null) {
+            return manager;
+        }
+        configureNewManager(config, manager);
+        return manager;
+    }
+
+    /**
+     * Called by the configuration system to register document types based on documentmanager.cfg.
+     *
+     * @param config the instance representing config in documentmanager.cfg.
+     */
+    @Override
+    public void configure(DocumentmanagerConfig config) {
+        DocumentTypeManager manager = configureNewManager(config);
+        int defaultTypeCount = new DocumentTypeManager().getDataTypes().size();
+        if (this.managerToConfigure.getDataTypes().size() != defaultTypeCount) {
+            log.log(LogLevel.DEBUG, "Live document config overwritten with new config.");
+        }
+        managerToConfigure.assign(manager);
+    }
+
+    private static void setupAnnotationRefTypes(DocumentmanagerConfig config, DocumentTypeManager manager) {
+        for (int i = 0; i < config.datatype().size(); i++) {
+            DocumentmanagerConfig.Datatype thisDataType = config.datatype(i);
+            int id = thisDataType.id();
+            for (Object o : thisDataType.annotationreftype()) {
+                DocumentmanagerConfig.Datatype.Annotationreftype annRefType = (DocumentmanagerConfig.Datatype.Annotationreftype) o;
+                AnnotationType annotationType = manager.getAnnotationTypeRegistry().getType(annRefType.annotation());
+                if (annotationType == null) {
+                    throw new IllegalArgumentException("Found reference to " + annRefType.annotation() + ", which does not exist!");
+                }
+                AnnotationReferenceDataType type = new AnnotationReferenceDataType(annotationType, id);
+                manager.register(type);
+            }
+        }
+    }
+
+    private static void setupAnnotationTypesWithoutPayloads(DocumentmanagerConfig config, DocumentTypeManager manager) {
+        for (DocumentmanagerConfig.Annotationtype annType : config.annotationtype()) {
+            AnnotationType annotationType = new AnnotationType(annType.name(), annType.id());
+            manager.getAnnotationTypeRegistry().register(annotationType);
+        }
+    }
+
+    private static void addAnnotationTypePayloads(DocumentmanagerConfig config, DocumentTypeManager manager) {
+        for (DocumentmanagerConfig.Annotationtype annType : config.annotationtype()) {
+            AnnotationType annotationType = manager.getAnnotationTypeRegistry().getType(annType.id());
+            DataType payload = manager.getDataType(annType.datatype());
+            if (!payload.equals(DataType.NONE)) {
+                annotationType.setDataType(payload);
+            }
+        }
+
+    }
+
+    private static void addAnnotationTypeInheritance(DocumentmanagerConfig config, DocumentTypeManager manager) {
+        for (DocumentmanagerConfig.Annotationtype annType : config.annotationtype()) {
+            if (annType.inherits().size() > 0) {
+                AnnotationType inheritedType = manager.getAnnotationTypeRegistry().getType(annType.inherits(0).id());
+                AnnotationType type = manager.getAnnotationTypeRegistry().getType(annType.id());
+                type.inherit(inheritedType);
+            }
+        }
+    }
+
+    private static void addStructInheritance(DocumentmanagerConfig config, DocumentTypeManager manager) {
+        for (int i = 0; i < config.datatype().size(); i++) {
+            DocumentmanagerConfig.Datatype thisDataType = config.datatype(i);
+            int id = thisDataType.id();
+            for (Object o : thisDataType.structtype()) {
+                DocumentmanagerConfig.Datatype.Structtype struct = (DocumentmanagerConfig.Datatype.Structtype) o;
+                StructDataType thisStruct = (StructDataType) manager.getDataType(id);
+
+                for (DocumentmanagerConfig.Datatype.Structtype.Inherits parent : struct.inherits()) {
+                    StructDataType parentStruct = (StructDataType) manager.getDataType(parent.name());
+                    thisStruct.inherit(parentStruct);
+                }
+            }
+        }
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/DocumentUpdate.java b/document/src/main/java/com/yahoo/document/DocumentUpdate.java
new file mode 100644
index 00000000000..0cfaa601b21
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/DocumentUpdate.java
@@ -0,0 +1,415 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import com.yahoo.document.fieldpathupdate.FieldPathUpdate;
+import com.yahoo.document.serialization.DocumentSerializerFactory;
+import com.yahoo.document.serialization.DocumentUpdateReader;
+import com.yahoo.document.serialization.DocumentUpdateWriter;
+import com.yahoo.document.update.FieldUpdate;
+import com.yahoo.io.GrowableByteBuffer;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+
+/**
+ * <p>Specifies one or more field updates to a document.</p> <p>A document update contains a list of {@link
+ * com.yahoo.document.update.FieldUpdate field updates} for fields to be updated by this update. Each field update is
+ * applied atomically, but the entire document update is not. A document update can only contain one field update per
+ * field. To make multiple updates to the same field in the same document update, add multiple {@link
+ * com.yahoo.document.update.ValueUpdate value updates} to the same field update.</p> <p>To update a document and
+ * set a string field to a new value:</p>
+ * <pre>
+ * DocumentType musicType = DocumentTypeManager.getInstance().getDocumentType("music", 0);
+ * DocumentUpdate docUpdate = new DocumentUpdate(musicType,
+ *         new DocumentId("doc:test:http://music.yahoo.com/"));
+ * FieldUpdate update = FieldUpdate.createAssign(musicType.getField("artist"), "lillbabs");
+ * docUpdate.addFieldUpdate(update);
+ * </pre>
+ *
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ * @see com.yahoo.document.update.FieldUpdate
+ * @see com.yahoo.document.update.ValueUpdate
+ */
+public class DocumentUpdate extends DocumentOperation implements Iterable<FieldPathUpdate> {
+
+    //see src/vespa/document/util/identifiableid.h
+    public static final int CLASSID = 0x1000 + 6;
+
+    private DocumentId docId;
+    private List<FieldUpdate> fieldUpdates;
+    private List<FieldPathUpdate> fieldPathUpdates;
+    private DocumentType documentType;
+    private boolean createIfNonExistent;
+
+    /**
+     * Creates a DocumentUpdate.
+     *
+     * @param docId   the ID of the update
+     * @param docType the document type that this update is valid for
+     */
+    public DocumentUpdate(DocumentType docType, DocumentId docId) {
+        this(docType, docId, new ArrayList<FieldUpdate>());
+    }
+
+    /**
+     * Creates a new document update using a reader
+     */
+    public DocumentUpdate(DocumentUpdateReader reader) {
+        docId = null;
+        documentType = null;
+        fieldUpdates = new ArrayList<>();
+        fieldPathUpdates = new ArrayList<>();
+        reader.read(this);
+    }
+
+    /**
+     * Creates a DocumentUpdate.
+     *
+     * @param docId   the ID of the update
+     * @param docType the document type that this update is valid for
+     */
+    public DocumentUpdate(DocumentType docType, String docId) {
+        this(docType, new DocumentId(docId));
+    }
+
+    private DocumentUpdate(DocumentType docType, DocumentId docId, List<FieldUpdate> fieldUpdates) {
+        this.docId = docId;
+        this.documentType = docType;
+        this.fieldUpdates = fieldUpdates;
+        this.fieldPathUpdates = new ArrayList<>();
+    }
+
+    public DocumentId getId() {
+        return docId;
+    }
+
+    /**
+     * Sets the document id of the document to update.
+     * Use only while deserializing - changing the document id after creation has undefined behaviour.
+     */
+    public void setId(DocumentId id) {
+        docId = id;
+    }
+
+    /**
+     * Applies this document update.
+     *
+     * @param doc the document to apply the update to
+     * @return a reference to itself
+     * @throws IllegalArgumentException if the document does not have the same document type as this update
+     */
+    public DocumentUpdate applyTo(Document doc) {
+        if (!documentType.equals(doc.getDataType())) {
+            throw new IllegalArgumentException(
+                    "Document " + doc + " must have same type as update, which is type " + documentType);
+        }
+
+        for (FieldUpdate fieldUpdate : fieldUpdates) {
+            fieldUpdate.applyTo(doc);
+        }
+        for (FieldPathUpdate fieldPathUpdate : fieldPathUpdates) {
+            fieldPathUpdate.applyTo(doc);
+        }
+        return this;
+    }
+
+    /**
+     * Get an unmodifiable list of all field updates that this document update specifies.
+     *
+     * @return a list of all FieldUpdates in this DocumentUpdate
+     */
+    public List<FieldUpdate> getFieldUpdates() {
+        return Collections.unmodifiableList(fieldUpdates);
+    }
+
+    /**
+     * Get an unmodifiable list of all field path updates this document update specifies.
+     *
+     * @return Returns a list of all field path updates in this document update.
+     */
+    public List<FieldPathUpdate> getFieldPathUpdates() {
+        return Collections.unmodifiableList(fieldPathUpdates);
+    }
+
+    /** Returns the type of the document this updates
+     *
+     * @return The documentype of the document
+     */
+    public DocumentType getDocumentType() {
+        return documentType;
+    }
+
+    /**
+     * Sets the document type. Use only while deserializing - changing the document type after creation
+     * has undefined behaviour.
+     */
+    public void setDocumentType(DocumentType type) {
+        documentType = type;
+    }
+
+    /**
+     * Get the field update at the specified index in the list of field updates.
+     *
+     * @param index the index of the FieldUpdate to return
+     * @return the FieldUpdate at the specified index
+     * @throws IndexOutOfBoundsException if index is out of range
+     */
+    public FieldUpdate getFieldUpdate(int index) {
+        return fieldUpdates.get(index);
+    }
+
+    /**
+     * Replaces the field update at the specified index in the list of field updates.
+     *
+     * @param index index of the FieldUpdate to replace
+     * @param upd   the FieldUpdate to be stored at the specified position
+     * @return the FieldUpdate previously at the specified position
+     * @throws IndexOutOfBoundsException if index is out of range
+     */
+    public FieldUpdate setFieldUpdate(int index, FieldUpdate upd) {
+        return fieldUpdates.set(index, upd);
+    }
+
+    /**
+     * Returns the update for a field
+     *
+     * @param field the field to return the update of
+     * @return the update for the field, or null if that field has no update in this
+     */
+    public FieldUpdate getFieldUpdate(Field field) {
+        return getFieldUpdate(field.getName());
+    }
+
+    /** Removes all field updates from the list for field updates. */
+    public void clearFieldUpdates() {
+        fieldUpdates.clear();
+    }
+
+    /**
+     * Returns the update for a field name
+     *
+     * @param fieldName the field name to return the update of
+     * @return the update for the field, or null if that field has no update in this
+     */
+    public FieldUpdate getFieldUpdate(String fieldName) {
+        for (FieldUpdate fieldUpdate : fieldUpdates) {
+            if (fieldUpdate.getField().getName().equals(fieldName)) {
+                return fieldUpdate;
+            }
+        }
+        return null;
+    }
+
+    /**
+     * Assigns the field updates of this document update.
+     * This document update receives ownership of the list - it can not be subsequently used
+     * by the caller. The list may not be unmodifiable.
+     *
+     * @param fieldUpdates the new list of updates of this
+     * @throws NullPointerException if the argument passed is null
+     */
+    public void setFieldUpdates(List<FieldUpdate> fieldUpdates) {
+        if (fieldUpdates == null) {
+            throw new NullPointerException("The field updates of a document update can not be null");
+        }
+        this.fieldUpdates = fieldUpdates;
+    }
+
+    /**
+     * Get the number of field updates in this document update.
+     *
+     * @return the size of the List of FieldUpdates
+     */
+    public int size() {
+        return fieldUpdates.size();
+    }
+
+    /**
+     * Adds the given {@link FieldUpdate} to this DocumentUpdate. If this DocumentUpdate already contains a FieldUpdate
+     * for the named field, the content of the given FieldUpdate is added to the existing one.
+     *
+     * @param update The FieldUpdate to add to this DocumentUpdate.
+     * @return This, to allow chaining.
+     * @throws IllegalArgumentException If the {@link DocumentType} of this DocumentUpdate does not have a corresponding
+     *                                  field.
+     */
+    public DocumentUpdate addFieldUpdate(FieldUpdate update) {
+        String fieldName = update.getField().getName();
+        if (!documentType.hasField(fieldName)) {
+            throw new IllegalArgumentException("Document type '" + documentType.getName() + "' does not have field '" +
+                                               fieldName + "'.");
+        }
+        FieldUpdate prevUpdate = getFieldUpdate(fieldName);
+        if (prevUpdate != update) {
+            if (prevUpdate != null) {
+                prevUpdate.addAll(update);
+            } else {
+                fieldUpdates.add(update);
+            }
+        }
+        return this;
+    }
+
+    /**
+     * Adds a field path update to perform on the document.
+     *
+     * @return a reference to itself.
+     */
+    public DocumentUpdate addFieldPathUpdate(FieldPathUpdate fieldPathUpdate) {
+        fieldPathUpdates.add(fieldPathUpdate);
+        return this;
+    }
+
+    // TODO: Remove this when we figure out correct behaviour.
+
+    public void addFieldUpdateNoCheck(FieldUpdate fieldUpdate) {
+        fieldUpdates.add(fieldUpdate);
+    }
+
+    /**
+     * Adds all the field- and field path updates of the given document update to this. If the given update refers to a
+     * different document or document type than this, this method throws an exception.
+     *
+     * @param update The update whose content to add to this.
+     * @throws IllegalArgumentException If the {@link DocumentId} or {@link DocumentType} of the given DocumentUpdate
+     *                                  does not match the content of this.
+     */
+    public void addAll(DocumentUpdate update) {
+        if (update == null) {
+            return;
+        }
+        if (!docId.equals(update.docId)) {
+            throw new IllegalArgumentException("Expected " + docId + ", got " + update.docId + ".");
+        }
+        if (!documentType.equals(update.documentType)) {
+            throw new IllegalArgumentException("Expected " + documentType + ", got " + update.documentType + ".");
+        }
+        for (FieldUpdate fieldUpd : update.fieldUpdates) {
+            addFieldUpdate(fieldUpd);
+        }
+        for (FieldPathUpdate pathUpd : update.fieldPathUpdates) {
+            addFieldPathUpdate(pathUpd);
+        }
+    }
+
+    /**
+     * Removes the field update at the specified position in the list of field updates.
+     *
+     * @param index the index of the FieldUpdate to remove
+     * @return the FieldUpdate previously at the specified position
+     * @throws IndexOutOfBoundsException if index is out of range
+     */
+    public FieldUpdate removeFieldUpdate(int index) {
+        return fieldUpdates.remove(index);
+    }
+
+    /**
+     * Returns the document type of this document update.
+     *
+     * @return the document type of this document update
+     */
+    public DocumentType getType() {
+        return documentType;
+    }
+
+    public final void serialize(GrowableByteBuffer buf) {
+        serialize(DocumentSerializerFactory.create42(buf));
+    }
+
+    public void serialize(DocumentUpdateWriter data) {
+        data.write(this);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (!(o instanceof DocumentUpdate)) return false;
+
+        DocumentUpdate that = (DocumentUpdate) o;
+
+        if (docId != null ? !docId.equals(that.docId) : that.docId != null) return false;
+        if (documentType != null ? !documentType.equals(that.documentType) : that.documentType != null) return false;
+        if (fieldPathUpdates != null ? !fieldPathUpdates.equals(that.fieldPathUpdates) : that.fieldPathUpdates != null)
+            return false;
+        if (fieldUpdates != null ? !fieldUpdates.equals(that.fieldUpdates) : that.fieldUpdates != null) return false;
+        if (createIfNonExistent != that.createIfNonExistent) return false;
+
+        return true;
+    }
+
+    @Override
+    public int hashCode() {
+        int result = docId != null ? docId.hashCode() : 0;
+        result = 31 * result + (fieldUpdates != null ? fieldUpdates.hashCode() : 0);
+        result = 31 * result + (fieldPathUpdates != null ? fieldPathUpdates.hashCode() : 0);
+        result = 31 * result + (documentType != null ? documentType.hashCode() : 0);
+        return result;
+    }
+
+    @Override
+    public String toString() {
+        StringBuilder string = new StringBuilder();
+        string.append("update of document '");
+        string.append(docId);
+        string.append("': ");
+        string.append("create-if-non-existent=");
+        string.append(createIfNonExistent ? "true" : "false");
+        string.append(": ");
+        string.append("[");
+
+        for (Iterator<FieldUpdate> i = fieldUpdates.iterator(); i.hasNext();) {
+            FieldUpdate fieldUpdate = i.next();
+            string.append(fieldUpdate);
+            if (i.hasNext()) {
+                string.append(", ");
+            }
+        }
+        string.append("]");
+
+        if (fieldPathUpdates.size() > 0) {
+            string.append(" [ ");
+            for (FieldPathUpdate up : fieldPathUpdates) {
+                string.append(up.toString() + " ");
+            }
+            string.append(" ]");
+        }
+
+        return string.toString();
+    }
+
+    public Iterator<FieldPathUpdate> iterator() {
+        return fieldPathUpdates.iterator();
+    }
+
+    /**
+     * Returns whether or not this field update contains any field- or field path updates.
+     *
+     * @return True if this update is empty.
+     */
+    public boolean isEmpty() {
+        return fieldUpdates.isEmpty() && fieldPathUpdates.isEmpty();
+    }
+
+    /**
+     * Sets whether this update should create the document it updates if that document does not exist.
+     * In this case an empty document is created before the update is applied.
+     *
+     * @since 5.17
+     * @param value Whether the document it updates should be created.
+     */
+    public void setCreateIfNonExistent(boolean value) {
+        createIfNonExistent = value;
+    }
+
+    /**
+     * Gets whether this update should create the document it updates if that document does not exist.
+     *
+     * @since 5.17
+     * @return Whether the document it updates should be created.
+     */
+    public boolean getCreateIfNonExistent() {
+        return createIfNonExistent;
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/DocumentUtil.java b/document/src/main/java/com/yahoo/document/DocumentUtil.java
new file mode 100644
index 00000000000..bce88a67ba8
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/DocumentUtil.java
@@ -0,0 +1,32 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+/**
+ * Class containing static utility function related to documents.
+ * @author einarmr
+ * @since 5.1.9
+ */
+public class DocumentUtil {
+    /**
+     * A convenience method that can be used to calculate a max pending queue size given
+     * the number of threads processing the documents, their size, and the memory available.
+     *
+     * @return the max pending size (in bytes) that should be used.
+     */
+    public static int calculateMaxPendingSize(double maxConcurrentFactor, double documentExpansionFactor, int containerCoreMemoryMb) {
+        final long heapBytes = Runtime.getRuntime().maxMemory();
+        final long heapMb = heapBytes / 1024L / 1024L;
+        final double maxPendingMb = ((double) (heapMb - containerCoreMemoryMb)) / (1.0d + (maxConcurrentFactor * documentExpansionFactor));
+        long maxPendingBytes = ((long) (maxPendingMb * 1024.0d)) * 1024L;
+        if (maxPendingBytes < (1024L * 1024L)) {
+            maxPendingBytes = 1024L * 1024L;  //1 MB
+        }
+        if (maxPendingBytes > (heapBytes / 5L)) {
+            maxPendingBytes = heapBytes / 5L;  //we do not want a maxPendingBytes greater than 1/5 heap (we probably have a very low expansion factor)
+        }
+        if (maxPendingBytes > (1<<30)) {  //we don't want a maxPendingBytes greater than 1G
+            maxPendingBytes = 1<<30;
+        }
+        return (int) maxPendingBytes;
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/Field.java b/document/src/main/java/com/yahoo/document/Field.java
new file mode 100644
index 00000000000..86543916b42
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/Field.java
@@ -0,0 +1,259 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import com.yahoo.collections.BobHash;
+import com.yahoo.document.fieldset.DocIdOnly;
+import com.yahoo.document.fieldset.FieldSet;
+import com.yahoo.document.fieldset.NoFields;
+import com.yahoo.vespa.objects.FieldBase;
+
+import java.io.Serializable;
+
+/**
+ * A name and type. Fields are contained in document types to describe their fields,
+ * but is also used to represent name/type pairs which are not part of document types.
+ *
+ * @author <a href="mailto:thomasg@yahoo-inc.com">Thomas Gundersen</a>
+ * @author bratseth
+ */
+public class Field extends FieldBase implements FieldSet, Comparable, Serializable {
+
+    protected DataType dataType;
+    protected int fieldId;
+    private int fieldIdV6;
+    private boolean isHeader;
+    private boolean forcedId = false;
+
+    /**
+     * Creates a new field.
+     *
+     * @param name     The name of the field
+     * @param dataType The datatype of the field
+     * @param isHeader Whether this is a "header" field or a "content" field
+     *                 (true = "header").
+     */
+    public Field(String name, int id, DataType dataType, boolean isHeader) {
+        super(name);
+        this.fieldId = id;
+        this.fieldIdV6 = id;
+        this.dataType = dataType;
+        this.isHeader = isHeader;
+        this.forcedId = true;
+        validateId(id, null, Document.SERIALIZED_VERSION);
+    }
+
+    public Field(String name) {
+        this(name, DataType.NONE);
+    }
+
+
+    /**
+     * Creates a new field.
+     *
+     * @param name     The name of the field
+     * @param dataType The datatype of the field
+     * @param isHeader Whether this is a "header" field or a "content" field
+     *                 (true = "header").
+     * @param owner    the owning document (used to check for id collisions)
+     */
+    public Field(String name, DataType dataType, boolean isHeader, DocumentType owner) {
+        this(name, 0, dataType, isHeader);
+        this.fieldId = calculateIdV7(owner);
+        this.fieldIdV6 = calculateIdV6(owner);
+        this.forcedId = false;
+    }
+
+    /**
+     * Creates a new field.
+     *
+     * @param name     The name of the field
+     * @param dataType The datatype of the field
+     * @param isHeader Whether this is a "header" field or a "content" field
+     *                 (true = "header").
+     */
+    public Field(String name, DataType dataType, boolean isHeader) {
+        this(name, dataType, isHeader, null);
+    }
+
+    /**
+     * Constructor for <b>header</b> fields
+     *
+     * @param name     The name of the field
+     * @param dataType The datatype of the field
+     */
+    public Field(String name, DataType dataType) {
+        this(name, dataType, true);
+    }
+
+    /**
+     * Creates a field with a new name and the other properties
+     * (excluding the id and owner) copied from another field
+     */
+    // TODO: Decide on one copy/clone idiom and do it for this and all it is calling
+    public Field(String name, Field field) {
+        this(name, field.dataType, field.isHeader, null);
+    }
+
+    /**
+     * The field id must be unique within a document type, and also
+     * within a (unknown at this time) hierarchy of document types.
+     * In addition it should be as resilient to doctype content changes
+     * and inheritance hierarchy changes as possible.
+     * All of this is enforced for names, so id's should follow names.
+     * Therefore we hash on name.
+     */
+    private int calculateIdV6(DocumentType owner) {
+        int newId = BobHash.hash(getName()); // Using a portfriendly hash
+        if (newId < 0) newId = -newId; // Highest bit is reserved to tell 7-bit id's from 31-bit ones
+        validateId(newId, owner, 6);
+        return newId;
+    }
+
+    public int compareTo(Object o) {
+        return fieldId - ((Field) o).fieldId;
+    }
+
+    /**
+     * The field id must be unique within a document type, and also
+     * within a (unknown at this time) hierarchy of document types.
+     * In addition it should be as resilient to doctype content changes
+     * and inheritance hierarchy changes as possible.
+     * All of this is enforced for names, so id's should follow names.
+     * Therefore we hash on name.
+     */
+    protected int calculateIdV7(DocumentType owner) {
+        String combined = getName() + dataType.getId();
+
+        int newId = BobHash.hash(combined); // Using a portfriendly hash
+        if (newId < 0) newId = -newId; // Highest bit is reserved to tell 7-bit id's from 31-bit ones
+        validateId(newId, owner, Document.SERIALIZED_VERSION);
+        return newId;
+    }
+
+    /**
+     * Sets the id of this field. Don't do this unless you know what you are doing
+     *
+     * @param newId the id - if this is less than 100 it will cause document to serialize
+     *              using just one byte for this field id. 100-127 are reserved values
+     * @param owner the owning document, this is checked for collisions and notified
+     *              of the id change. It can not be null
+     */
+    public void setId(int newId, DocumentType owner) {
+        if (owner == null) {
+            throw new NullPointerException("Can not assign an id of " + this + " without knowing the owner");
+        }
+
+        validateId(newId, owner, Document.SERIALIZED_VERSION);
+
+        owner.removeField(getName());
+        this.fieldId = newId;
+        this.fieldIdV6 = newId;
+        this.forcedId = true;
+        owner.addField(this);
+    }
+
+    private void validateId(int newId, DocumentType owner, int version) {
+        if (newId >= 100 && newId <= 127) {
+            throw new IllegalArgumentException("Attempt to set the id of " + this + " to " + newId +
+                                               " failed, values from 100 to 127 " + "are reserved for internal use");
+        }
+
+        if ((newId & 0x80000000) != 0) // Highest bit must not be set
+        {
+            throw new IllegalArgumentException("Attempt to set the id of " + this + " to " + newId +
+                                               " failed, negative id values " + " are illegal");
+        }
+
+
+        if (owner == null) return;
+        {
+            Field existing = owner.getField(newId, version);
+            if (existing != null && !existing.getName().equals(getName())) {
+                throw new IllegalArgumentException("Couldn't set id of " + this + " to " + newId + ", " + existing +
+                                                   " already has this id in " + owner);
+            }
+        }
+    }
+
+    /** @return Returns the datatype of the field */
+    public final DataType getDataType() {
+        return dataType;
+    }
+
+    /**
+     * Set the data type of the field. This will cause recalculation of fieldid for version 7+.
+     *
+     * @deprecated do not use
+     * @param type The new type of the field.
+     */
+    @Deprecated // Do not remove on Vespa 6
+    public void setDataType(DataType type) {
+        dataType = type;
+        fieldId = calculateIdV7(null);
+        forcedId = false;
+    }
+
+    /** Returns the numeric ID used to represent this field when serialized */
+    public final int getId(int version) {
+        return (version > 6) ? getId() : getIdV6();
+    }
+
+    public final int getId() {
+        return fieldId;
+    }
+
+    public final int getIdV6() {
+        return fieldIdV6;
+    }
+
+    /**
+     *
+     * @return true if the field has a forced id
+     */
+    public final boolean hasForcedId() {
+        return forcedId;
+    }
+
+    /** @return Returns true if this field should be a part of "header" serializations. */
+    public boolean isHeader() {
+        return isHeader;
+    }
+
+    /** Sets whether this is a header field */
+    public void setHeader(boolean header) {
+        this.isHeader = header;
+    }
+
+    /** Two fields are equal if they have the same name and the same data type */
+    @Override
+    public boolean equals(Object o) {
+        return this == o || o instanceof Field && super.equals(o) && dataType.equals(((Field) o).dataType);
+    }
+
+    @Override
+    public int hashCode() {
+        return getId();
+    }
+
+    public String toString() {
+        return super.toString() + "(" + dataType + ")";
+    }
+
+    @Override
+    public boolean contains(FieldSet o) {
+        if (o instanceof NoFields || o instanceof DocIdOnly) {
+            return true;
+        }
+
+        if (o instanceof Field) {
+            return equals(o);
+        }
+
+        return false;
+    }
+
+    @Override
+    public FieldSet clone() throws CloneNotSupportedException {
+        return (Field)super.clone();
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/FieldPath.java b/document/src/main/java/com/yahoo/document/FieldPath.java
new file mode 100755
index 00000000000..0497f2b139d
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/FieldPath.java
@@ -0,0 +1,127 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+
+/**
+ * This class represents a path into a document, that can be used to iterate through the document and extract the field
+ * values you're interested in.
+ *
+ * @author <a href="mailto:thomasg@yahoo-inc.com">Thomas Gundersen</a>
+ */
+public class FieldPath implements Iterable<FieldPathEntry> {
+
+    private final List<FieldPathEntry> list;
+    /**
+     * Constructs an empty path.
+     */
+    public FieldPath() {
+        list = Collections.emptyList();
+    }
+
+    /**
+     * Constructs a path containing the entries of the specified path, in the order they are returned by that path's
+     * iterator.
+     *
+     * @param path The path whose entries are to be placed into this path.
+     * @throws NullPointerException If the specified path is null.
+     */
+    public FieldPath(FieldPath path) {
+        this(path.list);
+    }
+
+    public FieldPath(List<FieldPathEntry> path) {
+        list = Collections.unmodifiableList(path);
+    }
+
+    public int size() { return list.size(); }
+    public FieldPathEntry get(int index) { return list.get(index); }
+    public boolean isEmpty() { return list.isEmpty(); }
+    public Iterator<FieldPathEntry> iterator() { return list.iterator(); }
+    public List<FieldPathEntry> getList() { return list; }
+
+    /**
+     * Compares this field path with the given field path, returns true if the field path starts with the other.
+     *
+     * @param other The field path to compare with.
+     * @return Returns true if this field path starts with the other field path, otherwise false
+     */
+    public boolean startsWith(FieldPath other) {
+        if (other.size() > size()) {
+            return false;
+        }
+
+        for (int i = 0; i < other.size(); i++) {
+            if (!other.get(i).equals(get(i))) {
+                return false;
+            }
+        }
+
+        return true;
+    }
+
+    /**
+     * @return Returns the datatype we can expect this field path to return.
+     */
+    public DataType getResultingDataType() {
+        if (isEmpty()) {
+            return null;
+        }
+
+        return get(size() - 1).getResultingDataType();
+    }
+
+    /**
+     * Convenience method to build a field path from a path string. This is a simple proxy for {@link
+     * DataType#buildFieldPath(String)}.
+     *
+     * @param fieldType The data type of the value to build a path for.
+     * @param fieldPath The path string to parse.
+     * @return The corresponding field path object.
+     */
+    public static FieldPath newInstance(DataType fieldType, String fieldPath) {
+        return fieldType.buildFieldPath(fieldPath);
+    }
+
+    @Override
+    public String toString() {
+        StringBuilder out = new StringBuilder();
+        DataType prevType = null;
+        for (FieldPathEntry entry : this) {
+            FieldPathEntry.Type type = entry.getType();
+            switch (type) {
+            case STRUCT_FIELD:
+                if (out.length() > 0) {
+                    out.append(".");
+                }
+                Field field = entry.getFieldRef();
+                out.append(field.getName());
+                prevType = field.getDataType();
+                break;
+            case ARRAY_INDEX:
+                out.append("[").append(entry.getLookupIndex()).append("]");
+                break;
+            case MAP_KEY:
+                break;
+            case MAP_ALL_KEYS:
+                out.append(".key");
+                break;
+            case MAP_ALL_VALUES:
+                out.append(".value");
+                break;
+            case VARIABLE:
+                if (prevType instanceof ArrayDataType) {
+                    out.append("[$").append(entry.getVariableName()).append("]");
+                } else if (prevType instanceof WeightedSetDataType || prevType instanceof MapDataType) {
+                    out.append("{$").append(entry.getVariableName()).append("}");
+                } else {
+                    out.append("$").append(entry.getVariableName());
+                }
+            }
+        }
+        return out.toString();
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/FieldPathEntry.java b/document/src/main/java/com/yahoo/document/FieldPathEntry.java
new file mode 100755
index 00000000000..d542acd430e
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/FieldPathEntry.java
@@ -0,0 +1,312 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import com.yahoo.document.datatypes.FieldValue;
+
+/**
+ * @author thomasg
+ */
+public class FieldPathEntry {
+    public enum Type {
+        STRUCT_FIELD,
+        ARRAY_INDEX,
+        MAP_KEY,
+        MAP_ALL_KEYS,
+        MAP_ALL_VALUES,
+        VARIABLE
+    }
+
+    private final Type type;
+    private final int lookupIndex;
+    private final FieldValue lookupKey;
+    private final String variableName;
+    private final Field fieldRef;
+    private final DataType resultingDataType;
+
+    public DataType getResultingDataType() {
+        return resultingDataType;
+    }
+
+    public Type getType() {
+        return type;
+    }
+
+    public Field getFieldRef() {
+        return fieldRef;
+    }
+
+    public int getLookupIndex() {
+        return lookupIndex;
+    }
+
+    public FieldValue getLookupKey() {
+        return lookupKey;
+    }
+
+    public String getVariableName() {
+        return variableName;
+    }
+
+    public String toString() {
+        String retVal = type.toString() + ": ";
+        switch (type) {
+            case STRUCT_FIELD:
+                retVal += getFieldRef();
+                break;
+            case ARRAY_INDEX:
+                retVal += getLookupIndex();
+                break;
+            case MAP_KEY:
+                retVal += getLookupKey();
+                break;
+            case MAP_ALL_KEYS:
+            case MAP_ALL_VALUES:
+                break;
+            case VARIABLE:
+                retVal += getVariableName();
+                break;
+        }
+        return retVal;
+    }
+
+    /**
+     * Creates a new field path entry that references a struct field.
+     * For these kinds of field path entries, getFieldRef() is valid.
+     *
+     * @param fieldRef The field to look up in the struct.
+     * @return The new field path entry
+     */
+    public static FieldPathEntry newStructFieldEntry(Field fieldRef) {
+        return new FieldPathEntry(fieldRef);
+    }
+
+    /**
+     * Creates a new field path entry that references an array index.
+     *
+     * @param lookupIndex The index to look up
+     * @param resultingDataType The datatype of the contents of the array
+     * @return The new field path entry
+     */
+    public static FieldPathEntry newArrayLookupEntry(int lookupIndex, DataType resultingDataType) {
+        return new FieldPathEntry(lookupIndex, resultingDataType);
+    }
+
+    /**
+     * Creates a new field path entry that references a map or weighted set.
+     *
+     * @param lookupKey The value of the key in the map or weighted set to recurse into.
+     * @param resultingDataType The datatype of values in the map or weighted set.
+     * @return The new field path entry
+     */
+    public static FieldPathEntry newMapLookupEntry(FieldValue lookupKey, DataType resultingDataType) {
+        return new FieldPathEntry(lookupKey, resultingDataType);
+    }
+
+    /**
+     * Creates a new field path entry that digs through all the keys of a map or weighted set.
+     *
+     * @param resultingDataType The datatype of the keys in the map or weighted set.
+     * @return The new field path entry.
+     */
+    public static FieldPathEntry newAllKeysLookupEntry(DataType resultingDataType) {
+        return new FieldPathEntry(true, false, resultingDataType);
+    }
+
+    /**
+     * Creates a new field path entry that digs through all the values of a map or weighted set.
+     *
+     * @param resultingDataType The datatype of the values in the map or weighted set.
+     * @return The new field path entry.
+     */
+    public static FieldPathEntry newAllValuesLookupEntry(DataType resultingDataType) {
+        return new FieldPathEntry(false, true, resultingDataType);
+    }
+
+    /**
+     * Creates a new field path entry that digs through all the keys in a map or weighted set, or all the indexes of an array,
+     * an sets the given variable name as it does so (or, if the variable is set, uses the set variable to look up the
+     * collection.
+     *
+     * @param variableName The name of the variable to lookup in the collection
+     * @param resultingDataType The value type of the collection we're digging through
+     * @return The new field path entry.
+     */
+    public static FieldPathEntry newVariableLookupEntry(String variableName, DataType resultingDataType) {
+        return new FieldPathEntry(variableName, resultingDataType);
+    }
+
+    private FieldPathEntry(Field fieldRef) {
+        type = Type.STRUCT_FIELD;
+        lookupIndex = 0;
+        lookupKey = null;
+        variableName = null;
+        this.fieldRef = fieldRef;
+        resultingDataType = fieldRef.getDataType();
+    }
+
+    private FieldPathEntry(int lookupIndex, DataType resultingDataType) {
+        type = Type.ARRAY_INDEX;
+        this.lookupIndex = lookupIndex;
+        lookupKey = null;
+        variableName = null;
+        fieldRef = null;
+        this.resultingDataType = resultingDataType;
+    }
+
+    private FieldPathEntry(FieldValue lookupKey, DataType resultingDataType) {
+        type = Type.MAP_KEY;
+        lookupIndex = 0;
+        this.lookupKey = lookupKey;
+        variableName = null;
+        fieldRef = null;
+        this.resultingDataType = resultingDataType;
+    }
+
+    private FieldPathEntry(boolean keysOnly, boolean valuesOnly, DataType resultingDataType) {
+        type = keysOnly ? Type.MAP_ALL_KEYS : Type.MAP_ALL_VALUES;
+        lookupIndex = 0;
+        lookupKey = null;
+        variableName = null;
+        fieldRef = null;
+        this.resultingDataType = resultingDataType;
+    }
+
+    private FieldPathEntry(String variableName, DataType resultingDataType) {
+        type = Type.VARIABLE;
+        lookupIndex = 0;
+        lookupKey = null;
+        this.variableName = variableName;
+        fieldRef = null;
+        this.resultingDataType = resultingDataType;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        FieldPathEntry that = (FieldPathEntry) o;
+
+        if (lookupIndex != that.lookupIndex) return false;
+        if (fieldRef != null ? !fieldRef.equals(that.fieldRef) : that.fieldRef != null) return false;
+        if (lookupKey != null ? !lookupKey.equals(that.lookupKey) : that.lookupKey != null) return false;
+        if (resultingDataType != null ? !resultingDataType.equals(that.resultingDataType) : that.resultingDataType != null)
+            return false;
+        if (type != that.type) return false;
+        if (variableName != null ? !variableName.equals(that.variableName) : that.variableName != null) return false;
+
+        return true;
+    }
+
+    @Override
+    public int hashCode() {
+        int result = type.hashCode();
+        result = 31 * result + lookupIndex;
+        result = 31 * result + (lookupKey != null ? lookupKey.hashCode() : 0);
+        result = 31 * result + (variableName != null ? variableName.hashCode() : 0);
+        result = 31 * result + (fieldRef != null ? fieldRef.hashCode() : 0);
+        result = 31 * result + (resultingDataType != null ? resultingDataType.hashCode() : 0);
+        return result;
+    }
+
+    public static class KeyParseResult {
+        public String parsed;
+        public int consumedChars;
+
+        public KeyParseResult(String parsed, int consumedChars) {
+            this.parsed = parsed;
+            this.consumedChars = consumedChars;
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+
+            KeyParseResult that = (KeyParseResult) o;
+
+            if (consumedChars != that.consumedChars) return false;
+            if (!parsed.equals(that.parsed)) return false;
+
+            return true;
+        }
+
+        @Override
+        public int hashCode() {
+            int result = parsed.hashCode();
+            result = 31 * result + consumedChars;
+            return result;
+        }
+
+        @Override
+        public String toString() {
+            return "KeyParseResult(parsed=\"" + parsed + "\", consumedChars=" + consumedChars + ")";
+        }
+    }
+
+    private static int parseQuotedString(String key, int offset,
+                                          int len, StringBuilder builder)
+    {
+        for (; offset < len && key.charAt(offset) != '"'; ++offset) {
+            if (key.charAt(offset) == '\\') {
+                ++offset; // Skip escape backslash
+                if (offset == len || key.charAt(offset) != '"') {
+                    throw new IllegalArgumentException("Escaped key '" + key + "' has bad quote character escape sequence. Expected '\"'");
+                }
+            }
+            if (offset < len) {
+                builder.append(key.charAt(offset));
+            }
+        }
+        if (offset < len && key.charAt(offset) == '"') {
+            return offset + 1;
+        } else {
+            throw new IllegalArgumentException("Escaped key '" + key + "' is incomplete. No matching '\"'");
+        }
+    }
+
+    private static int skipWhitespace(String str, int offset, int len) {
+        while (offset < len && Character.isSpaceChar(str.charAt(offset))) {
+            ++offset;
+        }
+        return offset;
+    }
+
+    /**
+     * Parse a field path map key of the form {xyz} or {"xyz"} with optional trailing data.
+     * If the key contains a '}' or '"' character, the key must be in quotes and all
+     * double-quote characters must be escaped. Only '"' chars may be escaped. Any
+     * trailing string data past the '}' is ignored.
+     *
+     * @param key Part of a field path that contains a key at its start
+     * @return A parse result containing the parsed/unescaped key and the number
+     *     of input characters the parse consumed. Does not include any characters
+     *     beyond the '}' char.
+     */
+    public static KeyParseResult parseKey(String key) {
+        StringBuilder parsed = new StringBuilder(key.length());
+        // Hooray for ad-hoc parsing
+        int len = key.length();
+        int i = 0;
+        if (i < len && key.charAt(0) == '{') {
+            i = skipWhitespace(key, i + 1, len);
+            if (i < len && key.charAt(i) == '"') {
+                i = parseQuotedString(key, i + 1, len, parsed);
+            } else {
+                // No quoting, use all of string until '}' verbatim
+                while (i < len && key.charAt(i) != '}') {
+                    parsed.append(key.charAt(i));
+                    ++i;
+                }
+            }
+            i = skipWhitespace(key, i, len);
+            if (i < len && key.charAt(i) == '}') {
+                return new KeyParseResult(parsed.toString(), i + 1);
+            } else {
+                throw new IllegalArgumentException("Key '" + key + "' is incomplete. No matching '}'");
+            }
+        } else {
+            throw new IllegalArgumentException("Key '" + key + "' does not start with '{'");
+        }
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/Generated.java b/document/src/main/java/com/yahoo/document/Generated.java
new file mode 100644
index 00000000000..40026eed51f
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/Generated.java
@@ -0,0 +1,17 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+
+/**
+ * Classes generated by vespa-documentgen-plugin are annotated with this.  It
+ * differs from <code>javax.annotation.Generated</code> in that the retention
+ * policy is Runtime.
+ *
+ * @author <a href="mailto:vegardh@yahoo-inc.com">Vegard Havdal</a>
+ */
+@Retention(RetentionPolicy.RUNTIME)
+public @interface Generated {
+
+}
diff --git a/document/src/main/java/com/yahoo/document/GlobalId.java b/document/src/main/java/com/yahoo/document/GlobalId.java
new file mode 100644
index 00000000000..d1ab8c9cea9
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/GlobalId.java
@@ -0,0 +1,152 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import com.yahoo.collections.MD5;
+import com.yahoo.document.idstring.IdString;
+import com.yahoo.text.Utf8;
+import com.yahoo.text.Utf8String;
+import com.yahoo.vespa.objects.Deserializer;
+import com.yahoo.vespa.objects.Serializer;
+
+import java.nio.ByteBuffer;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.Arrays;
+
+/**
+ * Implements an incredibly light-weight version of the document global id. There is a lot of functionality in the C++
+ * version of this that is missing. However, this should be sufficient for now.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class GlobalId implements Comparable {
+
+    /**
+     * The number of bytes in a global id. This must match the C++ constant in "document/base/globalid.h".
+     */
+    public static final int LENGTH = 12;
+
+    // The raw bytes that constitutes this global id.
+    private final byte[] raw;
+
+    /**
+     * Constructs a new global id by copying the content of the given raw byte array.
+     *
+     * @param raw The array to copy.
+     */
+    public GlobalId(byte[] raw) {
+        this.raw = new byte [12];
+        int len = Math.min(LENGTH, raw.length);
+        System.arraycopy(raw, 0, this.raw, 0, len);
+    }
+
+    /**
+     * Constructs a new global id from a document id string.
+     *
+     * @param id The document id to derive from.
+     */
+    public GlobalId(IdString id) {
+        byte [] raw = MD5.md5.get().digest(id.toUtf8().wrap().array());
+        long location = id.getLocation();
+        this.raw = new byte [LENGTH];
+        for (int i = 0; i < 4; ++i) {
+            this.raw[i] = (byte)((location >> (8 * i)) & 0xFF);
+        }
+        for (int i=4; i < LENGTH; i++) {
+            this.raw[i] = raw[i];
+        }
+    }
+
+    /**
+     * Constructs a global id by deserializing content from the given byte buffer.
+     *
+     * @param buf The buffer to deserialize from.
+     */
+    public GlobalId(Deserializer buf) {
+        raw = buf.getBytes(null, LENGTH);
+    }
+
+    /**
+     * Serializes the content of this global id into the given byte buffer.
+     *
+     * @param buf The buffer to serialize to.
+     */
+    public void serialize(Serializer buf) {
+        buf.put(null, raw);
+    }
+
+    /**
+     * Returns the raw byte array that constitutes this global id.
+     *
+     * @return The byte array.
+     */
+    public byte[] getRawId() {
+        return raw;
+    }
+
+    // Inherit doc from Object.
+    @Override
+    public int hashCode() {
+        return Arrays.hashCode(raw);
+    }
+
+    public BucketId toBucketId() {
+        /**
+         * Explanation time: since Java was designed so mankind could suffer,
+         * shift ops on bytes have an implicit int conversion with sign-extend.
+         * When a byte is negative, you end up with an int/long with a 0xFFFFFF
+         * prefix, in turn causing your other friendly bitwise ORs to act
+         * pretty far from what was originally intended.
+         * To get around this, we explicitly sign extend before the compiler can
+         * do so for us and make sure to OR away any sign extensions.
+         */
+        long location = ((long)raw[0] & 0xFF)
+                | (((long)raw[1] & 0xFF) << 8)
+                | (((long)raw[2] & 0xFF) << 16)
+                | (((long)raw[3] & 0xFF) << 24);
+        long md5 = 0;
+        for (int i = 4, j = 0; i < LENGTH; i++, j += 8) {
+            md5 |= ((long)raw[i] & 0xFF) << j;
+        }
+        // Drumroll: this is why 'location' is of type long. Otherwise, the
+        // ORing would sign-extend it and cause havoc when its MSB is set.
+        long rawBucketId = (md5 & 0xFFFFFFFF00000000L) | location;
+        return new BucketId(58, rawBucketId);
+    }
+
+    // Inherit doc from Object.
+    @Override
+    public boolean equals(Object obj) {
+        if (!(obj instanceof GlobalId)) {
+            return false;
+        }
+        GlobalId rhs = (GlobalId) obj;
+        return Arrays.equals(raw, rhs.raw);
+    }
+
+    public int compareTo(Object o) {
+        GlobalId other = (GlobalId) o;
+
+        for (int i=0 ; i<LENGTH; i++) {
+            int thisByte = 0xF & (int) raw[i];
+            int otherByte = 0xF & (int) other.raw[i];
+
+            if (thisByte < otherByte) {
+                return -1;
+            } else if (thisByte > otherByte) {
+                return 1;
+            }
+        }
+        return 0;
+    }
+
+    @Override
+    public String toString() {
+        StringBuilder strb = new StringBuilder(50);
+        for (byte b : raw) {
+            strb.append(" ").append(0xFF & (int) b);
+        }
+        return strb.toString().trim();
+    }
+
+}
diff --git a/document/src/main/java/com/yahoo/document/MapDataType.java b/document/src/main/java/com/yahoo/document/MapDataType.java
new file mode 100644
index 00000000000..2f3c6be99dc
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/MapDataType.java
@@ -0,0 +1,135 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.MapFieldValue;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Represents a map type.
+ *
+ * @author vegardh
+ */
+public class MapDataType extends DataType {
+
+    private DataType keyType;
+    private DataType valueType;
+
+    public MapDataType(DataType keyType, DataType valueType, int id) {
+        super("Map<"+keyType.getName()+","+valueType.getName()+">", id);
+        this.keyType=keyType;
+        this.valueType = valueType;
+
+    }
+
+    public MapDataType(DataType keyType, DataType valueType) {
+        this(keyType, valueType, 0);
+        setId(getName().toLowerCase().hashCode());
+    }
+
+    @Override
+    public MapDataType clone() {
+        MapDataType type = (MapDataType)super.clone();
+        type.keyType = keyType.clone();
+        type.valueType = valueType.clone();
+        return type;
+    }
+
+    @Override
+    protected FieldValue createByReflection(Object arg) { return null; }
+
+    @Override
+    public boolean isValueCompatible(FieldValue value) {
+        return value.getDataType().equals(this);
+    }
+
+    public DataType getKeyType() {
+        return keyType;
+    }
+
+    public DataType getValueType() {
+        return valueType;
+    }
+
+    /**
+     * Sets the key type of this MapDataType.&nbsp;WARNING! Do not use! Only to be used by config system!
+     */
+    public void setKeyType(DataType keyType) {
+        this.keyType = keyType;
+    }
+
+    /**
+     * Sets the key type of this MapDataType.&nbsp;WARNING! Do not use! Only to be used by config system!
+     */
+    public void setValueType(DataType valueType) {
+        this.valueType = valueType;
+    }
+
+
+    @Override
+    public MapFieldValue createFieldValue() {
+        return new MapFieldValue(this);
+    }
+
+    @Override
+    public Class getValueClass() {
+        return MapFieldValue.class;
+    }
+
+    @Override
+    protected void register(DocumentTypeManager manager,
+            List<DataType> seenTypes) {
+        seenTypes.add(this);
+        if (!seenTypes.contains(getKeyType())) {
+            getKeyType().register(manager, seenTypes);
+        }
+        if (!seenTypes.contains(getValueType())) {
+            getValueType().register(manager, seenTypes);
+        }
+        super.register(manager, seenTypes);
+    }
+
+    public static FieldPath buildFieldPath(String remainFieldName, DataType keyType, DataType valueType) {
+        if (remainFieldName.length() > 0 && remainFieldName.charAt(0) == '{') {
+            FieldPathEntry.KeyParseResult result = FieldPathEntry.parseKey(remainFieldName);
+            String keyValue = result.parsed;
+
+            FieldPath path = valueType.buildFieldPath(skipDotInString(remainFieldName, result.consumedChars - 1));
+            List<FieldPathEntry> tmpPath = new ArrayList<FieldPathEntry>(path.getList());
+
+            if (remainFieldName.charAt(1) == '$') {
+                tmpPath.add(0, FieldPathEntry.newVariableLookupEntry(keyValue.substring(1), valueType));
+            } else {
+                FieldValue fv = keyType.createFieldValue();
+                fv.assign(keyValue);
+                tmpPath.add(0, FieldPathEntry.newMapLookupEntry(fv, valueType));
+            }
+
+            return new FieldPath(tmpPath);
+
+        } else if (remainFieldName.startsWith("key")) {
+            FieldPath path = keyType.buildFieldPath(skipDotInString(remainFieldName, 2));
+            List<FieldPathEntry> tmpPath = new ArrayList<FieldPathEntry>(path.getList());
+            tmpPath.add(0, FieldPathEntry.newAllKeysLookupEntry(keyType));
+            return new FieldPath(tmpPath);
+        } else if (remainFieldName.startsWith("value")) {
+            FieldPath path = valueType.buildFieldPath(skipDotInString(remainFieldName, 4));
+            List<FieldPathEntry> tmpPath = new ArrayList<FieldPathEntry>(path.getList());
+            tmpPath.add(0, FieldPathEntry.newAllValuesLookupEntry(valueType));
+            return new FieldPath(tmpPath);
+        }
+
+        return keyType.buildFieldPath(remainFieldName);
+    }
+
+    @Override
+    public FieldPath buildFieldPath(String remainFieldName) {
+        return buildFieldPath(remainFieldName, getKeyType(), getValueType());
+    }
+
+    @Override
+    public boolean isMultivalue() { return true; }
+
+}
diff --git a/document/src/main/java/com/yahoo/document/NumericDataType.java b/document/src/main/java/com/yahoo/document/NumericDataType.java
new file mode 100644
index 00000000000..20e02914a07
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/NumericDataType.java
@@ -0,0 +1,27 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import com.yahoo.vespa.objects.Ids;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class NumericDataType extends PrimitiveDataType {
+    // The global class identifier shared with C++.
+    public static int classId = registerClass(Ids.document + 52, NumericDataType.class);
+    /**
+     * Creates a datatype
+     *
+     * @param name      the name of the type
+     * @param code      the code (id) of the type
+     * @param type      the field value used for this type
+     */
+    protected NumericDataType(java.lang.String name, int code, Class type, Factory factory) {
+        super(name, code, type, factory);
+    }
+
+    @Override
+    public NumericDataType clone() {
+        return (NumericDataType) super.clone();
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/PositionDataType.java b/document/src/main/java/com/yahoo/document/PositionDataType.java
new file mode 100644
index 00000000000..a2a1c6012a1
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/PositionDataType.java
@@ -0,0 +1,93 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.IntegerFieldValue;
+import com.yahoo.document.datatypes.Struct;
+import com.yahoo.geo.DegreesParser;
+import com.yahoo.document.serialization.XmlStream;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public final class PositionDataType {
+
+    public static final StructDataType INSTANCE = newInstance();
+    public static final String STRUCT_NAME = "position";
+
+    public static final String FIELD_X = "x";
+    public static final String FIELD_Y = "y";
+    private static final Field FFIELD_X = INSTANCE.getField(FIELD_X);
+    private static final Field FFIELD_Y = INSTANCE.getField(FIELD_Y);
+
+    private PositionDataType() {
+        // unreachable
+    }
+
+    public static String renderAsString(Struct pos) {
+        StringBuilder buf = new StringBuilder();
+        double ns = getYValue(pos).getInteger() / 1.0e6;
+        double ew = getXValue(pos).getInteger() / 1.0e6;
+        buf.append(ns < 0 ? "S" : "N");
+        buf.append(ns < 0 ? (-ns) : ns);
+        buf.append(";");
+        buf.append(ew < 0 ? "W" : "E");
+        buf.append(ew < 0 ? (-ew) : ew);
+        return buf.toString();
+    }
+
+    public static void renderXml(Struct pos, XmlStream target) {
+        target.addContent(renderAsString(pos));
+    }
+
+    public static Struct valueOf(Integer x, Integer y) {
+        Struct ret = new Struct(INSTANCE);
+        ret.setFieldValue(FIELD_X, x != null ? new IntegerFieldValue(x) : null);
+        ret.setFieldValue(FIELD_Y, y != null ? new IntegerFieldValue(y) : null);
+        return ret;
+    }
+
+    public static Struct fromLong(long val) {
+        return valueOf((int)(val >> 32), (int)val);
+    }
+
+    public static Struct fromString(String str) {
+        try {
+            DegreesParser d = new DegreesParser(str);
+            return valueOf((int)(d.longitude * 1000000), (int)(d.latitude * 1000000));
+        } catch (IllegalArgumentException e) {
+            // empty
+        }
+        String[] arr = str.split(";", 2);
+        return valueOf(Integer.parseInt(arr[0]), Integer.parseInt(arr[1]));
+    }
+
+    public static IntegerFieldValue getXValue(FieldValue pos) {
+        return Struct.getFieldValue(pos, INSTANCE, FFIELD_X, IntegerFieldValue.class);
+    }
+
+    public static IntegerFieldValue getYValue(FieldValue pos) {
+        return Struct.getFieldValue(pos, INSTANCE, FFIELD_Y, IntegerFieldValue.class);
+    }
+
+    public static String getZCurveFieldName(String fieldName) {
+        return fieldName + "_zcurve";
+    }
+
+    public static String getPositionSummaryFieldName(String fieldName) {
+        // TODO for 6.0, rename to _position to use a field name that is actually legal
+        return fieldName + ".position";
+    }
+
+    public static String getDistanceSummaryFieldName(String fieldName) {
+        // TODO for 6.0, rename to _distance to use a field name that is actually legal
+        return fieldName + ".distance";
+    }
+
+    private static StructDataType newInstance() {
+        StructDataType ret = new StructDataType(STRUCT_NAME);
+        ret.addField(new Field(FIELD_X, DataType.INT));
+        ret.addField(new Field(FIELD_Y, DataType.INT));
+        return ret;
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/PrimitiveDataType.java b/document/src/main/java/com/yahoo/document/PrimitiveDataType.java
new file mode 100644
index 00000000000..a0024bb0497
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/PrimitiveDataType.java
@@ -0,0 +1,67 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.vespa.objects.Ids;
+import com.yahoo.vespa.objects.ObjectVisitor;
+
+import java.util.Objects;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class PrimitiveDataType extends DataType {
+    public static abstract class Factory {
+        public abstract FieldValue create();
+    }
+
+    // The global class identifier shared with C++.
+    public static final int classId = registerClass(Ids.document + 51, PrimitiveDataType.class);
+    private final Class<? extends FieldValue> valueClass;
+    private final Factory factory;
+
+    /**
+     * Creates a datatype
+     *
+     * @param name     the name of the type
+     * @param code     the code (id) of the type
+     * @param factory  the factory for creating field values of this type
+     */
+    protected PrimitiveDataType(java.lang.String name, int code, Class<? extends FieldValue> valueClass, Factory factory) {
+        super(name, code);
+        Objects.requireNonNull(valueClass, "valueClass");
+        Objects.requireNonNull(factory, "factory");
+        this.valueClass = valueClass;
+        this.factory = factory;
+    }
+
+    @Override
+    public PrimitiveDataType clone() {
+        return (PrimitiveDataType)super.clone();
+    }
+
+    public FieldValue createFieldValue() {
+        return factory.create();
+    }
+
+    @Override
+    public Class<? extends FieldValue> getValueClass() {
+        return valueClass;
+    }
+
+    @Override
+    public boolean isValueCompatible(FieldValue value) {
+        return value != null && valueClass.isAssignableFrom(value.getClass());
+    }
+
+    @Override
+    public PrimitiveDataType getPrimitiveType() {
+        return this;
+    }
+
+    @Override
+    public void visitMembers(ObjectVisitor visitor) {
+        super.visitMembers(visitor);
+        visitor.visit("valueclass", valueClass.getName());
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/SimpleDocument.java b/document/src/main/java/com/yahoo/document/SimpleDocument.java
new file mode 100644
index 00000000000..b461356ecef
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/SimpleDocument.java
@@ -0,0 +1,72 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.StructuredFieldValue;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class SimpleDocument {
+
+    private final Document document;
+
+    public SimpleDocument(Document document) {
+        this.document = document;
+    }
+
+    public final Object get(Field field) {
+        return get(document, field);
+    }
+
+    public final Object get(String fieldName) {
+        return get(document, document.getField(fieldName));
+    }
+
+    public final Object set(Field field, Object value) {
+        return set(document, field, value);
+    }
+
+    public final Object set(String fieldName, Object value) {
+        return set(document.getField(fieldName), value);
+    }
+
+    public final Object remove(Field field) {
+        return remove(document, field);
+    }
+
+    public final Object remove(String fieldName) {
+        return remove(document.getField(fieldName));
+    }
+
+    public static Object get(StructuredFieldValue struct, Field field) {
+        return field == null ? null : unwrapValue(struct.getFieldValue(field));
+    }
+
+    public static Object set(StructuredFieldValue struct, Field field, Object value) {
+        return unwrapValue(struct.setFieldValue(field, wrapValue(field.getDataType(), value)));
+    }
+
+    public static Object remove(StructuredFieldValue struct, Field field) {
+        return field == null ? null : unwrapValue(struct.removeFieldValue(field));
+    }
+
+    private static FieldValue wrapValue(DataType type, Object val) {
+        if (val == null) {
+            return null;
+        }
+        if (val instanceof FieldValue) {
+            return (FieldValue)val;
+        }
+        FieldValue ret = type.createFieldValue();
+        ret.assign(val);
+        return ret;
+    }
+
+    private static Object unwrapValue(FieldValue val) {
+        if (val == null) {
+            return null;
+        }
+        return val.getWrappedValue();
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/StructDataType.java b/document/src/main/java/com/yahoo/document/StructDataType.java
new file mode 100644
index 00000000000..3f26cbe054f
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/StructDataType.java
@@ -0,0 +1,191 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import com.google.common.collect.ImmutableList;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.Struct;
+import com.yahoo.vespa.objects.Ids;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class StructDataType extends BaseStructDataType {
+
+    public static final int classId = registerClass(Ids.document + 57, StructDataType.class);
+    private StructDataType superType = null;
+
+    public StructDataType(String name) {
+        super(name);
+    }
+
+    public StructDataType(int id,String name) {
+        super(id, name);
+    }
+
+    @Override
+    public Struct createFieldValue() {
+        return new Struct(this);
+    }
+
+    @Override
+    public FieldValue createFieldValue(Object o) {
+        Struct struct;
+        if (o.getClass().equals(Struct.class)) {
+            struct = new Struct(this);
+        } else {
+            // This indicates for example that o is a generated struct subtype, try the empty constructor
+            try {
+                struct = (Struct) o.getClass().getConstructor().newInstance();
+            } catch (Exception e) {
+                // Fallback, let assign handle the error if o is completely bogus
+                struct = new Struct(this);
+            }
+        }
+        struct.assign(o);
+        return struct;
+    }
+
+    @Override
+    public StructDataType clone() {
+        StructDataType type = (StructDataType) super.clone();
+        type.superType = this.superType;
+        return type;
+    }
+
+    public void assign(StructDataType type) {
+        super.assign(type);
+        superType = type.superType;
+    }
+
+    @Override
+    public Field getField(Integer fieldId, int version) {
+        Field f = super.getField(fieldId, version);
+        if (f == null && superType != null) {
+            f = superType.getField(fieldId, version);
+        }
+        return f;
+    }
+
+    @Override
+    public Field getField(String fieldName) {
+        Field f = super.getField(fieldName);
+        if (f == null && superType != null) {
+            f = superType.getField(fieldName);
+        }
+        return f;
+    }
+
+    @Override
+    public Field getField(int id) {
+        Field f = super.getField(id);
+        if (f == null && superType != null) {
+            f = superType.getField(id);
+        }
+        return f;
+    }
+
+    @Override
+    public void addField(Field field) {
+        if (hasField(field)) {
+            throw new IllegalArgumentException("Struct already has field " + field);
+        }
+        if ((superType != null) && superType.hasField(field)) {
+            throw new IllegalArgumentException(field.toString() + " already present in inherited type '" + superType.toString() + "', " + this.toString() + " cannot override.");
+        }
+        super.addField(field);
+    }
+
+    @Override
+    public boolean hasField(Field field, int version) {
+        boolean f = super.hasField(field, version);
+        if (!f && superType != null) {
+            f = superType.hasField(field, version);
+        }
+        return f;
+    }
+
+    @Override
+    public Collection<Field> getFields() {
+        if (superType == null) {
+            return Collections.unmodifiableCollection(super.getFields());
+        }
+        Collection<Field> fieldsBuilder = new ArrayList<>();
+        fieldsBuilder.addAll(super.getFields());
+        fieldsBuilder.addAll(superType.getFields());
+        return ImmutableList.copyOf(fieldsBuilder);
+    }
+
+    public Collection<Field> getFieldsThisTypeOnly() {
+        return Collections.unmodifiableCollection(super.getFields());
+    }
+
+    @Override
+    public int getFieldCount() {
+        return getFields().size();
+    }
+
+    @Override
+    public Class getValueClass() {
+        return Struct.class;
+    }
+
+    @Override
+    public boolean isValueCompatible(FieldValue value) {
+        if (!(value instanceof Struct)) {
+            return false;
+        }
+        Struct structValue = (Struct) value;
+        if (structValue.getDataType().inherits(this)) {
+            //the value is of this type; or the supertype of the value is of this type, etc....
+            return true;
+        }
+        return false;
+    }
+
+    public void inherit(StructDataType type) {
+        if (superType != null) {
+            throw new IllegalArgumentException("Already inherits type " + superType + ", multiple inheritance not currently supported.");
+        }
+        for (Field f : type.getFields()) {
+            if (hasField(f)) {
+                throw new IllegalArgumentException(f + " already present in " + type + ", " + this + " cannot inherit from it");
+            }
+        }
+        superType = type;
+    }
+
+    public Collection<StructDataType> getInheritedTypes() {
+        if (superType == null) {
+            return ImmutableList.of();
+        }
+        return ImmutableList.of(superType);
+    }
+
+    public boolean inherits(StructDataType type) {
+        if (equals(type)) return true;
+        if (superType != null && superType.inherits(type)) return true;
+        return false;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (!(o instanceof StructDataType)) return false;
+        if (!super.equals(o)) return false;
+
+        StructDataType that = (StructDataType) o;
+        if (superType != null ? !superType.equals(that.superType) : that.superType != null) return false;
+        return true;
+    }
+
+    @Override
+    public int hashCode() {
+        int result = super.hashCode();
+        result = 31 * result + (superType != null ? superType.hashCode() : 0);
+        return result;
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/StructuredDataType.java b/document/src/main/java/com/yahoo/document/StructuredDataType.java
new file mode 100644
index 00000000000..18318913742
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/StructuredDataType.java
@@ -0,0 +1,124 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.StructuredFieldValue;
+import com.yahoo.vespa.objects.Ids;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+/**
+ * TODO: What is this and why
+ *
+ * @author <a href="mailto:humbe@yahoo-inc.com">Håkon Humberset</a>
+ */
+public abstract class StructuredDataType extends DataType {
+
+    public static final int classId = registerClass(Ids.document + 56, StructuredDataType.class);
+
+    protected static int createId(String name) {
+        if (name.equals("document")) return 8;
+
+        // This is broken really because we now depend on String.hashCode staying the same in Java vm's
+        // which is likely for pragmatic reasons but not by contract
+        return (name+".0").hashCode(); // the ".0" must be preserved to keep hashCodes the same after we removed version
+    }
+
+    public StructuredDataType(String name) {
+        super(name, createId(name));
+    }
+
+    public StructuredDataType(int id, String name) {
+        super(name, id);
+    }
+
+    @Override
+    public abstract StructuredFieldValue createFieldValue();
+
+    @Override
+    protected FieldValue createByReflection(Object arg) { return null; }
+
+    /**
+     * Returns the name of this as a DataTypeName
+     *
+     * @return Return the Documentname of this doumenttype.
+     */
+    public DataTypeName getDataTypeName() {
+        return new DataTypeName(getName());
+    }
+
+    /**
+     * Gets the field  matching a given name.
+     *
+     * @param name The name of a field.
+     * @return Returns the matching field, or null if not found.
+     */
+    public abstract Field getField(String name);
+
+    /**
+     * Gets the field with the specified id.
+     *
+     * @param id the id of the field to return.
+     * @return the matching field, or null if not found.
+     */
+    public abstract Field getField(int id);
+
+    public abstract Collection<Field> getFields();
+
+    @Override
+    public boolean equals(Object o) {
+        return ((o instanceof StructuredDataType) && super.equals(o));
+    }
+
+    @Override
+    public int hashCode() {
+        return super.hashCode();
+    }
+
+    @Override
+    protected void register(DocumentTypeManager manager, List<DataType> seenTypes) {
+        seenTypes.add(this);
+        for (Field field : getFields()) {
+            if (!seenTypes.contains(field.getDataType())) {
+                //we haven't seen this one before, register it:
+                field.getDataType().register(manager, seenTypes);
+            }
+        }
+        super.register(manager, seenTypes);
+    }
+
+    @Override
+    public FieldPath buildFieldPath(String remainFieldName) {
+        if (remainFieldName.length() == 0) {
+            return new FieldPath();
+        }
+
+        String currFieldName = remainFieldName;
+        String subFieldName = "";
+
+        for (int i = 0; i < remainFieldName.length(); i++) {
+            if (remainFieldName.charAt(i) == '.') {
+                currFieldName = remainFieldName.substring(0, i);
+                subFieldName = remainFieldName.substring(i + 1);
+                break;
+            } else if (remainFieldName.charAt(i) == '{' || remainFieldName.charAt(i) == '[') {
+                currFieldName = remainFieldName.substring(0, i);
+                subFieldName = remainFieldName.substring(i);
+                break;
+            }
+        }
+
+        Field f = getField(currFieldName);
+        if (f != null) {
+            FieldPath fieldPath = f.getDataType().buildFieldPath(subFieldName);
+            List<FieldPathEntry> tmpPath = new ArrayList<FieldPathEntry>(fieldPath.getList());
+            tmpPath.add(0, FieldPathEntry.newStructFieldEntry(f));
+            return new FieldPath(tmpPath);
+        } else {
+            throw new IllegalArgumentException("Field '" + currFieldName + "' not found in type " + this);
+        }
+    }
+
+}
diff --git a/document/src/main/java/com/yahoo/document/TemporaryDataType.java b/document/src/main/java/com/yahoo/document/TemporaryDataType.java
new file mode 100644
index 00000000000..da65dde72da
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/TemporaryDataType.java
@@ -0,0 +1,28 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import com.yahoo.document.datatypes.FieldValue;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+class TemporaryDataType extends DataType {
+    TemporaryDataType(int dataTypeId) {
+        super("temporary_" + dataTypeId, dataTypeId);
+    }
+
+    @Override
+    public FieldValue createFieldValue() {
+        return null;
+    }
+
+    @Override
+    public Class getValueClass() {
+        return null;
+    }
+
+    @Override
+    public boolean isValueCompatible(FieldValue value) {
+        return false;
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/TemporaryStructuredDataType.java b/document/src/main/java/com/yahoo/document/TemporaryStructuredDataType.java
new file mode 100644
index 00000000000..51cc6f94f2e
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/TemporaryStructuredDataType.java
@@ -0,0 +1,23 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+/**
+ * Internal class, DO NOT USE!!&nbsp;Only public because it must be used from com.yahoo.searchdefinition.parser.
+ *
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class TemporaryStructuredDataType extends StructDataType {
+    TemporaryStructuredDataType(String name) {
+        super(name);
+    }
+
+    public static TemporaryStructuredDataType create(String name) {
+        return new TemporaryStructuredDataType(name);
+    }
+
+    @Override
+    protected void setName(String name) {
+        super.setName(name);
+        setId(createId(getName()));
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/TestAndSetCondition.java b/document/src/main/java/com/yahoo/document/TestAndSetCondition.java
new file mode 100644
index 00000000000..5c0d83678c8
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/TestAndSetCondition.java
@@ -0,0 +1,46 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import com.google.common.annotations.Beta;
+
+import java.util.Optional;
+
+/**
+ * The TestAndSetCondition class represents a test and set condition.
+ * A test and set condition is an (optional) string representing a
+ * document selection (cf. document selection language), which is used
+ * to match a document for test and set. If #isPresent evaluates to false,
+ * the condition is not present and matches any document.
+ *
+ * @author Vegard Sjonfjell
+ */
+@Beta
+public class TestAndSetCondition {
+    public static final TestAndSetCondition NOT_PRESENT_CONDITION = new TestAndSetCondition();
+
+    private final String conditionStr;
+
+    public TestAndSetCondition() {
+        this("");
+    }
+
+    public TestAndSetCondition(String conditionStr) {
+        this.conditionStr = conditionStr;
+    }
+
+    public String getSelection() { return conditionStr; }
+
+    public boolean isPresent() { return !conditionStr.isEmpty(); }
+
+    /**
+     * Maps and optional test and set conditiong string to a TestAndSetCondition.
+     * If the condition string is not present, a "not present" condition is returned
+     * @param conditionString test and set conditiong string (document selection)
+     * @return a TestAndSetCondition representing the condition string or a "not present" condition
+     */
+    public static TestAndSetCondition fromConditionString(Optional<String> conditionString) {
+        return conditionString
+                .map(TestAndSetCondition::new)
+                .orElse(TestAndSetCondition.NOT_PRESENT_CONDITION);
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/WeightedSetDataType.java b/document/src/main/java/com/yahoo/document/WeightedSetDataType.java
new file mode 100644
index 00000000000..9674d39fea8
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/WeightedSetDataType.java
@@ -0,0 +1,118 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import com.yahoo.document.datatypes.WeightedSet;
+import com.yahoo.vespa.objects.Ids;
+import com.yahoo.vespa.objects.ObjectVisitor;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class WeightedSetDataType extends CollectionDataType {
+    // The global class identifier shared with C++.
+    public static int classId = registerClass(Ids.document + 55, WeightedSetDataType.class);
+
+    /** Should an arith operation to a non-existant member of a weightedset cause the member to be created */
+    private boolean createIfNonExistent = false;
+
+    /** Should a member of a weightedset with weight 0 be removed */
+    private boolean removeIfZero = false;
+
+    /** The tag type is ambiguous, this flag is true if the user explicitly set a field to tag */
+    private boolean tag = false;
+
+    public WeightedSetDataType(DataType nestedType, boolean createIfNonExistent, boolean removeIfZero) {
+        this(nestedType, createIfNonExistent, removeIfZero, 0);
+        if ((nestedType == STRING) && createIfNonExistent && removeIfZero) {
+            setId(18);
+        } else {
+            setId(getName().toLowerCase().hashCode());
+        }
+    }
+
+    public WeightedSetDataType(DataType nestedType, boolean createIfNonExistent, boolean removeIfZero, int id) {
+        super(createName(nestedType, createIfNonExistent, removeIfZero), id, nestedType);
+        this.createIfNonExistent = createIfNonExistent;
+        this.removeIfZero = removeIfZero;
+    }
+
+    public WeightedSetDataType(String typeName, int code, DataType nestedType, boolean createIfNonExistent, boolean removeIfZero) {
+        super(typeName != null ? createName(nestedType, createIfNonExistent, removeIfZero) : null, code, nestedType);
+        if ((code >= 0) && (code <= DataType.lastPredefinedDataTypeId()) && (code != 18)) // 18 == DataType.TAG.getId() is not yet initialized
+            throw new IllegalArgumentException("Cannot create a weighted set datatype with code " + code);
+        this.createIfNonExistent = createIfNonExistent;
+        this.removeIfZero = removeIfZero;
+    }
+
+    @Override
+    public WeightedSetDataType clone() {
+        return (WeightedSetDataType) super.clone();
+    }
+
+    /**
+     * Called by SD parser if a data type is explicitly tag.
+     * @param tag True if this is a tag set.
+     */
+    public void setTag(boolean tag) {
+        this.tag = tag;
+    }
+
+    /**
+     * Returns whether or not this is a <em>tag</em> type weighted set.
+     * @return True if this is a tag set.
+     */
+    public boolean isTag() {
+        return tag;
+    }
+
+    static private String createName(DataType nested, boolean createIfNonExistant, boolean removeIfZero) {
+        if (nested == DataType.STRING && createIfNonExistant && removeIfZero) {
+            return "tag";
+        } else {
+            String name = "WeightedSet<" + nested.getName() + ">";
+            if (createIfNonExistant) name += ";Add";
+            if (removeIfZero) name += ";Remove";
+            return name;
+        }
+    }
+
+    @Override
+    public WeightedSet createFieldValue() {
+        return new WeightedSet(this);
+    }
+
+    @Override
+    public Class getValueClass() {
+        return WeightedSet.class;
+    }
+
+    /**
+     * Returns true if this has the property createIfNonExistent (only relevant for weighted sets)
+     *
+     * @return createIfNonExistent property
+     */
+    public boolean createIfNonExistent() {
+        return createIfNonExistent;
+    }
+
+    /**
+     * Returns true if this has the property removeIfZero (only relevant for weighted sets)
+     *
+     * @return removeIfZero property
+     */
+    public boolean removeIfZero() {
+        return removeIfZero;
+    }
+    @Override
+    public void visitMembers(ObjectVisitor visitor) {
+        super.visitMembers(visitor);
+        visitor.visit("removeIfZero", removeIfZero);
+        visitor.visit("createIfNonExistent", createIfNonExistent);
+    }
+
+    @Override
+    public FieldPath buildFieldPath(String remainFieldName)
+    {
+        return MapDataType.buildFieldPath(remainFieldName, getNestedType(), DataType.INT);
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/annotation/AlternateSpanList.java b/document/src/main/java/com/yahoo/document/annotation/AlternateSpanList.java
new file mode 100644
index 00000000000..9f2f8a9b160
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/annotation/AlternateSpanList.java
@@ -0,0 +1,634 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+import com.yahoo.document.serialization.SpanNodeReader;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.ListIterator;
+
+/**
+ * A node in a {@link SpanNode} tree that can have a <strong>multiple</strong> trees of child nodes, each with its own probability.
+ * This class has quite a few convenience methods for accessing the <strong>first</strong> subtree.
+ *
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ * @see com.yahoo.document.annotation.SpanList
+ */
+public class AlternateSpanList extends SpanList {
+    public static final byte ID = 4;
+    private final List<Children> childTrees = new LinkedList<Children>();
+    private static final Comparator<Children> childComparator = new ProbabilityComparator();
+
+    /** Create a new AlternateSpanList instance, having a single subtree with probability 1.0. */
+    public AlternateSpanList() {
+        super((List<SpanNode>) null);
+        ensureAtLeastOneSubTree();
+    }
+
+    /*
+     * Deep-copies another AlternateSpanList.
+     *
+     * @param otherSpanList the instance to deep-copy.
+     */
+    public AlternateSpanList(AlternateSpanList otherSpanList) {
+        super((List<SpanNode>) null);
+        for (Children otherSubtree : otherSpanList.childTrees) {
+            //create our own subtree:
+            Children children = new Children(this);
+            //copy nodes:
+            for (SpanNode otherNode : otherSubtree.children()) {
+                if (otherNode instanceof Span) {
+                    children.add(new Span((Span) otherNode));
+                } else if (otherNode instanceof AlternateSpanList) {
+                    children.add(new AlternateSpanList((AlternateSpanList) otherNode));
+                } else if (otherNode instanceof SpanList) {
+                    children.add(new SpanList((SpanList) otherNode));
+                } else if (otherNode instanceof DummySpanNode) {
+                    children.add(otherNode);  //shouldn't really happen
+                } else {
+                    throw new IllegalStateException("Cannot create copy of " + otherNode + " with class "
+                                                    + ((otherNode == null) ? "null" : otherNode.getClass()));
+                }
+            }
+            //add this subtree to our subtrees:
+            childTrees.add(children);
+        }
+    }
+
+    public AlternateSpanList(SpanNodeReader reader) {
+        this();
+        reader.read(this);
+    }
+
+    private void ensureAtLeastOneSubTree() {
+        if (childTrees.isEmpty()) {
+            childTrees.add(new Children(getParent()));
+        }
+    }
+
+    /**
+     * Adds a child node to the <strong>first</strong> subtree of this AlternateSpanList. Note
+     * that it might be a good idea to call {@link #sortSubTreesByProbability()} first.
+     *
+     * @param node the node to add.
+     * @return this, for call chaining
+     */
+    @Override
+    public AlternateSpanList add(SpanNode node) {
+        return add(0, node);
+    }
+
+    /**
+     * Sorts the subtrees under this AlternateSpanList by descending probability, such that the most probable
+     * subtree becomes the first subtree, and so on.
+     */
+    public void sortSubTreesByProbability() {
+        resetCachedFromAndTo();
+        Collections.sort(childTrees, childComparator);
+    }
+
+    /**
+     * Returns a modifiable {@link List} of child nodes of <strong>first</strong> subtree.
+     *
+     * @return a modifiable {@link List} of child nodes of <strong>first</strong> subtree
+     */
+    @Override
+    protected List<SpanNode> children() {
+        return children(0);
+    }
+
+    /**
+     * Returns the number of subtrees under this node.
+     *
+     * @return the number of subtrees under this node.
+     */
+    public int getNumSubTrees() {
+        return childTrees.size();
+    }
+
+    /** Clears all subtrees (the subtrees themselves are kept, but their contents are cleared and become invalidated). */
+    @Override
+    public void clearChildren() {
+        for (Children c : childTrees) {
+            c.clearChildren();
+        }
+    }
+
+    /**
+     * Clears a given subtree (the subtree itself is kept, but its contents are cleared and become invalidated).
+     *
+     * @param i the index of the subtree to clear
+     */
+    public void clearChildren(int i) {
+        Children c = childTrees.get(i);
+        if (c != null) {
+            c.clearChildren();
+        }
+    }
+
+    /**
+     * Sorts children in <strong>all</strong> subtrees by occurrence in the text covered.
+     *
+     * @see SpanNode#compareTo(SpanNode)
+     */
+    @Override
+    public void sortChildren() {
+        for (Children children : childTrees) {
+            Collections.sort(children.children());
+        }
+    }
+
+    /**
+     * Sorts children in subtree i by occurrence in the text covered.
+     *
+     * @param i the index of the subtree to sort
+     * @see SpanNode#compareTo(SpanNode)
+     */
+    public void sortChildren(int i) {
+        Children children = childTrees.get(i);
+        Collections.sort(children.children());
+    }
+
+    /**
+     * Recursively sorts all children in <strong>all</strong> subtrees by occurrence in the text covered.
+     */
+    public void sortChildrenRecursive() {
+        for (Children children : childTrees) {
+            for (SpanNode node : children.children()) {
+                if (node instanceof SpanList) {
+                    ((SpanList) node).sortChildrenRecursive();
+                }
+            }
+            Collections.sort(children.children());
+        }
+    }
+
+    /**
+     * Recursively sorts all children in subtree i by occurrence in the text covered.
+     *
+     * @param i the index of the subtree to sort recursively
+     */
+    public void sortChildrenRecursive(int i) {
+        Children children = childTrees.get(i);
+        for (SpanNode node : children.children()) {
+            if (node instanceof SpanList) {
+                ((SpanList) node).sortChildrenRecursive();
+            }
+        }
+        Collections.sort(children.children());
+    }
+
+
+    /**
+     * Moves a child of this SpanList to another SpanList.
+     *
+     * @param i the index of the subtree to remove the node from
+     * @param node the node to move
+     * @param target the SpanList to add the node to
+     * @throws IllegalArgumentException if the given node is not a child of this SpanList
+     */
+    public void move(int i, SpanNode node, SpanList target) {
+        boolean removed = children(i).remove(node);
+        if (removed) {
+            //we found the node
+            node.setParent(null);
+            resetCachedFromAndTo();
+            target.add(node);
+        } else {
+            throw new IllegalArgumentException("Node " + node + " is not a child of this SpanList, cannot move.");
+        }
+    }
+
+    /**
+     * Moves a child of this SpanList to another SpanList.
+     *
+     * @param i the index of the subtree to remove the node from
+     * @param nodeNum the index of the node to move
+     * @param target the SpanList to add the node to
+     * @throws IndexOutOfBoundsException if the given index is out of range
+     */
+    public void move(int i, int nodeNum, SpanList target) {
+        SpanNode node = children(i).remove(nodeNum);
+        if (node != null) {
+            //we found the node
+            node.setParent(null);
+            resetCachedFromAndTo();
+            target.add(node);
+        }
+    }
+
+    /**
+     * Moves a child of this SpanList to another SpanList.
+     *
+     * @param i the index of the subtree to remove the node from
+     * @param node the node to move
+     * @param target the SpanList to add the node to
+     * @param targetSubTree the index of the subtree of the given AlternateSpanList to add the node to
+     * @throws IllegalArgumentException if the given node is not a child of this SpanList
+     * @throws IndexOutOfBoundsException if the given index is out of range, or if the target subtree index is out of range
+     */
+    public void move(int i, SpanNode node, AlternateSpanList target, int targetSubTree) {
+        if (targetSubTree < 0 || targetSubTree >= target.getNumSubTrees()) {
+            throw new IndexOutOfBoundsException(target + " has no subtree at index " + targetSubTree);
+        }
+        boolean removed = children(i).remove(node);
+        if (removed) {
+            //we found the node
+            node.setParent(null);
+            resetCachedFromAndTo();
+            target.add(targetSubTree, node);
+        } else {
+            throw new IllegalArgumentException("Node " + node + " is not a child of this SpanList, cannot move.");
+        }
+    }
+
+    /**
+     * Moves a child of this SpanList to another SpanList.
+     *
+     * @param i the index of the subtree to remove the node from
+     * @param nodeNum the index of the node to move
+     * @param target the SpanList to add the node to
+     * @param targetSubTree the index of the subtree of the given AlternateSpanList to add the node to
+     * @throws IndexOutOfBoundsException if any of the given indeces are out of range, or the target subtree index is out of range
+     */
+    public void move(int i, int nodeNum, AlternateSpanList target, int targetSubTree) {
+        if (targetSubTree < 0 || targetSubTree >= target.getNumSubTrees()) {
+            throw new IndexOutOfBoundsException(target + " has no subtree at index " + targetSubTree);
+        }
+        SpanNode node = children(i).remove(nodeNum);
+        if (node != null) {
+            //we found the node
+            node.setParent(null);
+            resetCachedFromAndTo();
+            target.add(targetSubTree, node);
+        }
+    }
+
+
+    /**
+     * Traverses all immediate children of all subtrees of this AlternateSpanList.
+     * The ListIterator only supports iteration forwards, and the optional operations that are implemented are
+     * remove() and set(). add() is not supported.
+     *
+     * @return a ListIterator which traverses all immediate children of this SpanNode
+     * @see java.util.ListIterator
+     */
+    @Override
+    public ListIterator<SpanNode> childIterator() {
+        List<ListIterator<SpanNode>> childIterators = new ArrayList<ListIterator<SpanNode>>();
+        for (Children ch : childTrees) {
+            childIterators.add(ch.childIterator());
+        }
+        return new SerialIterator(childIterators);
+    }
+
+    /**
+     * Recursively traverses all children (not only leaf nodes) of all subtrees of this AlternateSpanList, in a
+     * depth-first fashion.
+     * The ListIterator only supports iteration forwards, and the optional operations that are implemented are
+     * remove() and set(). add() is not supported.
+     *
+     * @return a ListIterator which recursively traverses all children and their children etc. of all subtrees of this AlternateSpanList
+     * @see java.util.ListIterator
+     */
+    @Override
+    public ListIterator<SpanNode> childIteratorRecursive() {
+        List<ListIterator<SpanNode>> childIterators = new ArrayList<ListIterator<SpanNode>>();
+        for (Children ch : childTrees) {
+            childIterators.add(ch.childIteratorRecursive());
+        }
+        return new SerialIterator(childIterators);
+    }
+
+    /**
+     * Traverses all immediate children of the given subtree of this AlternateSpanList.
+     * The ListIterator returned supports all optional operations
+     * specified in the ListIterator interface.
+     *
+     * @param i the index of the subtree to iterate over
+     * @return a ListIterator which traverses all immediate children of this SpanNode
+     * @see java.util.ListIterator
+     */
+    public ListIterator<SpanNode> childIterator(int i) {
+        return childTrees.get(i).childIterator();
+    }
+
+    /**
+     * Recursively traverses all children (not only leaf nodes) of the given subtree of this AlternateSpanList, in a
+     * depth-first fashion.
+     * The ListIterator only supports iteration forwards, and the optional operations that are implemented are
+     * remove() and set(). add() is not supported.
+     *
+     * @param i the index of the subtree to iterate over
+     * @return a ListIterator which recursively traverses all children and their children etc. of the given subtree of this AlternateSpanList.
+     * @see java.util.ListIterator
+     */
+    public ListIterator<SpanNode> childIteratorRecursive(int i) {
+        return childTrees.get(i).childIteratorRecursive();
+    }
+
+    public int numChildren(int i) {
+        return children(i).size();
+    }
+
+
+    /**
+     * Returns a modifiable {@link List} of child nodes of the specified subtree.
+     *
+     * @param i the index of the subtree to search
+     * @return a modifiable {@link List} of child nodes of the specified subtree
+     */
+    protected List<SpanNode> children(int i) {
+        return childTrees.get(i).children();
+    }
+
+
+    @Override
+    void setParent(SpanNodeParent parent) {
+        super.setParent(parent);
+        for (Children ch : childTrees) {
+            ch.setParent(parent);
+        }
+    }
+
+    /**
+     * Adds a possible subtree of this AlternateSpanList, with the given probability. Note that the first subtree is
+     * always available through the use of children(), so this method is only used for adding the second or higher
+     * subtree.
+     *
+     * @param subtree     the subtree to add
+     * @param probability the probability of this subtree
+     * @return true if successful
+     * @see #children()
+     */
+    public boolean addChildren(List<SpanNode> subtree, double probability) {
+        Children childTree = new Children(getParent(), subtree, probability);
+        resetCachedFromAndTo();
+        return childTrees.add(childTree);
+
+    }
+
+    /**
+     * Adds a possible subtree of this AlternateSpanList, with the given probability, at index i. Note that the first subtree is
+     * always available through the use of children(), so this method is only used for adding the second or higher
+     * subtree.
+     *
+     * @param i           the index of where to insert the subtree
+     * @param subtree     the subtree to add
+     * @param probability the probability of this subtree
+     * @see #children()
+     */
+    public void addChildren(int i, List<SpanNode> subtree, double probability) {
+        Children childTree = new Children(getParent(), subtree, probability);
+        resetCachedFromAndTo();
+        childTrees.add(i, childTree);
+    }
+
+    /**
+     * Removes the subtree at index i (both the subtree itself and its contents, which become invalidated).
+     * Note that if this AlternateSpanList has only one subtree and index 0 is given,
+     * a new empty subtree is automatically added, since an AlternateSpanList always has at least one subtree.
+     *
+     * @param i the index of the subtree to remove
+     * @return the subtree removed, if any (note: invalidated)
+     */
+    public List<SpanNode> removeChildren(int i) {
+        Children retval = childTrees.remove(i);
+        ensureAtLeastOneSubTree();
+        resetCachedFromAndTo();
+        if (retval != null) {
+            retval.setInvalid();
+            retval.setParent(null);
+            for (SpanNode node : retval.children()) {
+                node.setParent(null);
+            }
+            return retval.children();
+        }
+        return null;
+    }
+
+    /**
+     * Removes all subtrees (both the subtrees themselves and their contents, which become invalidated).
+     * Note that a new empty subtree is automatically added at index 0, since an AlternateSpanList always has at
+     * least one subtree.
+     */
+    public void removeChildren() {
+        for (Children ch : childTrees) {
+            ch.setInvalid();
+            ch.setParent(null);
+            ch.clearChildren();
+        }
+        childTrees.clear();
+        resetCachedFromAndTo();
+        ensureAtLeastOneSubTree();
+    }
+
+    @Override
+    void setInvalid() {
+        //invalidate ourselves:
+        super.setInvalid();
+        //invalidate all child trees
+        for (Children ch : childTrees) {
+            ch.setInvalid();
+        }
+    }
+
+
+    /**
+     * Sets the subtree at index i.
+     *
+     * @param i           the index of where to set the subtree
+     * @param subtree     the subtree to set
+     * @param probability the probability to set
+     * @return the overwritten subtree, if any
+     */
+    public List<SpanNode> setChildren(int i, List<SpanNode> subtree, double probability) {
+        resetCachedFromAndTo();
+        if (childTrees.size() == 1 && i == 0) {
+            //replace the first subtree
+            Children sub = new Children(getParent(), subtree, probability);
+            Children retval = childTrees.set(i, sub);
+            if (retval == null) {
+                return null;
+            } else {
+                retval.setParent(null);
+                for (SpanNode node : retval.children()) {
+                    node.setParent(null);
+                }
+                return retval.children();
+            }
+        }
+        List<SpanNode> retval = removeChildren(i);
+        addChildren(i, subtree, probability);
+        return retval;
+    }
+
+    /**
+     * Returns the character index where this {@link SpanNode} starts (inclusive), i.e.&nbsp;the smallest {@link com.yahoo.document.annotation.SpanNode#getFrom()} of all children in subtree i.
+     *
+     * @param i the index of the subtree to use
+     * @return the lowest getFrom() of all children in subtree i, or -1 if this SpanList has no children in subtree i.
+     * @throws IndexOutOfBoundsException if this AlternateSpanList has no subtree i
+     */
+    public int getFrom(int i) {
+        return childTrees.get(i).getFrom();
+    }
+
+
+    /**
+     * Returns the character index where this {@link SpanNode} ends (exclusive), i.e.&nbsp;the greatest {@link com.yahoo.document.annotation.SpanNode#getTo()} of all children in subtree i.
+     *
+     * @param i the index of the subtree to use
+     * @return the greatest getTo() of all children, or -1 if this SpanList has no children in subtree i.
+     * @throws IndexOutOfBoundsException if this AlternateSpanList has no subtree i
+     */
+    public int getTo(int i) {
+        return childTrees.get(i).getTo();
+    }
+
+    /**
+     * Returns the length of this span according to subtree i, i.e.&nbsp;getFrom(i) - getTo(i).
+     *
+     * @param i the index of the subtree to use
+     * @return the length of this span according to subtree i
+     */
+    public int getLength(int i) {
+        return getTo(i) - getFrom(i);
+    }
+
+    /**
+     * Returns the text covered by this span as given by subtree i, or null if subtree i is empty.
+     *
+     * @param i    the index of the subtree to use
+     * @param text the text to get a substring from
+     * @return the text covered by this span as given by subtree i, or null if subtree i is empty
+     */
+    public CharSequence getText(int i, CharSequence text) {
+        if (children(i).isEmpty()) {
+            return null;
+        }
+        StringBuilder str = new StringBuilder();
+        List<SpanNode> ch = children(i);
+        for (SpanNode node : ch) {
+            CharSequence childText = node.getText(text);
+            if (childText != null) {
+                str.append(node.getText(text));
+            }
+        }
+        return str;
+    }
+
+    /**
+     * Returns the probability of the given subtree.
+     *
+     * @param i the subtree to return the probability of
+     * @return the probability of the given subtree
+     */
+    public double getProbability(int i) {
+        return childTrees.get(i).getProbability();
+    }
+
+    /**
+     * Sets the probability of the given subtree.
+     *
+     * @param i           the subtree to set the probability of
+     * @param probability the probability to set
+     */
+    public void setProbability(int i, double probability) {
+        childTrees.get(i).setProbability(probability);
+    }
+
+    /** Normalizes all probabilities between 0.0 (inclusive) and 1.0 (exclusive). */
+    public void normalizeProbabilities() {
+        double sum = 0.0;
+        for (Children c : childTrees) {
+            sum += c.getProbability();
+        }
+        double coeff = 1.0 / sum;
+
+        for (Children childTree : childTrees) {
+            double newProb = childTree.getProbability() * coeff;
+            childTree.setProbability(newProb);
+        }
+    }
+
+
+    /**
+     * Convenience method to add a span node to the child tree at index i. This is equivalent to calling
+     * <code>
+     * AlternateSpanList.children(i).add(node);
+     * </code>
+     *
+     * @param i    index
+     * @param node span node
+     */
+    public AlternateSpanList add(int i, SpanNode node) {
+        checkValidity(node, children(i));
+        node.setParent(this);
+        children(i).add(node);
+        return this;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (!(o instanceof AlternateSpanList)) return false;
+        if (!super.equals(o)) return false;
+
+        AlternateSpanList that = (AlternateSpanList) o;
+
+        if (!childTrees.equals(that.childTrees)) return false;
+
+        return true;
+    }
+
+    @Override
+    public int hashCode() {
+        int result = super.hashCode();
+        result = 31 * result + childTrees.hashCode();
+        return result;
+    }
+
+    @Override
+    public String toString() {
+        return "AlternateSpanList, num subtrees=" + getNumSubTrees();
+    }
+
+
+    private static class ProbabilityComparator implements Comparator<Children> {
+        @Override
+        public int compare(Children o1, Children o2) {
+            return Double.compare(o2.probability, o1.probability);  //note: opposite of natural ordering!
+        }
+    }
+
+    private class Children extends SpanList {
+        private double probability = 1.0;
+
+        private Children(SpanNodeParent parent) {
+            setParent(parent);
+        }
+
+        private Children(SpanNodeParent parent, List<SpanNode> children, double probability) {
+            super(children);
+            setParent(parent);
+            if (children != null) {
+                for (SpanNode node : children) {
+                    node.setParent(AlternateSpanList.this);
+                }
+            }
+            this.probability = probability;
+        }
+
+        public double getProbability() {
+            return probability;
+        }
+
+        public void setProbability(double probability) {
+            this.probability = probability;
+        }
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/annotation/Annotation.java b/document/src/main/java/com/yahoo/document/annotation/Annotation.java
new file mode 100644
index 00000000000..1823584eb6d
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/annotation/Annotation.java
@@ -0,0 +1,260 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.datatypes.FieldValue;
+
+/**
+ * An Annotation describes some kind of information associated with
+ * a {@link SpanNode}.
+ *
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ * @see com.yahoo.document.annotation.SpanNode
+ * @see com.yahoo.document.annotation.AnnotationType
+ */
+public class Annotation implements Comparable<Annotation> {
+    private AnnotationType type;
+    private SpanNode spanNode = null;
+    private FieldValue value = null;
+    /**
+     * This scratch id is used to avoid using IdentityHashMaps as they are very costly.
+     */
+    private int scratchId = -1;
+    public void setScratchId(int id) {
+        scratchId = id;
+    }
+
+    public int getScratchId() {
+        return scratchId;
+    }
+
+
+    /**
+     * Constructs an Annotation without a type.&nbsp;BEWARE! Should only be used during deserialization.
+     */
+    public Annotation() {
+    }
+
+    /**
+     * Constructs a new annotation of the specified type.
+     *
+     * @param type the type of the new annotation
+     */
+    public Annotation(AnnotationType type) {
+        this.type = type;
+    }
+
+    /**
+     * Constructs a copy of the input annotation.
+     *
+     * @param other annotation
+     */
+    public Annotation(Annotation other) {
+        this.type = other.type;
+        this.value = ((other.value == null) ? null : other.value.clone());
+        //do not copy spanNode now
+    }
+
+    /**
+     * Constructs a new annotation of the specified type, and having the specified value.
+     *
+     * @param type  the type of the new annotation
+     * @param value the value of the new annotation
+     * @throws UnsupportedOperationException if the annotation type does not allow this annotation to have values.
+     */
+    public Annotation(AnnotationType type, FieldValue value) {
+        this(type);
+        setFieldValue(value);
+    }
+
+    /**
+     * Returns the type of this annotation.
+     *
+     * @return the type of this annotation
+     */
+    public AnnotationType getType() {
+        return type;
+    }
+
+    /**
+     * Sets the type of this annotation.&nbsp;BEWARE! Should only be used during deserialization.
+     *
+     * @param type the type of this annotation
+     */
+    public void setType(AnnotationType type) {
+        this.type = type;
+    }
+
+    /**
+     * Returns true if this Annotation is associated with a SpanNode (irrespective of the SpanNode being valid or not).
+     *
+     * @return true if this Annotation is associated with a SpanNode, false otherwise.
+     * @see com.yahoo.document.annotation.SpanNode#isValid()
+     */
+    public boolean hasSpanNode() {
+        return spanNode != null;
+    }
+
+    /**
+     * Returns true iff.&nbsp;this Annotation is associated with a SpanNode and the SpanNode is valid.
+     *
+     * @return true iff.&nbsp;this Annotation is associated with a SpanNode and the SpanNode is valid.
+     * @see com.yahoo.document.annotation.SpanNode#isValid()
+     */
+    public boolean isSpanNodeValid() {
+        return spanNode != null && spanNode.isValid();
+    }
+
+
+    /**
+     * Returns the {@link SpanNode} this Annotation is annotating, if any.
+     *
+     * @return the {@link SpanNode} this Annotation is annotating, or null
+     * @throws IllegalStateException if this Annotation is associated with a SpanNode and the SpanNode is invalid.
+     */
+    public SpanNode getSpanNode() {
+        if (spanNode != null && !spanNode.isValid()) {
+            throw new IllegalStateException("Span node is invalid: " + spanNode);
+        }
+        return spanNode;
+    }
+
+    /**
+     * Returns the {@link SpanNode} this Annotation is annotating, if any.
+     *
+     * @return the {@link SpanNode} this Annotation is annotating, or null
+     */
+    public final SpanNode getSpanNodeFast() {
+        return spanNode;
+    }
+
+    /**
+     * WARNING!&nbsp;Should only be used by deserializers!&nbsp;Sets the span node that this annotation points to.
+     *
+     * @param spanNode the span node that this annotation shall point to.
+     */
+    public void setSpanNode(SpanNode spanNode) {
+        if (this.spanNode != null && spanNode != null) {
+            throw new IllegalStateException("WARNING! " + this + " is already attached to node " + this.spanNode
+                                            + ". Attempt to attach to node " + spanNode
+                                            + ". Annotation instances MUST NOT be shared among SpanNodes.");
+        }
+        if (spanNode != null && !spanNode.isValid()) {
+            throw new IllegalStateException("Span node is invalid: " + spanNode);
+        }
+        if (spanNode == DummySpanNode.INSTANCE) {
+            //internal safeguard
+            throw new IllegalStateException("BUG!! Annotations should never be attached to DummySpanNode.");
+        }
+        this.spanNode = spanNode;
+    }
+
+    /**
+     * WARNING!&nbsp;Should only be used by deserializers!&nbsp;Sets the span node that this annotation points to.
+     *
+     * @param spanNode the span node that this annotation shall point to.
+     */
+    public final void setSpanNodeFast(SpanNode spanNode) {
+        this.spanNode = spanNode;
+    }
+
+    /**
+     * Returns the value of the annotation, if any.
+     *
+     * @return the value of the annotation, or null
+     */
+    public FieldValue getFieldValue() {
+        return value;
+    }
+
+    /**
+     * Sets the value of the annotation.
+     *
+     * @param fieldValue the value to set
+     * @throws UnsupportedOperationException if the annotation type does not allow this annotation to have values.
+     */
+    public void setFieldValue(FieldValue fieldValue) {
+        if (fieldValue == null) {
+            value = null;
+            return;
+        }
+
+        DataType type = getType().getDataType();
+        if (type != null && type.isValueCompatible(fieldValue)) {
+            this.value = fieldValue;
+        } else {
+            String typeName = (type == null) ? "null" : type.getValueClass().getName();
+            throw new IllegalArgumentException("Argument is of wrong type, must be of type " + typeName
+                                               + ", was " + fieldValue.getClass().getName());
+        }
+    }
+
+    /**
+     * Returns true if this Annotation has a value.
+     *
+     * @return true if this Annotation has a value, false otherwise.
+     */
+    public boolean hasFieldValue() {
+        return value != null;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (!(o instanceof Annotation)) return false;
+
+        Annotation that = (Annotation) o;
+        if (!type.equals(that.type)) return false;
+        if (spanNode != null ? !spanNode.equals(that.spanNode) : that.spanNode != null) return false;
+        if (value != null ? !value.equals(that.value) : that.value != null) return false;
+
+        return true;
+    }
+
+    @Override
+    public int hashCode() {
+        int result = type.hashCode();
+        result = 31 * result + (spanNode != null ? spanNode.hashCode() : 0);
+        result = 31 * result + (value != null ? value.toString().hashCode() : 0);
+        return result;
+    }
+
+    @Override
+    public String toString() {
+        String retval = "annotation of type " + type;
+        retval += ((value == null) ? " (no value)" : " (with value)");
+        return retval;
+    }
+
+
+    @Override
+    public int compareTo(Annotation annotation) {
+        int comp;
+
+        if (spanNode == null) {
+            comp = (annotation.spanNode == null) ? 0 : -1;
+        } else {
+            comp = (annotation.spanNode == null) ? 1 : spanNode.compareTo(annotation.spanNode);
+        }
+
+        if (comp != 0) {
+            return comp;
+        }
+
+        comp = type.compareTo(annotation.type);
+
+        if (comp != 0) {
+            return comp;
+        }
+
+        //types are equal, too, compare values
+        if (value == null) {
+            comp = (annotation.value == null) ? 0 : -1;
+        } else {
+            comp = (annotation.value == null) ? 1 : value.compareTo(annotation.value);
+        }
+
+        return comp;
+    }
+}
+
diff --git a/document/src/main/java/com/yahoo/document/annotation/AnnotationContainer.java b/document/src/main/java/com/yahoo/document/annotation/AnnotationContainer.java
new file mode 100644
index 00000000000..3a44d2a78f6
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/annotation/AnnotationContainer.java
@@ -0,0 +1,51 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+import java.util.Collection;
+import java.util.Iterator;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+abstract class AnnotationContainer {
+
+    /**
+     * Adds all annotations of the given collection to this container.
+     *
+     * @param annotations the annotations to add.
+     */
+    abstract void annotateAll(Collection<Annotation> annotations);
+
+    /**
+     * Adds an annotation to this container.
+     *
+     * @param annotation the annotation to add.
+     */
+    abstract void annotate(Annotation annotation);
+
+    /**
+     * Returns a mutable collection of annotations.
+     *
+     * @return a mutable collection of annotations.
+     */
+    abstract Collection<Annotation> annotations();
+
+    /**
+     * Returns an Iterator over all annotations that annotate the given node.
+     *
+     * @param node the node to return annotations for.
+     * @return an Iterator over all annotations that annotate the given node.
+     */
+    abstract Iterator<Annotation> iterator(SpanNode node);
+
+    /**
+     * Returns a recursive Iterator over all annotations that annotate the given node and its subnodes.
+     *
+     * @param node the node to recursively return annotations for.
+     * @return a recursive Iterator over all annotations that annotate the given node and its subnodes.
+     */
+    abstract Iterator<Annotation> iteratorRecursive(SpanNode node);
+
+
+    //TODO: remember equals and hashcode in subclasses!
+}
diff --git a/document/src/main/java/com/yahoo/document/annotation/AnnotationReference.java b/document/src/main/java/com/yahoo/document/annotation/AnnotationReference.java
new file mode 100644
index 00000000000..aa6ea1b0040
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/annotation/AnnotationReference.java
@@ -0,0 +1,185 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.Field;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.serialization.FieldReader;
+import com.yahoo.document.serialization.FieldWriter;
+import com.yahoo.document.serialization.XmlStream;
+import com.yahoo.vespa.objects.Ids;
+
+/**
+ * A FieldValue which holds a reference to an annotation of a specified type.
+ *
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ * @see Annotation#setFieldValue(com.yahoo.document.datatypes.FieldValue)
+ */
+public class AnnotationReference extends FieldValue {
+
+    public static int classId = registerClass(Ids.annotation + 2, AnnotationReference.class);
+    private Annotation reference;
+    private AnnotationReferenceDataType dataType;
+
+    /**
+     * Constructs a new AnnotationReference, with a reference to the given {@link Annotation}.
+     *
+     * @param type      the data type of this AnnotationReference
+     * @param reference the reference to set
+     * @throws IllegalArgumentException if the given annotation has a type that is not compatible with this reference
+     */
+    public AnnotationReference(AnnotationReferenceDataType type, Annotation reference) {
+        this.dataType = type;
+        setReference(reference);
+    }
+
+    /**
+     * Constructs a new AnnotationReference.
+     *
+     * @param type the data type of this AnnotationReference
+     */
+    public AnnotationReference(AnnotationReferenceDataType type) {
+        this(type, null);
+    }
+
+    /**
+     * Clones this AnnotationReference.&nbsp;Note: No deep-copying, so the AnnotationReference returned
+     * refers to the same Annotation as this AnnotationReference.
+     *
+     * @return a copy of this object, referring to the same Annotation instance.
+     */
+    @Override
+    public AnnotationReference clone() {
+        return (AnnotationReference) super.clone();
+        //do not clone annotation that we're referring to. See wizardry in SpanTree for that.
+    }
+
+    /**
+     * Returns the Annotation that this AnnotationReference refers to.
+     *
+     * @return the Annotation that this AnnotationReference refers to.
+     */
+    public Annotation getReference() {
+        return reference;
+    }
+
+    @Override
+    public void assign(Object o) {
+        if (o != null && (!(o instanceof Annotation))) {
+            throw new IllegalArgumentException("Cannot assign object of type " + o.getClass().getName() + " to an AnnotationReference, must be of type " + Annotation.class.getName());
+        }
+        setReference((Annotation) o);
+    }
+
+    /**
+     * Set an {@link Annotation} that this AnnotationReference shall refer to.
+     *
+     * @param reference an Annotation that this AnnotationReference shall refer to.
+     * @throws IllegalArgumentException if the given annotation has a type that is not compatible with this reference
+     */
+    public void setReference(Annotation reference) {
+        if (reference == null) {
+            this.reference = null;
+            return;
+        }
+        AnnotationReferenceDataType type = getDataType();
+        if (type.getAnnotationType().isValueCompatible(reference)
+                // The case if concrete annotation type
+                || reference.getType() instanceof AnnotationType) {
+            this.reference = reference;
+        } else {
+            throw new IllegalArgumentException("Cannot set reference, must be of type " + type + " (was of type " + reference.getType() + ")");
+        }
+    }
+
+
+    /**
+     * WARNING!&nbsp;Only to be used by deserializers when reference is not fully deserialized yet!&nbsp;Sets
+     * an {@link Annotation} that this AnnotationReference shall refer to.
+     *
+     * @param reference an Annotation that this AnnotationReference shall refer to.
+     * @throws IllegalArgumentException if the given annotation has a type that is not compatible with this reference
+     */
+    public void setReferenceNoCompatibilityCheck(Annotation reference) {
+        if (reference == null) {
+            this.reference = null;
+            return;
+        }
+        this.reference = reference;
+    }
+
+    @Override
+    public AnnotationReferenceDataType getDataType() {
+        return dataType;
+    }
+
+    public void setDataType(DataType dataType) {
+        if (dataType instanceof AnnotationReferenceDataType) {
+            this.dataType = (AnnotationReferenceDataType) dataType;
+        } else {
+            throw new IllegalArgumentException("Cannot set dataType to " + dataType + ", must be of type AnnotationReferenceDataType.");
+        }
+    }
+
+    @Override
+    public void printXml(XmlStream xml) {
+        //TODO: Implement AnnotationReference.printXml()
+    }
+
+    @Override
+    public void clear() {
+        this.reference = null;
+    }
+
+    @Override
+    public void serialize(Field field, FieldWriter writer) {
+        writer.write(field, this);
+    }
+
+    @Override
+    public void deserialize(Field field, FieldReader reader) {
+        reader.read(field, this);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (!(o instanceof AnnotationReference)) return false;
+        if (!super.equals(o)) return false;
+
+        AnnotationReference that = (AnnotationReference) o;
+
+        if (reference != null ? !reference.toString().equals(that.reference.toString()) : that.reference != null) {
+            return false;
+        }
+
+        return true;
+    }
+
+    @Override
+    public int hashCode() {
+        int result = super.hashCode();
+        result = 31 * result + (reference != null ? reference.toString().hashCode() : 0);
+        return result;
+    }
+
+    @Override
+    public String toString() {
+        return "AnnotationReference " + getDataType() + " referring to " + reference;
+    }
+
+    @Override
+    public int compareTo(FieldValue fieldValue) {
+        int comp = super.compareTo(fieldValue);
+        if (comp == 0) {
+            //types are equal, this must be of this type
+            AnnotationReference value = (AnnotationReference) fieldValue;
+            if (reference == null) {
+                comp = (value.reference == null) ? 0 : -1;
+            } else {
+                comp = (value.reference == null) ? 1 : (reference.toString().compareTo(value.reference.toString()));
+            }
+        }
+        return comp;
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/annotation/AnnotationReferenceDataType.java b/document/src/main/java/com/yahoo/document/annotation/AnnotationReferenceDataType.java
new file mode 100644
index 00000000000..e75e29f5e75
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/annotation/AnnotationReferenceDataType.java
@@ -0,0 +1,87 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.datatypes.FieldValue;
+
+/**
+ * A data type describing a field value having a reference to an annotation of a given type.
+ *
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class AnnotationReferenceDataType extends DataType {
+    private AnnotationType aType;
+
+    /**
+     * Creates an AnnotationReferenceDataType with a generated id.
+     *
+     * @param aType the annotation type that AnnotationRefs shall refer to.
+     */
+    public AnnotationReferenceDataType(AnnotationType aType) {
+        super("annotationreference<" + ((aType == null) ? "" : aType.getName()) + ">", 0);
+        setAnnotationType(aType);
+    }
+
+    /**
+     * Creates an AnnotationReferenceDataType with a given id.
+     *
+     * @param aType the annotation type that AnnotationRefs shall refer to.
+     * @param id    the id to use
+     */
+    public AnnotationReferenceDataType(AnnotationType aType, int id) {
+        super("annotationreference<" + ((aType == null) ? "" : aType.getName()) + ">", id);
+        this.aType = aType;
+    }
+
+    /**
+     * Creates an AnnotationReferenceDataType.&nbsp;WARNING! Do not use!
+     */
+    protected AnnotationReferenceDataType() {
+        super("annotationreference<>", 0);
+    }
+
+    private int createId() {
+        //TODO: This should be Java's hashCode(), since all other data types use it, and using something else here will probably lead to collisions
+        return getName().toLowerCase().hashCode();
+    }
+
+    @Override
+    public FieldValue createFieldValue() {
+        return new AnnotationReference(this);
+    }
+
+    @Override
+    public Class getValueClass() {
+        return AnnotationReference.class;
+    }
+
+    @Override
+    public boolean isValueCompatible(FieldValue value) {
+        if (!(value instanceof AnnotationReference)) {
+            return false;
+        }
+        AnnotationReference reference = (AnnotationReference) value;
+        if (equals(reference.getDataType())) {
+            return true;
+        }
+        return false;
+    }
+
+    /**
+     * Returns the annotation type of this AnnotationReferenceDataType.
+     *
+     * @return the annotation type of this AnnotationReferenceDataType.
+     */
+    public AnnotationType getAnnotationType() {
+        return aType;
+    }
+
+    /**
+     * Sets the annotation type that this AnnotationReferenceDataType points to.&nbsp;WARNING! Do not use.
+     * @param type the annotation type of this AnnotationReferenceDataType.
+     */
+    protected void setAnnotationType(AnnotationType type) {
+        this.aType = type;
+        setId(createId());
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/annotation/AnnotationType.java b/document/src/main/java/com/yahoo/document/annotation/AnnotationType.java
new file mode 100644
index 00000000000..f66080e3a5a
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/annotation/AnnotationType.java
@@ -0,0 +1,186 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+import com.google.common.collect.ImmutableList;
+import com.yahoo.collections.MD5;
+import com.yahoo.document.DataType;
+
+import java.util.Collection;
+import java.util.Collections;
+
+/**
+ * An AnnotationType describes a certain type of annotations; they are
+ * generally distinguished by a name, an id, and an optional data type.
+ * <p>
+ * If an AnnotationType has a {@link com.yahoo.document.DataType}, this means that {@link Annotation}s of
+ * that type are allowed to have a {@link com.yahoo.document.datatypes.FieldValue} of the given
+ * {@link com.yahoo.document.DataType} as an optional payload.
+ *
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class AnnotationType implements Comparable<AnnotationType> {
+    private final int id;
+    private final String name;
+    private DataType dataType;
+    private AnnotationType superType = null;
+
+    /**
+     * Creates a new annotation type that cannot have values (hence no data type).
+     *
+     * @param name the name of the new annotation type
+     */
+    public AnnotationType(String name) {
+        this(name, null);
+    }
+
+    /**
+     * Creates a new annotation type that can have values of the specified type.
+     *
+     * @param name     the name of the new annotation type
+     * @param dataType the data type of the annotation value
+     */
+    public AnnotationType(String name, DataType dataType) {
+        this.name = name;
+        this.dataType = dataType;
+        //always keep this as last statement in here:
+        this.id = computeHash();
+    }
+
+    /**
+     * Creates a new annotation type that can have values of the specified type.
+     *
+     * @param name     the name of the new annotation type
+     * @param dataType the data type of the annotation value
+     * @param id   the ID of the new annotation type
+     */
+    public AnnotationType(String name, DataType dataType, int id) {
+        this.name = name;
+        this.dataType = dataType;
+        this.id = id;
+    }
+
+    /**
+     * Creates a new annotation type, with the specified ID.&nbsp;WARNING! Only to be used by configuration
+     * system, do not use!!
+     *
+     * @param name the name of the new annotation type
+     * @param id   the ID of the new annotation type
+     */
+    public AnnotationType(String name, int id) {
+        this.id = id;
+        this.name = name;
+    }
+
+    /**
+     * Returns the name of this annotation.
+     *
+     * @return the name of this annotation.
+     */
+    public String getName() {
+        return name;
+    }
+
+    /**
+     * Returns the data type of this annotation, if any.
+     *
+     * @return the data type of this annotation, or null.
+     */
+    public DataType getDataType() {
+        return dataType;
+    }
+
+    /**
+     * Sets the data type of this annotation.&nbsp;WARNING! Only to be used by configuration
+     * system, do not use!!
+     *
+     * @param dataType the data type of the annotation value
+     */
+    public void setDataType(DataType dataType) {
+        this.dataType = dataType;
+    }
+
+    /**
+     * Returns the ID of this annotation.
+     *
+     * @return the ID of this annotation.
+     */
+    public int getId() {
+        return id;
+    }
+
+    private int computeHash() {
+        return new MD5().hash(name);
+    }
+
+    public boolean isValueCompatible(Annotation structValue) {
+        if (structValue.getType().inherits(this)) {
+            //the value is of this type; or the supertype of the value is of this type, etc....
+            return true;
+        }
+        return false;
+    }
+
+    /**
+     * WARNING!&nbsp;Only to be used by the configuration system and in unit tests.&nbsp;Not to be used in production code.
+     *
+     * @param type the type to inherit from
+     */
+    public void inherit(AnnotationType type) {
+        if (superType != null) {
+            throw new IllegalArgumentException("Already inherits type " + superType + ", multiple inheritance not currently supported.");
+        }
+        superType = type;
+    }
+
+    public Collection<AnnotationType> getInheritedTypes() {
+        if (superType == null) {
+            return ImmutableList.of();
+        }
+        return ImmutableList.of(superType);
+    }
+
+    public boolean inherits(AnnotationType type) {
+        if (equals(type)) return true;
+        if (superType != null && superType.inherits(type)) return true;
+        return false;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (!(o instanceof AnnotationType)) return false;
+
+        AnnotationType that = (AnnotationType) o;
+
+        return name.equals(that.name);
+    }
+
+    @Override
+    public int hashCode() {
+        return id;
+    }
+
+    @Override
+    public String toString() {
+        StringBuilder strb = new StringBuilder();
+        strb.append(name).append(" (id ").append(id);
+        if (dataType != null) {
+            strb.append(", data type ").append(dataType);
+        }
+        strb.append(")");
+        return strb.toString();
+    }
+
+    @Override
+    public int compareTo(AnnotationType annotationType) {
+        if (annotationType == null) {
+            return 1;
+        }
+        if (id < annotationType.id) {
+            return -1;
+        } else if (id > annotationType.id) {
+            return 1;
+        }
+        return 0;
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/annotation/AnnotationType2AnnotationContainer.java b/document/src/main/java/com/yahoo/document/annotation/AnnotationType2AnnotationContainer.java
new file mode 100644
index 00000000000..778e6f50a40
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/annotation/AnnotationType2AnnotationContainer.java
@@ -0,0 +1,107 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+import org.apache.commons.collections.map.MultiValueMap;
+
+import java.util.Collection;
+import java.util.IdentityHashMap;
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+
+/**
+ *
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+// TODO: Should this be removed?
+public class AnnotationType2AnnotationContainer extends IteratingAnnotationContainer {
+    private final MultiValueMap annotationType2Annotation = MultiValueMap.decorate(new IdentityHashMap());
+
+    @Override
+    void annotateAll(Collection<Annotation> annotations) {
+        for (Annotation a : annotations) {
+            annotate(a);
+        }
+    }
+
+    @Override
+    void annotate(Annotation annotation) {
+        annotationType2Annotation.put(annotation.getType(), annotation);
+    }
+
+    @Override
+    @SuppressWarnings("unchecked")
+    Collection<Annotation> annotations() {
+        return annotationType2Annotation.values();
+    }
+
+    @Override
+    Iterator<Annotation> iterator(IdentityHashMap<SpanNode, SpanNode> nodes) {
+        return new NonRecursiveIterator(nodes);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (!(o instanceof AnnotationType2AnnotationContainer)) return false;
+        AnnotationType2AnnotationContainer that = (AnnotationType2AnnotationContainer) o;
+        if (!annotationType2Annotation.equals(that.annotationType2Annotation)) return false;
+        return true;
+    }
+
+    @Override
+    public int hashCode() {
+        return annotationType2Annotation.hashCode();
+    }
+
+    private class NonRecursiveIterator implements Iterator<Annotation> {
+        private final IdentityHashMap<SpanNode, SpanNode> nodes;
+        private final Iterator<Annotation> annotationIt;
+        private Annotation next = null;
+        private boolean nextCalled;
+
+        @SuppressWarnings("unchecked")
+        public NonRecursiveIterator(IdentityHashMap<SpanNode, SpanNode> nodes) {
+            this.nodes = nodes;
+            this.annotationIt = annotationType2Annotation.values().iterator();
+        }
+
+        @Override
+        public boolean hasNext() {
+            if (next != null) {
+                return true;
+            }
+            while (annotationIt.hasNext()) {
+                Annotation tmp = annotationIt.next();
+                if (nodes.containsKey(tmp.getSpanNodeFast())) {
+                    next = tmp;
+                    return true;
+                }
+            }
+            next = null;
+            return false;
+        }
+
+        @Override
+        public Annotation next() {
+            if (hasNext()) {
+                Annotation tmp = next;
+                next = null;
+                nextCalled = true;
+                return tmp;
+            }
+            //there is no 'next'
+            throw new NoSuchElementException("No next element found.");
+        }
+
+        @Override
+        public void remove() {
+            //only allowed to call remove immediately after next()
+            if (!nextCalled) {
+                //we have not next'ed the iterator, cannot do this:
+                throw new IllegalStateException("Cannot remove() before next()");
+            }
+            annotationIt.remove();
+            nextCalled = false;
+        }
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/annotation/AnnotationTypeRegistry.java b/document/src/main/java/com/yahoo/document/annotation/AnnotationTypeRegistry.java
new file mode 100644
index 00000000000..21e0338ae0e
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/annotation/AnnotationTypeRegistry.java
@@ -0,0 +1,130 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * A registry of annotation types.&nbsp;This can be set up programmatically or from config.
+ *
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class AnnotationTypeRegistry {
+    private Map<Integer, AnnotationType> idMap = new HashMap<Integer, AnnotationType>();
+    private Map<String, AnnotationType> nameMap = new HashMap<String, AnnotationType>();
+
+    /** Creates a new empty registry. */
+    public AnnotationTypeRegistry() {
+    }
+
+    /**
+     * Register a new annotation type.&nbsp;WARNING!&nbsp;Only to be used by the configuration system and in unit tests.&nbsp;Not to be used in production code.
+     *
+     * @param type the type to register
+     * @throws IllegalArgumentException if a type is already registered with this name or this id, and it is non-equal to the argument.
+     */
+    public void register(AnnotationType type) {
+        if (idMap.containsKey(type.getId()) || nameMap.containsKey(type.getName())) {
+            AnnotationType idType = idMap.get(type.getId());
+            AnnotationType nameType = nameMap.get(type.getName());
+            if (type.equals(idType) && type.equals(nameType)) {
+                //it's the same one being re-registered, we're OK!
+                return;
+            }
+            throw new IllegalArgumentException("A type is already registered with this name or this id.");
+        }
+        idMap.put(type.getId(), type);
+        nameMap.put(type.getName(), type);
+    }
+
+    /**
+     * Unregisters the type given by the argument.&nbsp;WARNING!&nbsp;Only to be used by the configuration system and in unit tests.&nbsp;Not to be used in production code.
+     *
+     * @param name the name of the type to unregister.
+     * @return true if the type was successfully unregistered, false otherwise (it was not present)
+     */
+    public boolean unregister(String name) {
+        AnnotationType oldType = nameMap.remove(name);
+        if (oldType != null) {
+            idMap.remove(oldType.getId());
+            return true;
+        }
+        return false;
+    }
+
+    /**
+     * Unregisters the type given by the argument.&nbsp;WARNING!&nbsp;Only to be used by the configuration system and in unit tests.&nbsp;Not to be used in production code.
+     *
+     * @param id the id of the type to unregister.
+     * @return true if the type was successfully unregistered, false otherwise (it was not present)
+     */
+    public boolean unregister(int id) {
+        AnnotationType oldType = idMap.remove(id);
+        if (oldType != null) {
+            nameMap.remove(oldType.getName());
+            return true;
+        }
+        return false;
+    }
+
+    /**
+     * Unregisters the type given by the argument.&nbsp;WARNING!&nbsp;Only to be used by the configuration system and in unit tests.&nbsp;Not to be used in production code.
+     *
+     * @param type the AnnotationType to unregister.
+     * @return true if the type was successfully unregistered, false otherwise (it was not present)
+     * @throws IllegalArgumentException if the ID and name of this annotation type are present, but they do not belong together.
+     */
+    public boolean unregister(AnnotationType type) {
+        if (idMap.containsKey(type.getId()) && nameMap.containsKey(type.getName())) {
+            AnnotationType idType = idMap.get(type.getId());
+            AnnotationType nameType = nameMap.get(type.getName());
+
+            if (idType == nameType) {
+                //name and id belong together in our maps
+                idMap.remove(type.getId());
+                nameMap.remove(type.getName());
+            } else {
+                throw new IllegalArgumentException("The ID and name of this annotation type are present, but they do not belong together. Not removing type.");
+            }
+            return true;
+        }
+        //it's not there, but that's no problem
+        return false;
+    }
+
+    /**
+     * Returns an annotation type with the given name.
+     *
+     * @param name the name of the annotation type to return
+     * @return an {@link AnnotationType} with the given name, or null if it is not registered
+     */
+    public AnnotationType getType(String name) {
+        return nameMap.get(name);
+    }
+
+    /**
+     * Returns an annotation type with the given id.
+     *
+     * @param id the id of the annotation type to return
+     * @return an {@link AnnotationType} with the given id, or null if it is not registered
+     */
+    public AnnotationType getType(int id) {
+        return idMap.get(id);
+    }
+
+    /**
+     * Returns an <strong>unmodifiable</strong> {@link Map} of all types registered.
+     *
+     * @return an unmodifiable {@link Map} of all types registered.
+     */
+    public Map<String, AnnotationType> getTypes() {
+        return Collections.unmodifiableMap(nameMap);
+    }
+
+    /** Clears all registered annotation types. */
+    public void clear() {
+        idMap.clear();
+        nameMap.clear();
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/annotation/AnnotationTypes.java b/document/src/main/java/com/yahoo/document/annotation/AnnotationTypes.java
new file mode 100644
index 00000000000..6d868ed4079
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/annotation/AnnotationTypes.java
@@ -0,0 +1,35 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+import com.yahoo.document.DataType;
+
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * This is a container for all {@link Annotation}s constants used by built-in Vespa features. These must be in sync with
+ * the corresponding class in the C++ file 'document/datatype/annotationtype.h'.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+@SuppressWarnings({ "UnusedDeclaration" })
+public final class AnnotationTypes {
+
+    private AnnotationTypes() {
+        // unreachable
+    }
+
+    public static final AnnotationType TERM = new AnnotationType("term", DataType.STRING, 1);
+    public static final AnnotationType TOKEN_TYPE = new AnnotationType("token_type", DataType.INT, 2);
+    public static final AnnotationType CANONICAL = new AnnotationType("canonical", DataType.STRING, 3);
+    public static final AnnotationType NORMALIZED = new AnnotationType("normalized", DataType.STRING, 4);
+    public static final AnnotationType READING = new AnnotationType("reading", DataType.STRING, 5);
+    public static final AnnotationType STEM = new AnnotationType("stem", DataType.STRING, 6);
+    public static final AnnotationType TRANSFORMED = new AnnotationType("transformed", DataType.STRING, 7);
+    public static final AnnotationType PROXIMITY_BREAK = new AnnotationType("proximity_break", DataType.DOUBLE, 8);
+    public static final AnnotationType SPECIAL_TOKEN = new AnnotationType("special_token", 9);
+
+    public static final List<AnnotationType> ALL_TYPES = Arrays.asList(TERM, TOKEN_TYPE, CANONICAL, NORMALIZED, READING,
+                                                                       STEM, TRANSFORMED, PROXIMITY_BREAK,
+                                                                       SPECIAL_TOKEN);
+}
diff --git a/document/src/main/java/com/yahoo/document/annotation/DummySpanNode.java b/document/src/main/java/com/yahoo/document/annotation/DummySpanNode.java
new file mode 100644
index 00000000000..c891e8c686b
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/annotation/DummySpanNode.java
@@ -0,0 +1,50 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+import java.util.Collections;
+import java.util.ListIterator;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+class DummySpanNode extends SpanNode {
+    final static DummySpanNode INSTANCE = new DummySpanNode();
+
+    private DummySpanNode() {
+    }
+
+    @Override
+    public boolean isLeafNode() {
+        return true;
+    }
+
+    @Override
+    public ListIterator<SpanNode> childIterator() {
+        return Collections.<SpanNode>emptyList().listIterator();
+    }
+
+    @Override
+    public ListIterator<SpanNode> childIteratorRecursive() {
+        return Collections.<SpanNode>emptyList().listIterator();
+    }
+
+    @Override
+    public int getFrom() {
+        return 0;
+    }
+
+    @Override
+    public int getTo() {
+        return 0;
+    }
+
+    @Override
+    public int getLength() {
+        return 0;
+    }
+
+    @Override
+    public CharSequence getText(CharSequence text) {
+        return null;
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/annotation/InvalidatingIterator.java b/document/src/main/java/com/yahoo/document/annotation/InvalidatingIterator.java
new file mode 100644
index 00000000000..3542ee247a2
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/annotation/InvalidatingIterator.java
@@ -0,0 +1,88 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+import java.util.ListIterator;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+class InvalidatingIterator implements ListIterator<SpanNode> {
+    private SpanList owner;
+    private ListIterator<SpanNode> base;
+    private SpanNode returnedFromNext = null;
+
+    InvalidatingIterator(SpanList owner, ListIterator<SpanNode> base) {
+        this.owner = owner;
+        this.base = base;
+    }
+
+    @Override
+    public boolean hasNext() {
+        returnedFromNext = null;
+        return base.hasNext();
+    }
+
+    @Override
+    public SpanNode next() {
+        SpanNode retval = null;
+        try {
+            retval = base.next();
+        } finally {
+            returnedFromNext = retval;
+        }
+        return returnedFromNext;
+    }
+
+    @Override
+    public boolean hasPrevious() {
+        returnedFromNext = null;
+        return base.hasPrevious();
+    }
+
+    @Override
+    public SpanNode previous() {
+        returnedFromNext = null;
+        return base.previous();
+    }
+
+    @Override
+    public int nextIndex() {
+        returnedFromNext = null;
+        return base.nextIndex();
+    }
+
+    @Override
+    public int previousIndex() {
+        returnedFromNext = null;
+        return base.previousIndex();
+    }
+
+    @Override
+    public void remove() {
+        if (returnedFromNext != null) {
+            returnedFromNext.setInvalid();
+            returnedFromNext.setParent(null);
+            owner.resetCachedFromAndTo();
+        }
+        returnedFromNext = null;
+        base.remove();
+    }
+
+    @Override
+    public void set(SpanNode spanNode) {
+        if (returnedFromNext != null) {
+            returnedFromNext.setInvalid();
+            returnedFromNext.setParent(null);
+        }
+        owner.resetCachedFromAndTo();
+        returnedFromNext = null;
+        base.set(spanNode);
+    }
+
+    @Override
+    public void add(SpanNode spanNode) {
+        returnedFromNext = null;
+        owner.resetCachedFromAndTo();
+        base.add(spanNode);
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/annotation/IteratingAnnotationContainer.java b/document/src/main/java/com/yahoo/document/annotation/IteratingAnnotationContainer.java
new file mode 100644
index 00000000000..588bf5f2826
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/annotation/IteratingAnnotationContainer.java
@@ -0,0 +1,34 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+import java.util.IdentityHashMap;
+import java.util.Iterator;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+abstract class IteratingAnnotationContainer extends AnnotationContainer {
+
+    @Override
+        Iterator<Annotation> iterator(SpanNode node) {
+        IdentityHashMap<SpanNode, SpanNode> nodes = new IdentityHashMap<SpanNode, SpanNode>();
+        nodes.put(node, node);
+        return iterator(nodes);
+    }
+
+    @Override
+    Iterator<Annotation> iteratorRecursive(SpanNode node) {
+        IdentityHashMap<SpanNode, SpanNode> nodes = new IdentityHashMap<SpanNode, SpanNode>();
+        nodes.put(node, node);
+        {
+            Iterator<SpanNode> childrenIt = node.childIteratorRecursive();
+            while (childrenIt.hasNext()) {
+                SpanNode child = childrenIt.next();
+                nodes.put(child, child);
+            }
+        }
+        return iterator(nodes);
+    }
+
+    abstract Iterator<Annotation> iterator(IdentityHashMap<SpanNode, SpanNode> nodes);
+}
diff --git a/document/src/main/java/com/yahoo/document/annotation/ListAnnotationContainer.java b/document/src/main/java/com/yahoo/document/annotation/ListAnnotationContainer.java
new file mode 100644
index 00000000000..0f03b9b269b
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/annotation/ListAnnotationContainer.java
@@ -0,0 +1,94 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+import java.util.Collection;
+import java.util.IdentityHashMap;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.ListIterator;
+import java.util.NoSuchElementException;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class ListAnnotationContainer extends IteratingAnnotationContainer {
+    private final List<Annotation> annotations = new LinkedList<Annotation>();
+
+    @Override
+    void annotateAll(Collection<Annotation> annotations) {
+        this.annotations.addAll(annotations);
+    }
+
+    @Override
+    void annotate(Annotation a) {
+        annotations.add(a);
+    }
+
+    @Override
+    Collection<Annotation> annotations() {
+        return annotations;
+    }
+
+    @Override
+    Iterator<Annotation> iterator(IdentityHashMap<SpanNode, SpanNode> nodes) {
+        return new AnnotationIterator(annotations.listIterator(), nodes);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (!(o instanceof ListAnnotationContainer)) return false;
+        ListAnnotationContainer that = (ListAnnotationContainer) o;
+        if (!annotations.equals(that.annotations)) return false;
+        return true;
+    }
+
+    @Override
+    public int hashCode() {
+        return annotations.hashCode();
+    }
+
+    private class AnnotationIterator implements Iterator<Annotation> {
+        private IdentityHashMap<SpanNode, SpanNode> nodes;
+        private PeekableListIterator<Annotation> base;
+        private boolean nextCalled = false;
+
+        AnnotationIterator(ListIterator<Annotation> baseIt, IdentityHashMap<SpanNode, SpanNode> nodes) {
+            this.base = new PeekableListIterator<Annotation>(baseIt);
+            this.nodes = nodes;
+        }
+
+        @Override
+        public boolean hasNext() {
+            nextCalled = false;
+            while (base.hasNext() && !nodes.containsKey(base.peek().getSpanNode())) {
+                base.next();
+            }
+            //now either, base has no next, or next is the correct node
+            if (base.hasNext()) {
+                return true;
+            }
+            return false;
+        }
+
+        @Override
+        public Annotation next() {
+            if (hasNext()) {
+                nextCalled = true;
+                return base.next();
+            } else {
+                throw new NoSuchElementException();
+            }
+        }
+
+        @Override
+        public void remove() {
+            if (!nextCalled) {
+                throw new IllegalStateException();
+            }
+            base.remove();
+            nextCalled = false;
+        }
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/annotation/PeekableListIterator.java b/document/src/main/java/com/yahoo/document/annotation/PeekableListIterator.java
new file mode 100644
index 00000000000..53a8a1b803e
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/annotation/PeekableListIterator.java
@@ -0,0 +1,107 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+import java.util.ListIterator;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+class PeekableListIterator<E> implements ListIterator<E> {
+    private E next;
+    private ListIterator<E> base;
+    boolean traversed = false;
+    private int position = -1;
+
+    PeekableListIterator(ListIterator<E> base) {
+        this.base = base;
+        this.traversed = false;
+    }
+
+    @Override
+    public boolean hasNext() {
+        return next != null || base.hasNext();
+    }
+
+    @Override
+    public E next() {
+        if (next == null) {
+            E n = base.next();
+            position++;
+            return n;
+        }
+        E retval = next;
+        next = null;
+        position++;
+        return retval;
+    }
+
+    @Override
+    public boolean hasPrevious() {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public E previous() {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public int nextIndex() {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public int previousIndex() {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public void remove() {
+        if (position < 0) {
+            //we have not next'ed the iterator, cannot do this:
+            throw new IllegalStateException("Cannot remove() before next()");
+        }
+        if (next != null) {
+            //we have already gone one step ahead. must back up two positions and then remove:
+            base.previous();
+            base.previous();
+            base.remove();
+        } else {
+            base.remove();
+        }
+        next = null;
+    }
+
+    @Override
+    public void set(E e) {
+        if (position < 0) {
+            //we have not next'ed the iterator, cannot do this:
+            throw new IllegalStateException("Cannot set() before next()");
+        }
+        if (next != null) {
+            //we have already gone one step ahead. must back up two positions and then remove:
+            base.previous();
+            base.previous();
+        }
+        base.set(e);
+        next = null;
+
+    }
+
+    @Override
+    public void add(E e) {
+        if (next != null) {
+            //we have already gone one step ahead. must back up one position and then add:
+            base.previous();
+        }
+        base.add(e);
+        next = null;
+    }
+
+    public E peek() {
+        if (next == null && base.hasNext()) {
+            next = base.next();
+        }
+        return next;
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/annotation/RecursiveNodeIterator.java b/document/src/main/java/com/yahoo/document/annotation/RecursiveNodeIterator.java
new file mode 100644
index 00000000000..ceecfcc2917
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/annotation/RecursiveNodeIterator.java
@@ -0,0 +1,107 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+import java.util.ListIterator;
+import java.util.NoSuchElementException;
+import java.util.Stack;
+
+/**
+ * ListIterator implementation which performs a depth-first traversal of SpanNodes.
+ *
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+class RecursiveNodeIterator implements ListIterator<SpanNode> {
+    protected Stack<PeekableListIterator<SpanNode>> stack = new Stack<PeekableListIterator<SpanNode>>();
+    protected ListIterator<SpanNode> iteratorFromLastCallToNext = null;
+
+    RecursiveNodeIterator(ListIterator<SpanNode> it) {
+        stack.push(new PeekableListIterator<SpanNode>(it));
+    }
+
+    protected RecursiveNodeIterator() {
+    }
+
+    @Override
+    public boolean hasNext() {
+        if (stack.isEmpty()) {
+            return false;
+        }
+        PeekableListIterator<SpanNode> iterator = stack.peek();
+        if (!iterator.hasNext()) {
+            stack.pop();
+            return hasNext();
+        }
+
+
+        SpanNode node = iterator.peek();
+
+        if (!iterator.traversed) {
+            //we set the traversed flag on our way down
+            iterator.traversed = true;
+            stack.push(new PeekableListIterator<SpanNode>(node.childIterator()));
+            return hasNext();
+        }
+
+        return true;
+    }
+
+    @Override
+    public SpanNode next() {
+        if (stack.isEmpty() || !hasNext()) {
+            iteratorFromLastCallToNext = null;
+            throw new NoSuchElementException("No next element available.");
+        }
+        stack.peek().traversed = false;
+        iteratorFromLastCallToNext = stack.peek();
+        return stack.peek().next();
+    }
+
+    @Override
+    public boolean hasPrevious() {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public SpanNode previous() {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public int nextIndex() {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public int previousIndex() {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public void remove() {
+        if (stack.isEmpty()) {
+            throw new IllegalStateException();
+        }
+        if (iteratorFromLastCallToNext != null) {
+            iteratorFromLastCallToNext.remove();
+        } else {
+            throw new IllegalStateException();
+        }
+    }
+
+    @Override
+    public void set(SpanNode spanNode) {
+        if (stack.isEmpty()) {
+            throw new IllegalStateException();
+        }
+        if (iteratorFromLastCallToNext != null) {
+            iteratorFromLastCallToNext.set(spanNode);
+        } else {
+            throw new IllegalStateException();
+        }
+    }
+
+    @Override
+    public void add(SpanNode spanNode) {
+        throw new UnsupportedOperationException();
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/annotation/SerialIterator.java b/document/src/main/java/com/yahoo/document/annotation/SerialIterator.java
new file mode 100644
index 00000000000..d86d90dc0d0
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/annotation/SerialIterator.java
@@ -0,0 +1,31 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+
+import java.util.List;
+import java.util.ListIterator;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class SerialIterator extends RecursiveNodeIterator {
+    SerialIterator(List<ListIterator<SpanNode>> iterators) {
+        //the first iterator must be on top of the stack:
+        for (int i = iterators.size() - 1; i > -1; i--) {
+            stack.push(new PeekableListIterator<SpanNode>(iterators.get(i)));
+        }
+    }
+
+    @Override
+    public boolean hasNext() {
+        if (stack.isEmpty()) {
+            return false;
+        }
+        PeekableListIterator<SpanNode> iterator = stack.peek();
+        if (!iterator.hasNext()) {
+            stack.pop();
+            return hasNext();
+        }
+        return true;
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/annotation/Span.java b/document/src/main/java/com/yahoo/document/annotation/Span.java
new file mode 100644
index 00000000000..87bd568b94a
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/annotation/Span.java
@@ -0,0 +1,180 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+import com.yahoo.document.serialization.SpanNodeReader;
+
+import java.util.ListIterator;
+import java.util.NoSuchElementException;
+
+/**
+ * This class represents a range of characters from a string.&nbsp;This is the leaf node
+ * in a Span tree. Its boundaries are defined as inclusive-from and exclusive-to.
+ *
+ * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a>
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class Span extends SpanNode {
+    public static final byte ID = 1;
+    private int from;
+    private int length;
+
+    /**
+     * This will construct a valid span or throw {@link IllegalArgumentException}
+     * if the span is invalid.
+     *
+     * @param from   Start of the span. Must be &gt;= 0.
+     * @param length of the span. Must be &gt;= 0.
+     * @throws IllegalArgumentException if illegal span
+     */
+    public Span(int from, int length) {
+        setFrom(from);
+        setLength(length);
+    }
+
+    /**
+     * Creates an empty Span, used mainly for deserialization.
+     *
+     * @param reader the reader that must populate this Span instance
+     */
+    public Span(SpanNodeReader reader) {
+        reader.read(this);
+    }
+
+    /**
+     * WARNING!&nbsp;Only to be used by deserializers!&nbsp;Creates an empty Span instance.
+     */
+    public Span() {
+    }
+
+    /**
+     * Copies the given Span into a new Span instance.
+     *
+     * @param spanToCopy the Span to copy.
+     */
+    public Span(Span spanToCopy) {
+        this(spanToCopy.getFrom(), spanToCopy.getLength());
+    }
+
+    @Override
+    public final int getFrom() {
+        return from;
+    }
+
+    /**
+     * NOTE: DO NOT USE. Should only be used by {@link com.yahoo.document.serialization.SpanNodeReader}.
+     * @param from the from value to set
+     */
+    public void setFrom(int from) {
+        if (from < 0) {
+            throw new IllegalArgumentException("From cannot be < 0. (Was " + from + ").");
+        }
+        this.from = from;
+    }
+
+    @Override
+    public final int getTo() {
+        return from + length;
+    }
+
+    @Override
+    public final int getLength() {
+        return length;
+    }
+
+    /**
+     * NOTE: DO NOT USE. Should only be used by {@link com.yahoo.document.serialization.SpanNodeReader}.
+     * @param length the length value to set
+     */
+    public void setLength(int length) {
+        if (length < 0) {
+            throw new IllegalArgumentException("Length cannot be < 0. (Was " + length + ").");
+        }
+        this.length = length;
+    }
+
+    public String toString() {
+        return new StringBuilder("span [").append(from).append(',').append(getTo()).append('>').toString();
+    }
+
+    @Override
+    public final CharSequence getText(CharSequence text) {
+        return text.subSequence(from, getTo());
+    }
+
+    /**
+     * Always returns true.
+     *
+     * @return always true.
+     */
+    @Override
+    public boolean isLeafNode() {
+        return true;
+    }
+
+    /**
+     * Returns a ListIterator that iterates over absolutely nothing.
+     *
+     * @return a ListIterator that iterates over absolutely nothing.
+     */
+    @Override
+    public ListIterator<SpanNode> childIterator() {
+        return new EmptyIterator();
+    }
+
+    /**
+     * Returns a ListIterator that iterates over absolutely nothing.
+     *
+     * @return a ListIterator that iterates over absolutely nothing.
+     */
+    @Override
+    public ListIterator<SpanNode> childIteratorRecursive() {
+        return childIterator();
+    }
+
+    private class EmptyIterator implements ListIterator<SpanNode> {
+        @Override
+        public boolean hasNext() {
+            return false;
+        }
+
+        @Override
+        public SpanNode next() {
+            throw new NoSuchElementException("A Span has no children");
+        }
+
+        @Override
+        public boolean hasPrevious() {
+            return false;
+        }
+
+        @Override
+        public SpanNode previous() {
+            throw new NoSuchElementException("A Span has no children");
+        }
+
+        @Override
+        public int nextIndex() {
+            return 0;
+        }
+
+        @Override
+        public int previousIndex() {
+            return 0;
+        }
+
+        @Override
+        public void remove() {
+            throw new UnsupportedOperationException("A Span has no children");
+        }
+
+        @Override
+        public void set(SpanNode spanNode) {
+            throw new UnsupportedOperationException("A Span has no children");
+        }
+
+        @Override
+        public void add(SpanNode spanNode) {
+            throw new UnsupportedOperationException("A Span has no children");
+        }
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/annotation/SpanList.java b/document/src/main/java/com/yahoo/document/annotation/SpanList.java
new file mode 100644
index 00000000000..baa86bd7a6f
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/annotation/SpanList.java
@@ -0,0 +1,418 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+import com.yahoo.document.serialization.SpanNodeReader;
+
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.ListIterator;
+
+/**
+ * A node in a Span tree that can have child nodes.
+ *
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class SpanList extends SpanNode {
+    public static final byte ID = 2;
+    private final List<SpanNode> children;
+    private int cachedFrom = Integer.MIN_VALUE; //triggers calculateFrom()
+    private int cachedTo = Integer.MIN_VALUE;  //triggers calculateTo()
+
+    /** Creates a new SpanList. */
+    public SpanList() {
+        this.children = new LinkedList<SpanNode>();
+    }
+
+    public SpanList(SpanNodeReader reader) {
+        this();
+        reader.read(this);
+    }
+
+    protected SpanList(List<SpanNode> children) {
+        this.children = children;
+    }
+
+    /**
+     * Deep-copies a SpanList.
+     *
+     * @param other the SpanList to copy.
+     */
+    public SpanList(SpanList other) {
+        this.children = new LinkedList<SpanNode>();
+        for (SpanNode otherNode : other.children) {
+            if (otherNode instanceof Span) {
+                children.add(new Span((Span) otherNode));
+            } else if (otherNode instanceof AlternateSpanList) {
+                children.add(new AlternateSpanList((AlternateSpanList) otherNode));
+            } else if (otherNode instanceof SpanList) {
+                children.add(new SpanList((SpanList) otherNode));
+            } else if (otherNode instanceof DummySpanNode) {
+                children.add(otherNode);  //shouldn't really happen
+            } else {
+                throw new IllegalStateException("Cannot create copy of " + otherNode + " with class "
+                                                + ((otherNode == null) ? "null" : otherNode.getClass()));
+            }
+        }
+    }
+
+    void checkValidity(SpanNode node, List<SpanNode> childrenToCheck) {
+        if (!node.isValid()) {
+            throw new IllegalStateException("Cannot reuse SpanNode instance " + node + ", is INVALID.");
+        }
+        if (node.getParent() != null) {
+            if (node.getParent() != this) {
+                throw new IllegalStateException(node + " is already a child of " + node.getParent() + ", cannot be added to " + this);
+            } else if (node.getParent() == this && childrenToCheck.contains(node)) {
+                throw new IllegalStateException(node + " is already a child of " + this + ", cannot be added twice to the same parent node.");
+            }
+        }
+    }
+
+    /**
+     * Adds a child node to this SpanList.
+     *
+     * @param node the node to add.
+     * @return this, for call chaining
+     * @throws IllegalStateException if SpanNode.isValid() returns false.
+     */
+    public SpanList add(SpanNode node) {
+        checkValidity(node, children());
+        node.setParent(this);
+        resetCachedFromAndTo();
+        children().add(node);
+        return this;
+    }
+
+    /** Create a span, add it to this list and return it */
+    public Span span(int from, int length) {
+        Span span=new Span(from,length);
+        add(span);
+        return span;
+    }
+
+    void setInvalid() {
+        //invalidate ourselves:
+        super.setInvalid();
+        //invalidate all our children:
+        for (SpanNode node : children()) {
+            node.setInvalid();
+        }
+    }
+
+    /**
+     * Moves a child of this SpanList to another SpanList.
+     *
+     * @param node the node to move
+     * @param target the SpanList to add the node to
+     * @throws IllegalArgumentException if the given node is not a child of this SpanList
+     */
+    public void move(SpanNode node, SpanList target) {
+        boolean removed = children().remove(node);
+        if (removed) {
+            //we found the node
+            node.setParent(null);
+            resetCachedFromAndTo();
+            target.add(node);
+        } else {
+            throw new IllegalArgumentException("Node " + node + " is not a child of this SpanList, cannot move.");
+        }
+    }
+
+    /**
+     * Moves a child of this SpanList to another SpanList.
+     *
+     * @param nodeNum the index of the node to move
+     * @param target the SpanList to add the node to
+     * @throws IndexOutOfBoundsException if the given index is out of range
+     */
+    public void move(int nodeNum, SpanList target) {
+        SpanNode node = children().remove(nodeNum);
+        if (node != null) {
+            //we found the node
+            node.setParent(null);
+            resetCachedFromAndTo();
+            target.add(node);
+        }
+    }
+
+    /**
+     * Moves a child of this SpanList to another SpanList.
+     *
+     * @param node the node to move
+     * @param target the SpanList to add the node to
+     * @param targetSubTree the index of the subtree of the given AlternateSpanList to add the node to
+     * @throws IllegalArgumentException if the given node is not a child of this SpanList
+     * @throws IndexOutOfBoundsException if the target subtree index is out of range
+     */
+    public void move(SpanNode node, AlternateSpanList target, int targetSubTree) {
+        if (targetSubTree < 0 || targetSubTree >= target.getNumSubTrees()) {
+            throw new IndexOutOfBoundsException(target + " has no subtree at index " + targetSubTree);
+        }
+        boolean removed = children().remove(node);
+        if (removed) {
+            //we found the node
+            node.setParent(null);
+            resetCachedFromAndTo();
+            target.add(targetSubTree, node);
+        } else {
+            throw new IllegalArgumentException("Node " + node + " is not a child of this SpanList, cannot move.");
+        }
+    }
+
+    /**
+     * Moves a child of this SpanList to another SpanList.
+     *
+     * @param nodeNum the index of the node to move
+     * @param target the SpanList to add the node to
+     * @param targetSubTree the index of the subtree of the given AlternateSpanList to add the node to
+     * @throws IndexOutOfBoundsException if the given index is out of range, or the target subtree index is out of range
+     */
+    public void move(int nodeNum, AlternateSpanList target, int targetSubTree) {
+        if (targetSubTree < 0 || targetSubTree >= target.getNumSubTrees()) {
+            throw new IndexOutOfBoundsException(target + " has no subtree at index " + targetSubTree);
+        }
+        SpanNode node = children().remove(nodeNum);
+        if (node != null) {
+            //we found the node
+            node.setParent(null);
+            resetCachedFromAndTo();
+            target.add(targetSubTree, node);
+        }
+    }
+
+
+    /**
+     * Removes and invalidates the given SpanNode from this.
+     *
+     * @param node the node to remove.
+     * @return this, for chaining.
+     */
+    public SpanList remove(SpanNode node) {
+        boolean removed = children().remove(node);
+        if (removed) {
+            node.setParent(null);
+            resetCachedFromAndTo();
+            node.setInvalid();
+        }
+        return this;
+    }
+
+    /**
+     * Removes and invalidates the SpanNode at the given index from this.
+     *
+     * @param i the index of the node to remove.
+     * @return this, for chaining.
+     */
+    public SpanList remove(int i) {
+        SpanNode node = children().remove(i);
+        if (node != null) {
+            node.setParent(null);
+            node.setInvalid();
+        }
+        return this;
+    }
+
+    /**
+     * Returns a <strong>modifiable</strong> list of the immediate children of this SpanList.
+     *
+     * @return a <strong>modifiable</strong> list of the immediate children of this SpanList.
+     */
+    protected List<SpanNode> children() {
+        return children;
+    }
+
+    /**
+     * Returns the number of children this SpanList holds.
+     *
+     * @return the number of children this SpanList holds.
+     */
+    public int numChildren() {
+        return children().size();
+    }
+
+    /**
+     * Traverses all immediate children of this SpanList. The ListIterator returned support all optional operations
+     * specified in the ListIterator interface.
+     *
+     * @return a ListIterator which traverses all immediate children of this SpanNode
+     * @see java.util.ListIterator
+     */
+    @Override
+    public ListIterator<SpanNode> childIterator() {
+        return new InvalidatingIterator(this, children().listIterator());
+    }
+
+    /**
+     * Recursively traverses all children (not only leaf nodes) of this SpanList, in a depth-first fashion.
+     * The ListIterator only supports iteration forwards, and the optional operations that are implemented are
+     * remove() and set(). add() is not supported.
+     *
+     * @return a ListIterator which recursively traverses all children and their children etc. of this SpanList.
+     * @see java.util.ListIterator
+     */
+    @Override
+    public ListIterator<SpanNode> childIteratorRecursive() {
+        return new InvalidatingIterator(this, new RecursiveNodeIterator(children().listIterator()));
+    }
+
+    /** Removes and invalidates all references to child nodes. */
+    public void clearChildren() {
+        for (SpanNode node : children()) {
+            node.setInvalid();
+            node.setParent(null);
+        }
+        children().clear();
+        resetCachedFromAndTo();
+    }
+
+    /**
+     * Sorts children by occurrence in the text covered.
+     *
+     * @see SpanNode#compareTo(SpanNode)
+     */
+    public void sortChildren() {
+        Collections.sort(children());
+    }
+
+    /**
+     * Recursively sorts all children by occurrence in the text covered.
+     */
+    public void sortChildrenRecursive() {
+        for (SpanNode node : children()) {
+            if (node instanceof SpanList) {
+                ((SpanList) node).sortChildrenRecursive();
+            }
+            Collections.sort(children());
+        }
+    }
+
+    /**
+     * Always returns false, even if this node has no children.
+     *
+     * @return always false, even if this node has no children
+     */
+    @Override
+    public boolean isLeafNode() {
+        return false;
+    }
+
+    private void calculateFrom() {
+        int smallestFrom = Integer.MAX_VALUE;
+        for (SpanNode n : children()) {
+            final int thisFrom = n.getFrom();
+            if (thisFrom != -1) {
+                smallestFrom = Math.min(thisFrom, smallestFrom);
+            }
+        }
+        if (smallestFrom == Integer.MAX_VALUE) {
+            //all children were empty SpanLists which returned -1
+            smallestFrom = -1;
+        }
+        cachedFrom = smallestFrom;
+    }
+
+    /**
+     * Returns the character index where this {@link SpanNode} starts (inclusive), i.e.&nbsp;the smallest {@link com.yahoo.document.annotation.SpanNode#getFrom()} of all children.
+     *
+     * @return the lowest getFrom() of all children, or -1 if this SpanList has no children.
+     */
+    @Override
+    public int getFrom() {
+        if (children().isEmpty()) {
+            return -1;
+        }
+        if (cachedFrom == Integer.MIN_VALUE) {
+            calculateFrom();
+        }
+        return cachedFrom;
+    }
+
+    private void calculateTo() {
+        int greatestTo = Integer.MIN_VALUE;
+        for (SpanNode n : children()) {
+            greatestTo = Math.max(n.getTo(), greatestTo);
+        }
+        cachedTo = greatestTo;
+    }
+
+    /**
+     * Returns the character index where this {@link SpanNode} ends (exclusive), i.e.&nbsp;the greatest {@link com.yahoo.document.annotation.SpanNode#getTo()} of all children.
+     *
+     * @return the greatest getTo() of all children, or -1 if this SpanList has no children.
+     */
+    @Override
+    public int getTo() {
+        if (children().isEmpty()) {
+            return -1;
+        }
+        if (cachedTo == Integer.MIN_VALUE) {
+            calculateTo();
+        }
+        return cachedTo;
+    }
+
+    void resetCachedFromAndTo() {
+        cachedFrom = Integer.MIN_VALUE;
+        cachedTo = Integer.MIN_VALUE;
+        if (getParent() instanceof SpanList) {
+            ((SpanList) getParent()).resetCachedFromAndTo();
+        }
+    }
+
+    /**
+     * Returns the length of this span, i.e.&nbsp;getFrom() - getTo().
+     *
+     * @return the length of this span
+     */
+    @Override
+    public int getLength() {
+        return getTo() - getFrom();
+    }
+
+    /**
+     * Returns the text that is covered by this SpanNode.
+     *
+     * @param text the input text
+     * @return the text that is covered by this SpanNode.
+     */
+    @Override
+    public CharSequence getText(CharSequence text) {
+        if (children().isEmpty()) {
+            return "";
+        }
+        StringBuilder str = new StringBuilder();
+        for (SpanNode node : children()) {
+            CharSequence childText = node.getText(text);
+            if (childText != null) {
+                str.append(node.getText(text));
+            }
+        }
+        return str;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (!(o instanceof SpanList)) return false;
+        if (!super.equals(o)) return false;
+
+        SpanList spanList = (SpanList) o;
+
+        if (children() != null ? !children().equals(spanList.children()) : spanList.children() != null) return false;
+
+        return true;
+    }
+
+    @Override
+    public int hashCode() {
+        int result = super.hashCode();
+        result = 31 * result + (children() != null ? children().hashCode() : 0);
+        return result;
+    }
+
+    @Override
+    public String toString() {
+        return "SpanList with " + children().size() + " children";
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/annotation/SpanNode.java b/document/src/main/java/com/yahoo/document/annotation/SpanNode.java
new file mode 100644
index 00000000000..712bb7bf5c5
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/annotation/SpanNode.java
@@ -0,0 +1,320 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.IntegerFieldValue;
+import com.yahoo.document.datatypes.StringFieldValue;
+
+import java.util.ListIterator;
+
+/**
+ * Base class of nodes in a Span tree.
+ *
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public abstract class SpanNode implements Comparable<SpanNode>, SpanNodeParent {
+
+    private boolean valid = true;
+    /**
+     * This scratch id is used to avoid using IdentityHashMaps as they are very costly.
+     */
+    private int scratchId = -1;
+    private SpanNodeParent parent;
+
+    protected SpanNode() {
+    }
+
+    /**
+     * Returns whether this node is valid or not.&nbsp;When a child node from a SpanList, the child
+     * is marked as invalid, and the reference to it is removed from the parent SpanList. However,
+     * Annotations in the global list kept in SpanTree may still have references to the removed SpanNode.
+     * Removing these references is costly, and is only done when calling {@link com.yahoo.document.annotation.SpanTree#cleanup()}.
+     *
+     * @return true if this node is valid, false otherwise.
+     */
+    public boolean isValid() {
+        return valid;
+    }
+
+    void setInvalid() {
+        valid = false;
+    }
+
+    public void setScratchId(int id) {
+        scratchId = id;
+    }
+
+    public int getScratchId() {
+        return scratchId;
+    }
+    /**
+     * Returns the parent node of this SpanNode, if any.
+     *
+     * @return the parent node, or null if this is not yet added to a parent SpanList
+     */
+    public SpanNodeParent getParent() {
+        return parent;
+    }
+
+    void setParent(SpanNodeParent parent) {
+        this.parent = parent;
+    }
+
+    /**
+     * Returns the SpanTree that this node belongs to, if any.
+     *
+     * @return the SpanTree that this node belongs to, or null if it is not yet added to a SpanTree.
+     */
+    @Override
+    public SpanTree getSpanTree() {
+        if (parent == null) {
+            return null;
+        }
+        return parent.getSpanTree();
+    }
+
+    /** Returns the SpanTree this belongs to and throws a nice NullPointerException if none */
+    private SpanTree getNonNullSpanTree() {
+        SpanTree spanTree=getSpanTree();
+        if (spanTree==null)
+            throw new NullPointerException(this + " is not attached to a SpanTree through its parent yet");
+        return spanTree;
+    }
+
+
+    /**
+     * Convenience method for adding an annotation to this span, same as
+     * <code>getSpanTree().{@link SpanTree#annotate(SpanNode,Annotation) spanTree.annotate(this,annotation)}</code>
+     *
+     * @param annotation the annotation to add
+     * @return this for chaining
+     * @throws NullPointerException if this span is not attached to a tree
+     */
+    public SpanNode annotate(Annotation annotation) {
+        getNonNullSpanTree().annotate(this, annotation);
+        return this;
+    }
+
+    /**
+     * Convenience method for adding an annotation to this span, same as
+     * <code>getSpanTree().{@link SpanTree#annotate(SpanNode,AnnotationType,FieldValue) spanTree.annotate(this,type,value)}</code>
+     *
+     * @param type the type of the annotation to add
+     * @param value the value of the annotation to add
+     * @return this for chaining
+     * @throws NullPointerException if this span is not attached to a tree
+     */
+    public SpanNode annotate(AnnotationType type,FieldValue value) {
+        getNonNullSpanTree().annotate(this,type,value);
+        return this;
+    }
+
+    /**
+     * Convenience method for adding an annotation to this span, same as
+     * <code>getSpanTree().{@link SpanTree#annotate(SpanNode,AnnotationType,FieldValue) spanTree.annotate(this,type,new StringFieldValue(value))}</code>
+     *
+     * @param type the type of the annotation to add
+     * @param value the string value of the annotation to add
+     * @return this for chaining
+     * @throws NullPointerException if this span is not attached to a tree
+     */
+    public SpanNode annotate(AnnotationType type,String value) {
+        getNonNullSpanTree().annotate(this, type, new StringFieldValue(value));
+        return this;
+    }
+
+    /**
+     * Convenience method for adding an annotation to this span, same as
+     * <code>getSpanTree().{@link SpanTree#annotate(SpanNode,AnnotationType,FieldValue) spanTree.annotate(this,type,new IntegerFieldValue(value))}</code>
+     *
+     * @param type the type of the annotation to add
+     * @param value the integer value of the annotation to add
+     * @return this for chaining
+     * @throws NullPointerException if this span is not attached to a tree
+     */
+    public SpanNode annotate(AnnotationType type,Integer value) {
+        getNonNullSpanTree().annotate(this, type, new IntegerFieldValue(value));
+        return this;
+    }
+
+    /**
+     * Convenience method for adding an annotation with no value to this span, same as
+     * <code>getSpanTree().{@link SpanTree#annotate(SpanNode,AnnotationType) spanTree.annotate(this,type)}</code>
+     *
+     * @param type the type of the annotation to add
+     * @return this for chaining
+     * @throws NullPointerException if this span is not attached to a tree
+     */
+    public SpanNode annotate(AnnotationType type) {
+        getNonNullSpanTree().annotate(this,type);
+        return this;
+    }
+
+    /**
+     * Returns the StringFieldValue that this node belongs to, if any.
+     *
+     * @return the StringFieldValue that this node belongs to, if any, otherwise null.
+     */
+    @Override
+    public StringFieldValue getStringFieldValue() {
+        if (parent == null) {
+            return null;
+        }
+        return parent.getStringFieldValue();
+    }
+
+    /**
+     * Returns true if this node is a leaf node in the tree.
+     *
+     * @return true if this node is a leaf node in the tree.
+     */
+    public abstract boolean isLeafNode();
+
+    /**
+     * Traverses all immediate children of this SpanNode.
+     *
+     * @return a ListIterator which traverses all immediate children of this SpanNode
+     */
+    public abstract ListIterator<SpanNode> childIterator();
+
+    /**
+     * Recursively traverses all possible children (not only leaf nodes) of this SpanNode, in a depth-first fashion.
+     *
+     * @return a ListIterator which recursively traverses all children and their children etc. of this SpanNode.
+     */
+    public abstract ListIterator<SpanNode> childIteratorRecursive();
+
+    /**
+     * Returns the character index where this SpanNode starts (inclusive).
+     *
+     * @return the character index where this SpanNode starts (inclusive).
+     */
+    public abstract int getFrom();
+
+    /**
+     * Returns the character index where this SpanNode ends (exclusive).
+     *
+     * @return the character index where this SpanNode ends (exclusive).
+     */
+    public abstract int getTo();
+
+    /**
+     * Returns the length of this span, i.e.&nbsp;getFrom() - getTo().
+     *
+     * @return the length of this span
+     */
+    public abstract int getLength();
+
+    /**
+     * Returns the text that is covered by this SpanNode.
+     *
+     * @param text the input text
+     * @return the text that is covered by this SpanNode.
+     */
+    public abstract CharSequence getText(CharSequence text);
+
+    /**
+     * Checks if the text covered by this span overlaps with the text covered by another span.
+     *
+     * @param o the other SpanNode to check
+     * @return true if spans are overlapping, false otherwise
+     */
+    public boolean overlaps(SpanNode o) {
+        int from = getFrom();
+        int otherFrom = o.getFrom();
+        int to = getTo();
+        int otherTo = o.getTo();
+
+        //is other from within our range, or vice versa?
+        if ((otherFrom >= from && otherFrom < to)
+            || (from >= otherFrom && from < otherTo)) {
+            return true;
+        }
+
+        //is other to within our range, or vice versa?
+        if ((otherTo > from && otherTo <= to)
+            || (to > otherFrom && to <= otherTo)) {
+            return true;
+        }
+        return false;
+    }
+
+    /**
+     * Checks if the text covered by another span is within the text covered by this span.
+     *
+     * @param o the other SpanNode to check.
+     * @return true if the text covered by another span is within the text covered by this span, false otherwise.
+     */
+    public boolean contains(SpanNode o) {
+        int from = getFrom();
+        int otherFrom = o.getFrom();
+        int to = getTo();
+        int otherTo = o.getTo();
+
+        if (otherFrom >= from && otherTo <= to) {
+            //other span node is within our range:
+            return true;
+        }
+        return false;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (!(o instanceof SpanNode)) return false;
+
+        SpanNode spanNode = (SpanNode) o;
+
+        if (getFrom() != spanNode.getFrom()) return false;
+        if (getTo() != spanNode.getTo()) return false;
+
+        return true;
+    }
+
+    @Override
+    public int hashCode() {
+        int result = getFrom();
+        result = 31 * result + getTo();
+        return result;
+    }
+
+    /**
+     * Compares two SpanNodes.&nbsp;Note: this class has a natural ordering that <strong>might be</strong> inconsistent with equals.
+     * <p>
+     * First, getFrom() is compared, and -1 or 1 is return if our getFrom() is smaller or greater that o.getFrom(), respectively.
+     * If and only if getFrom() is equal, getTo() is compared, and  -1 or 1 is return if our getTo() is smaller or greater that o.getTo(), respectively.
+     * In all other cases, the two SpanNodes are equal both for getFrom() and getTo(), and 0 is returned.
+     * <p>
+     * Note that if a subclass has overridden equals(), which is very likely, but has not overridden compareTo(), then that subclass
+     * will have a natural ordering that is inconsistent with equals.
+     *
+     * @param o the SpanNode to compare to
+     * @return a negative integer, zero, or a positive integer as this object is less than, equal to, or greater than the specified object
+     */
+    @Override
+    public int compareTo(SpanNode o) {
+        int from = getFrom();
+        int otherFrom = o.getFrom();
+
+        if (from < otherFrom) {
+            return -1;
+        }
+        if (from > otherFrom) {
+            return 1;
+        }
+
+        //so from is equal. Check to:
+        int to = getTo();
+        int otherTo = o.getTo();
+
+        if (to < otherTo) {
+            return -1;
+        }
+        if (to > otherTo) {
+            return 1;
+        }
+
+        //both from and to are equal
+        return 0;
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/annotation/SpanNode2AnnotationContainer.java b/document/src/main/java/com/yahoo/document/annotation/SpanNode2AnnotationContainer.java
new file mode 100644
index 00000000000..8f10d7c0140
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/annotation/SpanNode2AnnotationContainer.java
@@ -0,0 +1,134 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+import org.apache.commons.collections.map.MultiValueMap;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.IdentityHashMap;
+import java.util.Iterator;
+import java.util.List;
+
+/**
+ * TODO: Should this be removed?
+ *
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+class SpanNode2AnnotationContainer extends AnnotationContainer {
+    private final MultiValueMap spanNode2Annotation = MultiValueMap.decorate(new IdentityHashMap());
+
+    @Override
+    void annotateAll(Collection<Annotation> annotations) {
+        for (Annotation a : annotations) {
+            annotate(a);
+        }
+    }
+
+    @Override
+    void annotate(Annotation a) {
+        if (a.getSpanNode() == null) {
+            spanNode2Annotation.put(DummySpanNode.INSTANCE, a);
+        } else {
+            spanNode2Annotation.put(a.getSpanNode(), a);
+        }
+    }
+
+    @Override
+    @SuppressWarnings("unchecked")
+    Collection<Annotation> annotations() {
+        return spanNode2Annotation.values();
+    }
+
+    @Override
+    @SuppressWarnings("unchecked")
+    Iterator<Annotation> iterator(SpanNode node) {
+        Collection<Annotation> annotationsForNode = spanNode2Annotation.getCollection(node);
+        if (annotationsForNode == null) {
+            return Collections.<Annotation>emptyList().iterator();
+        }
+        return annotationsForNode.iterator();    }
+
+    @Override
+    @SuppressWarnings("unchecked")
+    Iterator<Annotation> iteratorRecursive(SpanNode node) {
+        IdentityHashMap<SpanNode, SpanNode> nodes = new IdentityHashMap<SpanNode, SpanNode>();
+        nodes.put(node, node);
+        {
+            Iterator<SpanNode> childrenIt = node.childIteratorRecursive();
+            while (childrenIt.hasNext()) {
+                SpanNode child = childrenIt.next();
+                nodes.put(child, child);
+            }
+        }
+        List<Collection<Annotation>> annotationLists = new ArrayList<Collection<Annotation>>(nodes.size());
+        for (SpanNode includedNode : nodes.keySet()) {
+            Collection<Annotation> includedAnnotations = spanNode2Annotation.getCollection(includedNode);
+            if (includedAnnotations != null) {
+                annotationLists.add(includedAnnotations);
+            }
+        }
+        return new AnnotationCollectionIterator(annotationLists);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (!(o instanceof SpanNode2AnnotationContainer)) return false;
+        SpanNode2AnnotationContainer that = (SpanNode2AnnotationContainer) o;
+        if (!spanNode2Annotation.equals(that.spanNode2Annotation)) return false;
+        return true;
+    }
+
+    @Override
+    public int hashCode() {
+        return spanNode2Annotation.hashCode();
+    }
+
+    private class AnnotationCollectionIterator implements Iterator<Annotation> {
+        private final List<Collection<Annotation>> annotationLists;
+        private Iterator<Annotation> currentIterator;
+        private boolean nextCalled = false;
+
+        AnnotationCollectionIterator(List<Collection<Annotation>> annotationLists) {
+            this.annotationLists = annotationLists;
+            if (annotationLists.isEmpty()) {
+                currentIterator = Collections.<Annotation>emptyList().iterator();
+            } else {
+                currentIterator = annotationLists.remove(0).iterator();
+            }
+        }
+
+        @Override
+        public boolean hasNext() {
+            nextCalled = false;
+            if (currentIterator.hasNext()) {
+                return true;
+            }
+            if (annotationLists.isEmpty()) {
+                return false;
+            }
+            currentIterator = annotationLists.remove(0).iterator();
+            return hasNext();
+        }
+
+        @Override
+        public Annotation next() {
+            if (hasNext()) {
+                nextCalled = true;
+                return currentIterator.next();
+            }
+            return null;
+        }
+
+        @Override
+        public void remove() {
+            if (nextCalled) {
+                currentIterator.remove();
+            } else {
+                throw new IllegalStateException();
+            }
+        }
+    }
+
+}
diff --git a/document/src/main/java/com/yahoo/document/annotation/SpanNodeParent.java b/document/src/main/java/com/yahoo/document/annotation/SpanNodeParent.java
new file mode 100644
index 00000000000..8618685d77a
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/annotation/SpanNodeParent.java
@@ -0,0 +1,26 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+import com.yahoo.document.datatypes.StringFieldValue;
+
+/**
+ * An interface to be implemented by classes that can be parents of SpanNodes.
+ *
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ * @see SpanNode#getParent()
+ */
+public interface SpanNodeParent {
+    /**
+     * Returns the SpanTree of this, if any.
+     *
+     * @return the SpanTree of this, if it belongs to a SpanTree, otherwise null.
+     */
+    public SpanTree getSpanTree();
+
+    /**
+     * Returns the StringFieldValue that this node belongs to, if any.
+     *
+     * @return the StringFieldValue that this node belongs to, if any, otherwise null.
+     */
+    public StringFieldValue getStringFieldValue();
+}
diff --git a/document/src/main/java/com/yahoo/document/annotation/SpanTree.java b/document/src/main/java/com/yahoo/document/annotation/SpanTree.java
new file mode 100644
index 00000000000..7385461504d
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/annotation/SpanTree.java
@@ -0,0 +1,699 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+import com.google.common.collect.ImmutableList;
+import com.yahoo.document.CollectionDataType;
+import com.yahoo.document.Field;
+import com.yahoo.document.MapDataType;
+import com.yahoo.document.StructuredDataType;
+import com.yahoo.document.datatypes.CollectionFieldValue;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.MapFieldValue;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.document.datatypes.StructuredFieldValue;
+import org.apache.commons.collections.CollectionUtils;
+
+import java.util.*;
+
+/**
+ * A SpanTree holds a root node of a tree of SpanNodes, and a List of Annotations pointing to these nodes
+ * or each other.&nbsp;It also has a name.
+ *
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ * @see com.yahoo.document.annotation.SpanNode
+ * @see com.yahoo.document.annotation.Annotation
+ */
+public class SpanTree implements Iterable<Annotation>, SpanNodeParent, Comparable<SpanTree> {
+
+    private String name;
+    private SpanNode root;
+    private AnnotationContainer annotations = new ListAnnotationContainer();
+    private StringFieldValue stringFieldValue;
+
+    /**
+     * WARNING!&nbsp;Only to be used by deserializers!&nbsp;Creates an empty SpanTree instance.
+     */
+    public SpanTree() {
+    }
+
+    /**
+     * Creates a new SpanTree with a given root node.
+     *
+     * @param name the name of the span tree
+     * @param root the root node of the span tree
+     * @throws IllegalStateException if the root node is invalid
+     */
+    public SpanTree(String name, SpanNode root) {
+        this.name = name;
+        setRoot(root);
+    }
+
+    /**
+     * Creates a new SpanTree with the given name and an empty SpanList as its root node.
+     *
+     * @param name the name of the span tree
+     */
+    public SpanTree(String name) {
+        this.name = name;
+        setRoot(new SpanList());
+    }
+
+    @SuppressWarnings("unchecked")
+    public SpanTree(SpanTree otherToCopy) {
+        name = otherToCopy.name;
+        setRoot(copySpan(otherToCopy.root));
+        List<Annotation> annotationsToCopy = new ArrayList<Annotation>(otherToCopy.getAnnotations());
+        List<Annotation> newAnnotations = new ArrayList<Annotation>(annotationsToCopy.size());
+
+        for (Annotation otherAnnotationToCopy : annotationsToCopy) {
+            newAnnotations.add(new Annotation(otherAnnotationToCopy));
+        }
+
+        IdentityHashMap<SpanNode, Integer> originalSpanNodes = getSpanNodes(otherToCopy);
+        List<SpanNode> copySpanNodes = getSpanNodes();
+
+        for (int i = 0; i < annotationsToCopy.size(); i++) {
+            Annotation originalAnnotation = annotationsToCopy.get(i);
+            if (!originalAnnotation.isSpanNodeValid()) {  //returns false also if spanNode is null!
+                continue;
+            }
+            Integer indexOfOriginalSpanNode = originalSpanNodes.get(originalAnnotation.getSpanNode());
+            if (indexOfOriginalSpanNode == null) {
+                throw new IllegalStateException("Could not clone tree, SpanNode of " + originalAnnotation + " not found.");
+            }
+            newAnnotations.get(i).setSpanNode(copySpanNodes.get(indexOfOriginalSpanNode));
+        }
+
+        IdentityHashMap<Annotation, Integer> originalAnnotations = getAnnotations(annotationsToCopy);
+
+        for (Annotation a : newAnnotations) {
+            if (!a.hasFieldValue()) {
+                continue;
+            }
+            setCorrectAnnotationReference(a.getFieldValue(), originalAnnotations, newAnnotations);
+        }
+
+        for (Annotation a : newAnnotations) {
+            annotate(a);
+        }
+        for (IndexKey key : otherToCopy.getCurrentIndexes()) {
+            createIndex(key);
+        }
+    }
+
+    private void setCorrectAnnotationReference(FieldValue value, IdentityHashMap<Annotation, Integer> originalAnnotations, List<Annotation> newAnnotations) {
+        if (value == null) {
+            return;
+        }
+
+        if (value.getDataType() instanceof AnnotationReferenceDataType) {
+            AnnotationReference ref = (AnnotationReference) value;
+            if (ref.getReference() == null) {
+                return;
+            }
+            Integer referenceIndex = originalAnnotations.get(ref.getReference());
+            if (referenceIndex == null) {
+                throw new IllegalStateException("Cannot find Annotation pointed to by " + ref);
+            }
+            try {
+                Annotation newReference = newAnnotations.get(referenceIndex);
+                ref.setReference(newReference);
+            } catch (IndexOutOfBoundsException ioobe) {
+                throw new IllegalStateException("Cannot find Annotation pointed to by " + ref, ioobe);
+            }
+        } else if (value.getDataType() instanceof StructuredDataType) {
+            setCorrectAnnotationReference((StructuredFieldValue) value, originalAnnotations, newAnnotations);
+        } else if (value.getDataType() instanceof CollectionDataType) {
+            setCorrectAnnotationReference((CollectionFieldValue) value, originalAnnotations, newAnnotations);
+        } else if (value.getDataType() instanceof MapDataType) {
+            setCorrectAnnotationReference((MapFieldValue) value, originalAnnotations, newAnnotations);
+        }
+    }
+
+    private void setCorrectAnnotationReference(StructuredFieldValue struct, IdentityHashMap<Annotation, Integer> originalAnnotations, List<Annotation> newAnnotations) {
+        for (Field f : struct.getDataType().getFields()) {
+            setCorrectAnnotationReference(struct.getFieldValue(f), originalAnnotations, newAnnotations);
+        }
+    }
+
+    private void setCorrectAnnotationReference(CollectionFieldValue collection, IdentityHashMap<Annotation, Integer> originalAnnotations, List<Annotation> newAnnotations) {
+        Iterator it = collection.fieldValueIterator();
+        while (it.hasNext()) {
+            setCorrectAnnotationReference((FieldValue) it.next(), originalAnnotations, newAnnotations);
+        }
+    }
+
+    private void setCorrectAnnotationReference(MapFieldValue map, IdentityHashMap<Annotation, Integer> originalAnnotations, List<Annotation> newAnnotations) {
+        for (Object o : map.values()) {
+            setCorrectAnnotationReference((FieldValue) o, originalAnnotations, newAnnotations);
+        }
+    }
+
+    private IdentityHashMap<Annotation, Integer> getAnnotations(List<Annotation> annotationsToCopy) {
+        IdentityHashMap<Annotation, Integer> map = new IdentityHashMap<Annotation, Integer>();
+        for (int i = 0; i < annotationsToCopy.size(); i++) {
+            map.put(annotationsToCopy.get(i), i);
+        }
+        return map;
+    }
+
+
+    private List<SpanNode> getSpanNodes() {
+        ArrayList<SpanNode> nodes = new ArrayList<SpanNode>();
+        nodes.add(root);
+        Iterator<SpanNode> it = root.childIteratorRecursive();
+        while (it.hasNext()) {
+            nodes.add(it.next());
+        }
+        return nodes;
+    }
+
+    private static IdentityHashMap<SpanNode, Integer> getSpanNodes(SpanTree otherToCopy) {
+        IdentityHashMap<SpanNode, Integer> map = new IdentityHashMap<SpanNode, Integer>();
+        int spanNodeCounter = 0;
+        map.put(otherToCopy.getRoot(), spanNodeCounter++);
+        Iterator<SpanNode> it = otherToCopy.getRoot().childIteratorRecursive();
+        while (it.hasNext()) {
+            map.put(it.next(), spanNodeCounter++);
+        }
+        return map;
+    }
+
+    private SpanNode copySpan(SpanNode spanTree) {
+        if (spanTree instanceof Span) {
+            return new Span((Span) spanTree);
+        } else if (spanTree instanceof AlternateSpanList) {
+            return new AlternateSpanList((AlternateSpanList) spanTree);
+        } else if (spanTree instanceof SpanList) {
+            return new SpanList((SpanList) spanTree);
+        } else if (spanTree instanceof DummySpanNode) {
+            return spanTree;  //shouldn't really happen
+        } else {
+            throw new IllegalStateException("Cannot create copy of " + spanTree + " with class "
+            + ((spanTree == null) ? "null" : spanTree.getClass()));
+        }
+    }
+
+    /**
+     * WARNING!&nbsp;Only to be used by deserializers!&nbsp;Sets the name of this SpanTree instance.
+     *
+     * @param name the name to set for this SpanTree instance.
+     */
+    public void setName(String name) {
+        this.name = name;
+    }
+
+    /**
+     * WARNING!&nbsp;Only to be used by deserializers!&nbsp;Sets the root of this SpanTree instance.
+     *
+     * @param root the root to set for this SpanTree instance.
+     */
+    public void setRoot(SpanNode root) {
+        if (!root.isValid()) {
+            throw new IllegalStateException("Cannot use invalid node " + root + " as root node.");
+        }
+        if (root.getParent() != null) {
+            if (root.getParent() != this) {
+                throw new IllegalStateException(root + " is already a child of " + root.getParent() + ", cannot be root of " + this);
+            }
+        }
+        this.root = root;
+        root.setParent(this);
+    }
+
+    /**
+     * Returns the name of this span tree.
+     * @return the name of this span tree.
+     */
+    public String getName() {
+        return name;
+    }
+
+    /**
+     * Returns the root node of this span tree.
+     * @return the root node of this span tree.
+     */
+    public SpanNode getRoot() {
+        return root;
+    }
+
+    /**
+     * Convenience shorthand for <code>(SpanList)getRoot()</code>.
+     * This must of course only be used when it is known that the root in this tree actually is a SpanList.
+     */
+    public SpanList spanList() {
+        return (SpanList)root;
+    }
+
+    /**
+     * Ensures consistency of the tree in case SpanNodes have been removed, and there are still
+     * Annotations pointing to them. This method has a maximum upper bound of O(3nm), where n is the
+     * total number of Annotations, and m is the number of SpanNodes that had been removed from the tree.
+     * The lower bound is Omega(n), if no SpanNodes had been removed from the tree.
+     */
+    @SuppressWarnings("unchecked")
+    public void cleanup() {
+        Map<Annotation, Annotation> removedAnnotations = removeAnnotationsThatPointToInvalidSpanNodes();
+
+        //here:
+        //iterate through all annotations;
+        //if any of those have ONLY an annotationreference as its value,
+        //    - null reference
+        //    - remove value from annotation
+        //    - remove annotation and add it to removedAnnotations map
+        if (!removedAnnotations.isEmpty()) {
+            Iterator<Annotation> annotationIt = iterator();
+            while (annotationIt.hasNext()) {
+                Annotation a = annotationIt.next();
+                if (!a.hasFieldValue()) {
+                    continue;
+                }
+                FieldValue value = a.getFieldValue();
+
+                if (value instanceof AnnotationReference) {
+                    //the annotation "a" has a reference
+                    AnnotationReference ref = (AnnotationReference) value;
+                    if (removedAnnotations.get(ref.getReference()) != null) {
+                        //this reference refers to a dead annotation
+                        ref.setReference(null);
+                        a.setFieldValue(null);
+                        if (!a.isSpanNodeValid()) {
+                            //this annotation has no span node, delete it
+                            annotationIt.remove();
+                            removedAnnotations.put(a, a);
+                        }
+                    }
+                }
+            }
+        }
+
+        //there may still be references to removed annotations,
+        //inside maps, weighted sets, structs, etc.
+        //if any of those have such references,
+        //   - null reference
+        //   - remove annotationref from struct, map, etc.
+        //   - apart from this, keep struct, map etc. and annotation
+        if (!removedAnnotations.isEmpty()) {
+            for (Annotation a : this) {
+                if (!a.hasFieldValue()) {
+                    continue;
+                }
+                removeObsoleteReferencesFromFieldValue(a.getFieldValue(), removedAnnotations, true);
+            }
+        }
+        //was any annotations removed from the global list? do we still have references to those annotations
+        //that have been removed? if so, remove the references
+        removeAnnotationReferencesThatPointToRemovedAnnotations();
+    }
+
+    private boolean removeObsoleteReferencesFromFieldValue(FieldValue value, Map<Annotation, Annotation> selectedAnnotations, boolean removeIfPresent) {
+        if (value == null) {
+            return false;
+        }
+
+        if (value.getDataType() instanceof AnnotationReferenceDataType) {
+            AnnotationReference ref = (AnnotationReference) value;
+            if (removeIfPresent) {
+                if (selectedAnnotations.containsValue(ref.getReference())) {
+                    //this reference refers to a dead annotation
+                    ref.setReference(null);
+                    return true;
+                }
+            } else {
+                if (!selectedAnnotations.containsValue(ref.getReference())) {
+                    //this reference refers to a dead annotation
+                    ref.setReference(null);
+                    return true;
+                }
+            }
+        } else if (value.getDataType() instanceof StructuredDataType) {
+            removeObsoleteReferencesFromStructuredType((StructuredFieldValue) value, selectedAnnotations, removeIfPresent);
+        } else if (value.getDataType() instanceof CollectionDataType) {
+            removeObsoleteReferencesFromCollectionType((CollectionFieldValue) value, selectedAnnotations, removeIfPresent);
+        } else if (value.getDataType() instanceof MapDataType) {
+            removeObsoleteReferencesFromMapType((MapFieldValue) value, selectedAnnotations, removeIfPresent);
+        }
+        return false;
+    }
+
+    private boolean removeObsoleteReferencesFromStructuredType(StructuredFieldValue struct, Map<Annotation, Annotation> selectedAnnotations, boolean removeIfPresent) {
+        for (Field f : struct.getDataType().getFields()) {
+            FieldValue fValue = struct.getFieldValue(f);
+            if (removeObsoleteReferencesFromFieldValue(fValue, selectedAnnotations, removeIfPresent)) {
+                struct.removeFieldValue(f);
+            }
+        }
+        return false;
+    }
+
+    private boolean removeObsoleteReferencesFromCollectionType(CollectionFieldValue collection, Map<Annotation, Annotation> selectedAnnotations, boolean removeIfPresent) {
+        Iterator it = collection.fieldValueIterator();
+        while (it.hasNext()) {
+            FieldValue fValue = (FieldValue) it.next();
+            if (removeObsoleteReferencesFromFieldValue(fValue, selectedAnnotations, removeIfPresent)) {
+                it.remove();
+            }
+        }
+        return false;
+    }
+
+    private boolean removeObsoleteReferencesFromMapType(MapFieldValue map, Map<Annotation, Annotation> selectedAnnotations, boolean removeIfPresent) {
+        Iterator valueIt = map.values().iterator();
+        while (valueIt.hasNext()) {
+            FieldValue fValue = (FieldValue) valueIt.next();
+            if (removeObsoleteReferencesFromFieldValue(fValue, selectedAnnotations, removeIfPresent)) {
+                valueIt.remove();
+            }
+        }
+        return false;
+    }
+
+    @SuppressWarnings("unchecked")
+    private Map<Annotation, Annotation> removeAnnotationsThatPointToInvalidSpanNodes() {
+        Map<Annotation, Annotation> removedAnnotations = new IdentityHashMap<Annotation, Annotation>();
+
+        Iterator<Annotation> annotationIt = iterator();
+        while (annotationIt.hasNext()) {
+            Annotation a = annotationIt.next();
+            if (a.hasSpanNode() && !a.isSpanNodeValid()) {
+                a.setSpanNode(null);
+                a.setFieldValue(null);
+                removedAnnotations.put(a, a);
+                annotationIt.remove();
+            }
+        }
+        return removedAnnotations;
+    }
+
+    private boolean hasAnyFieldValues() {
+        for (Annotation a : this) {
+            if (a.hasFieldValue()) {
+                return true;
+            }
+        }
+        return false;
+    }
+    private void removeAnnotationReferencesThatPointToRemovedAnnotations() {
+        if (hasAnyFieldValues()) {
+            Map<Annotation, Annotation> annotationsStillPresent = new IdentityHashMap<Annotation, Annotation>();
+            for (Annotation a : this) {
+                annotationsStillPresent.put(a, a);
+            }
+            for (Annotation a : this) {
+                if (!a.hasFieldValue()) {
+                    continue;
+                }
+                //do we have any references to annotations that are NOT in this global list??
+                removeObsoleteReferencesFromFieldValue(a.getFieldValue(), annotationsStillPresent, false);
+            }
+        }
+    }
+
+    private void annotateInternal(SpanNode node, Annotation annotation) {
+        annotations.annotate(annotation);
+    }
+
+    @SuppressWarnings("unchecked")
+    private Collection<Annotation> getAnnotations() {
+        return annotations.annotations();
+    }
+
+    /**
+     * Adds an Annotation to the internal list of annotations for this SpanTree.&nbsp;Use this when
+     * adding an Annotation that uses an AnnotationReference, and does not annotate a SpanNode.
+     *
+     * @param a the Annotation to add
+     * @return this, for chaining
+     * @see com.yahoo.document.annotation.Annotation
+     * @see com.yahoo.document.annotation.AnnotationReference
+     * @see com.yahoo.document.annotation.AnnotationReferenceDataType
+     */
+    public SpanTree annotate(Annotation a) {
+        if (a.getSpanNode() == null) {
+            annotateInternal(DummySpanNode.INSTANCE, a);
+        } else {
+            annotateInternal(a.getSpanNode(), a);
+        }
+        return this;
+    }
+
+    /**
+     * Adds an Annotation to the internal list of annotations for this SpanTree.&nbsp;Use this when
+     * adding an Annotation that shall annotate a SpanNode. Upon return, Annotation.getSpanNode()
+     * returns the given node.
+     *
+     * @param node the node to annotate
+     * @param annotation the Annotation to add
+     * @return this, for chaining
+     * @see com.yahoo.document.annotation.Annotation
+     */
+    public SpanTree annotate(SpanNode node, Annotation annotation) {
+        annotation.setSpanNode(node);
+        return annotate(annotation);
+    }
+
+    /**
+     * Adds an Annotation to the internal list of annotations for this SpanTree.&nbsp;Use this when
+     * adding an Annotation that shall annotate a SpanNode. Upon return, Annotation.getSpanNode()
+     * returns the given node. This one is unchecked and assumes that the SpanNode is valid and has
+     * already been attached to the Annotation.
+     *
+     * @param node the node to annotate
+     * @param annotation the Annotation to add
+     * @return this, for chaining
+     * @see com.yahoo.document.annotation.Annotation
+     */
+    public final SpanTree annotateFast(SpanNode node, Annotation annotation) {
+        annotateInternal(node, annotation);
+        return this;
+    }
+
+    /**
+     * Adds an Annotation.
+     * Convenience shorthand for <code>annotate(node,new Annotation(type,value)</code>
+     *
+     * @param node the node to annotate
+     * @param type the type of the Annotation to add
+     * @param value the value of the Annotation to add
+     * @return this, for chaining
+     * @see com.yahoo.document.annotation.Annotation
+     */
+    public SpanTree annotate(SpanNode node, AnnotationType type,FieldValue value) {
+        return annotate(node, new Annotation(type, value));
+    }
+
+    /**
+     * Creates an Annotation based on the given AnnotationType, and adds it to the internal list of
+     * annotations for this SpanTree (convenience method).&nbsp;Use this when
+     * adding an Annotation (that does not have a FieldValue) that shall annotate a SpanNode.
+     * Upon return, Annotation.getSpanNode()
+     * returns the given node.
+     *
+     * @param node the node to annotate
+     * @param type the AnnotationType to create an Annotation from
+     * @return this, for chaining
+     * @see com.yahoo.document.annotation.Annotation
+     * @see com.yahoo.document.annotation.AnnotationType
+     */
+    public SpanTree annotate(SpanNode node, AnnotationType type) {
+        Annotation a = new Annotation(type);
+        return annotate(node, a);
+    }
+
+    /**
+     * Removes an Annotation from the internal list of annotations.
+     *
+     * @param a the annotation to remove
+     * @return true if the Annotation was successfully removed, false otherwise
+     */
+    public boolean remove(Annotation a) {
+        return getAnnotations().remove(a);
+    }
+
+    /**
+     * Returns the total number of annotations in the tree.
+     *
+     * @return the total number of annotations in the tree.
+     */
+    public int numAnnotations() {
+        return annotations.annotations().size();
+    }
+
+    /**
+     * Clears all Annotations for a given SpanNode.
+     *
+     * @param node the SpanNode to clear all Annotations for.
+     */
+    public void clearAnnotations(SpanNode node) {
+        Iterator<Annotation> annIt = iterator(node);
+        while (annIt.hasNext()) {
+            annIt.next();
+            annIt.remove();
+        }
+    }
+
+    /**
+     * Clears all Annotations for a given SpanNode and its child nodes.
+     *
+     * @param node the SpanNode to clear all Annotations for.
+     */
+    public void clearAnnotationsRecursive(SpanNode node) {
+        Iterator<Annotation> annIt = iteratorRecursive(node);
+        while (annIt.hasNext()) {
+            annIt.next();
+            annIt.remove();
+        }
+    }
+
+    /**
+     * Returns an Iterator over all annotations in this tree.&nbsp;Note that the iteration order is non-deterministic.
+     * @return an Iterator over all annotations in this tree.
+     */
+    @SuppressWarnings("unchecked")
+    public Iterator<Annotation> iterator() {
+        return annotations.annotations().iterator();
+    }
+
+    /**
+     * Returns an Iterator over all annotations that annotate the given node.
+     *
+     * @param node the node to return annotations for.
+     * @return an Iterator over all annotations that annotate the given node.
+     */
+    @SuppressWarnings("unchecked")
+    public Iterator<Annotation> iterator(SpanNode node) {
+        return annotations.iterator(node);
+    }
+
+    /**
+     * Returns a recursive Iterator over all annotations that annotate the given node and its subnodes.
+     *
+     * @param node the node to recursively return annotations for.
+     * @return a recursive Iterator over all annotations that annotate the given node and its subnodes.
+     */
+    @SuppressWarnings("unchecked")
+    public Iterator<Annotation> iteratorRecursive(SpanNode node) {
+        return annotations.iteratorRecursive(node);
+    }
+
+    /**
+     * Returns itself.&nbsp;Needed for this class to be able to be a parent of SpanNodes.
+     *
+     * @return this SpanTree instance.
+     */
+    @Override
+    public SpanTree getSpanTree() {
+        return this;
+    }
+
+    /**
+     * Sets the StringFieldValue that this SpanTree belongs to.&nbsp;This is called by
+     * {@link StringFieldValue#setSpanTree(SpanTree)} and there is no need for the user to call this
+     * except in unit tests.
+     *
+     * @param stringFieldValue the StringFieldValue that this SpanTree should belong to (might be null to clear the current value)
+     */
+    public void setStringFieldValue(StringFieldValue stringFieldValue) {
+        this.stringFieldValue = stringFieldValue;
+    }
+
+    /**
+     * Returns the StringFieldValue that this SpanTree belongs to.
+     *
+     * @return the StringFieldValue that this SpanTree belongs to, if any, otherwise null.
+     */
+    @Override
+    public StringFieldValue getStringFieldValue() {
+        return stringFieldValue;
+    }
+
+    public void createIndex(IndexKey key) {
+        if (key == IndexKey.SPAN_NODE && annotations instanceof ListAnnotationContainer) {
+            AnnotationContainer tmpAnnotations = new SpanNode2AnnotationContainer();
+            tmpAnnotations.annotateAll(annotations.annotations());
+            annotations = tmpAnnotations;
+        } else if (key == IndexKey.ANNOTATION_TYPE && annotations instanceof ListAnnotationContainer) {
+            AnnotationContainer tmpAnnotations = new AnnotationType2AnnotationContainer();
+            tmpAnnotations.annotateAll(annotations.annotations());
+            annotations = tmpAnnotations;
+        } else {
+            throw new IllegalArgumentException("Multiple indexes not yet supported. Use clearIndex() or clearIndexes() first.");
+        }
+    }
+
+    public void clearIndex(IndexKey key) {
+        if (key == IndexKey.SPAN_NODE && annotations instanceof SpanNode2AnnotationContainer) {
+            clearIndex();
+        } else if (key == IndexKey.ANNOTATION_TYPE && annotations instanceof AnnotationType2AnnotationContainer) {
+            clearIndex();
+        }
+    }
+
+    public void clearIndexes() {
+        if (!(annotations instanceof ListAnnotationContainer)) {
+            clearIndex();
+        }
+    }
+
+    private void clearIndex() {
+        AnnotationContainer tmpAnnotations = new ListAnnotationContainer();
+        tmpAnnotations.annotateAll(annotations.annotations());
+        annotations = tmpAnnotations;
+    }
+
+    public Collection<IndexKey> getCurrentIndexes() {
+        if (annotations instanceof AnnotationType2AnnotationContainer)
+            return ImmutableList.of(IndexKey.ANNOTATION_TYPE);
+        if (annotations instanceof SpanNode2AnnotationContainer)
+            return ImmutableList.of(IndexKey.SPAN_NODE);
+        return ImmutableList.of();
+    }
+
+    @Override
+    public String toString() {
+        return "SpanTree '" + name + "'";
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (!(o instanceof SpanTree)) return false;
+
+        SpanTree tree = (SpanTree) o;
+        if (!annotationsEquals(tree)) return false;
+        if (!name.equals(tree.name)) return false;
+        if (!root.equals(tree.root)) return false;
+
+        return true;
+    }
+
+    @SuppressWarnings("unchecked")
+    private boolean annotationsEquals(SpanTree tree) {
+        List<Annotation> annotationCollection = new LinkedList<Annotation>(getAnnotations());
+        List<Annotation> otherAnnotations = new LinkedList<Annotation>(tree.getAnnotations());
+
+        return annotationCollection.size() == otherAnnotations.size() && CollectionUtils.isEqualCollection(annotationCollection, otherAnnotations);
+    }
+
+    @Override
+    public int hashCode() {
+        int result = name.hashCode();
+        result = 31 * result + root.hashCode();
+        result = 31 * result + annotations.hashCode();
+        return result;
+    }
+
+    @Override
+    public int compareTo(SpanTree spanTree) {
+        int comp = name.compareTo(spanTree.name);
+        if (comp != 0) {
+            comp = root.compareTo(spanTree.root);
+        }
+        return comp;
+    }
+
+    public enum IndexKey {
+        SPAN_NODE,
+        ANNOTATION_TYPE
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/annotation/SpanTrees.java b/document/src/main/java/com/yahoo/document/annotation/SpanTrees.java
new file mode 100644
index 00000000000..449e803a248
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/annotation/SpanTrees.java
@@ -0,0 +1,18 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+/**
+ * This is a container for all {@link SpanTree}s constants used by built-in Vespa features.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+@SuppressWarnings({ "UnusedDeclaration" })
+// TODO: Remove. This is the wrong place.
+public final class SpanTrees {
+
+    private SpanTrees() {
+        // unreachable
+    }
+
+    public static final String LINGUISTICS = "linguistics";
+}
diff --git a/document/src/main/java/com/yahoo/document/annotation/package-info.java b/document/src/main/java/com/yahoo/document/annotation/package-info.java
new file mode 100644
index 00000000000..235252cd030
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/annotation/package-info.java
@@ -0,0 +1,11 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * Provides classes and interfaces for creating trees of spans over string
+ * values in Vespa documents, and annotating these spans.
+ */
+@ExportPackage
+@PublicApi
+package com.yahoo.document.annotation;
+
+import com.yahoo.api.annotations.PublicApi;
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/document/src/main/java/com/yahoo/document/config/package-info.java b/document/src/main/java/com/yahoo/document/config/package-info.java
new file mode 100644
index 00000000000..d28ae619a10
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/config/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.document.config;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/document/src/main/java/com/yahoo/document/datatypes/Array.java b/document/src/main/java/com/yahoo/document/datatypes/Array.java
new file mode 100644
index 00000000000..66cc472de69
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/datatypes/Array.java
@@ -0,0 +1,543 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.datatypes;
+
+import com.yahoo.collections.CollectionComparator;
+import com.yahoo.document.ArrayDataType;
+import com.yahoo.document.DataType;
+import com.yahoo.document.Field;
+import com.yahoo.document.FieldPath;
+import com.yahoo.document.serialization.FieldReader;
+import com.yahoo.document.serialization.FieldWriter;
+import com.yahoo.document.serialization.XmlSerializationHelper;
+import com.yahoo.document.serialization.XmlStream;
+
+import java.util.*;
+
+/**
+ * FieldValue which encapsulates a Array value
+ *
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public final class Array<T extends FieldValue> extends CollectionFieldValue<T> implements List<T> {
+
+    private List<T> values;
+
+    public Array(DataType type) {
+        this(type, 1);
+    }
+
+    public Array(DataType type, int initialCapacity) {
+        super((ArrayDataType) type);
+        this.values = new ArrayList<>(initialCapacity);
+    }
+
+    public Array(DataType type, List<T> values) {
+        this(type);
+        for (T v : values) {
+            if (!((ArrayDataType)type).getNestedType().isValueCompatible(v)) {
+                throw new IllegalArgumentException("FieldValue " + v +
+                        " is not compatible with " + type + ".");
+            }
+        }
+        this.values.addAll(values);
+    }
+
+    @Override
+    public ArrayDataType getDataType() {
+        return (ArrayDataType) super.getDataType();
+    }
+
+    @Override
+    public Iterator<T> fieldValueIterator() {
+        return values.iterator();
+    }
+
+    @Override
+    public Array<T> clone() {
+        Array<T> array = (Array<T>) super.clone();
+        array.values = new ArrayList<>(values.size());
+        for (T fval : values) {
+            array.values.add((T) fval.clone());
+        }
+        return array;
+    }
+
+    @Override
+    public void clear() {
+        values.clear();
+    }
+
+    @Override
+    public void assign(Object o) {
+        if (!checkAssign(o)) {
+            return;
+        }
+
+        if (o instanceof Array) {
+            if (o == this) return;
+            Array a = (Array) o;
+            values.clear();
+            addAll(a.values);
+        } else if (o instanceof List) {
+            values = new ListWrapper<T>((List) o);
+        } else {
+            throw new IllegalArgumentException("Class " + o.getClass() + " not applicable to an " + this.getClass() + " instance.");
+        }
+    }
+
+    @Override
+    public Object getWrappedValue() {
+        if (values instanceof ListWrapper) {
+            return ((ListWrapper) values).myvalues;
+        }
+        List tmpWrappedList = new ArrayList();
+        for (T value : values) {
+            tmpWrappedList.add(value.getWrappedValue());
+        }
+        return tmpWrappedList;
+    }
+
+    public List<T> getValues() {
+        return values;
+    }
+
+    public FieldValue getFieldValue(int index) {
+        return values.get(index);
+    }
+
+    @Override
+    public void printXml(XmlStream xml) {
+        XmlSerializationHelper.printArrayXml(this, xml);
+    }
+
+    @Override
+    public String toString() {
+        return values.toString();
+    }
+
+    @Override
+    public int hashCode() {
+        int hashCode = super.hashCode();
+        for (FieldValue val : values) {
+            hashCode ^= val.hashCode();
+        }
+        return hashCode;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (!(o instanceof Array)) return false;
+        if (!super.equals(o)) return false;
+        Array a = (Array) o;
+        // Compare independent of container used.
+        Iterator it1 = values.iterator();
+        Iterator it2 = a.values.iterator();
+        while (it1.hasNext() && it2.hasNext()) {
+            if (!it1.next().equals(it2.next())) return false;
+        }
+        return !(it1.hasNext() || it2.hasNext());
+    }
+
+    // List implementation
+
+    public void add(int index, T o) {
+        verifyElementCompatibility(o);
+        values.add(index, o);
+    }
+
+    public boolean remove(Object o) {
+        return values.remove(o);
+    }
+
+    public boolean add(T o) {
+        verifyElementCompatibility(o);
+        return values.add(o);
+    }
+
+    @Override
+    public boolean contains(Object o) {
+        return values.contains(o);
+    }
+
+    @Override
+    public boolean isEmpty() {
+        return super.isEmpty(values);
+    }
+
+    @Override
+    public Iterator<T> iterator() {
+        return values.iterator();
+    }
+
+    @Override
+    public boolean removeValue(FieldValue o) {
+        return super.removeValue(o, values);
+    }
+
+    @Override
+    public int size() {
+        return values.size();
+    }
+
+    public boolean addAll(Collection<? extends T> c) {
+        for (T t : c) {
+            verifyElementCompatibility(t);
+        }
+        return values.addAll(c);
+    }
+
+    public boolean containsAll(Collection<?> c) {
+        return values.containsAll(c);
+    }
+
+    public Object[] toArray() {
+        return values.toArray();
+    }
+
+    @SuppressWarnings({"unchecked"})
+    public <T> T[] toArray(T[] a) {
+        return values.toArray(a);
+    }
+
+    public boolean addAll(int index, Collection<? extends T> c) {
+        for (T t : c) {
+            verifyElementCompatibility(t);
+        }
+        return values.addAll(index, c);
+    }
+
+    @SuppressWarnings("deprecation")
+    public boolean retainAll(Collection<?> c) {
+        return values.retainAll(c);
+    }
+
+    @SuppressWarnings("deprecation")
+    public boolean removeAll(Collection<?> c) {
+        return values.removeAll(c);
+    }
+
+    public T get(int index) {
+        return values.get(index);
+    }
+
+    @SuppressWarnings("deprecation")
+    public int indexOf(Object o) {
+        return values.indexOf(o);
+    }
+
+    @SuppressWarnings("deprecation")
+    public int lastIndexOf(Object o) {
+        return values.lastIndexOf(o);
+    }
+
+    public ListIterator<T> listIterator() {
+        return values.listIterator();
+    }
+
+    public ListIterator<T> listIterator(final int index) {
+        return values.listIterator(index);
+    }
+
+    public T remove(int index) {
+        return values.remove(index);
+    }
+
+    @SuppressWarnings("deprecation")
+    public T set(int index, T o) {
+        verifyElementCompatibility(o);
+        T fval = values.set(index, o);
+        return fval;
+    }
+
+    public List<T> subList(int fromIndex, int toIndex) {
+        return values.subList(fromIndex, toIndex);
+    }
+
+    FieldPathIteratorHandler.ModificationStatus iterateSubset(int startPos, int endPos, FieldPath fieldPath, String variable, int nextPos, FieldPathIteratorHandler handler) {
+        FieldPathIteratorHandler.ModificationStatus retVal = FieldPathIteratorHandler.ModificationStatus.NOT_MODIFIED;
+
+        LinkedList<Integer> indicesToRemove = new LinkedList<Integer>();
+
+        for (int i = startPos; i <= endPos && i < values.size(); i++) {
+            if (variable != null) {
+                handler.getVariables().put(variable, new FieldPathIteratorHandler.IndexValue(i));
+            }
+
+            FieldValue fv = values.get(i);
+            FieldPathIteratorHandler.ModificationStatus status = fv.iterateNested(fieldPath, nextPos, handler);
+
+            if (status == FieldPathIteratorHandler.ModificationStatus.REMOVED) {
+                indicesToRemove.addFirst(i);
+                retVal = FieldPathIteratorHandler.ModificationStatus.MODIFIED;
+            } else if (status == FieldPathIteratorHandler.ModificationStatus.MODIFIED) {
+                retVal = status;
+            }
+        }
+
+        if (variable != null) {
+            handler.getVariables().remove(variable);
+        }
+
+        for (Integer idx : indicesToRemove) {
+            values.remove(idx.intValue());
+        }
+        return retVal;
+    }
+
+    @Override
+    FieldPathIteratorHandler.ModificationStatus iterateNested(FieldPath fieldPath, int pos, FieldPathIteratorHandler handler) {
+        if (pos < fieldPath.size()) {
+            switch (fieldPath.get(pos).getType()) {
+                case ARRAY_INDEX:
+                    return iterateSubset(fieldPath.get(pos).getLookupIndex(), fieldPath.get(pos).getLookupIndex(), fieldPath, null, pos + 1, handler);
+                case VARIABLE: {
+                    FieldPathIteratorHandler.IndexValue val = handler.getVariables().get(fieldPath.get(pos).getVariableName());
+                    if (val != null) {
+                        int idx = val.getIndex();
+
+                        if (idx == -1) {
+                            throw new IllegalArgumentException("Mismatch between variables - trying to iterate through map and array with the same variable.");
+                        }
+
+                        if (idx < values.size()) {
+                            return iterateSubset(idx, idx, fieldPath, null, pos + 1, handler);
+                        }
+                    } else {
+                        return iterateSubset(0, values.size() - 1, fieldPath, fieldPath.get(pos).getVariableName(), pos + 1, handler);
+                    }
+                    break;
+                }
+                default:
+            }
+            return iterateSubset(0, values.size() - 1, fieldPath, null, pos, handler);
+        } else {
+            FieldPathIteratorHandler.ModificationStatus status = handler.modify(this);
+
+            if (status == FieldPathIteratorHandler.ModificationStatus.REMOVED) {
+                return status;
+            }
+
+            if (handler.onComplex(this)) {
+                if (iterateSubset(0, values.size() - 1, fieldPath, null, pos, handler) != FieldPathIteratorHandler.ModificationStatus.NOT_MODIFIED) {
+                    status = FieldPathIteratorHandler.ModificationStatus.MODIFIED;
+                }
+            }
+
+            return status;
+        }
+    }
+
+    /**
+     * This wrapper class is used to wrap a list that isn't a list of field
+     * values. This is done, as to not alter behaviour from previous state,
+     * where people could add whatever list to a document, and then keep adding
+     * stuff to the list afterwards.
+     *
+     * <p>
+     * TODO: Remove this class and only allow instance of Array to be added.
+     */
+    class ListWrapper<E> implements List<E>, RandomAccess {
+        private final List myvalues;
+
+        private Object unwrap(Object o) {
+            return (o instanceof FieldValue ? ((FieldValue) o).getWrappedValue() : o);
+        }
+
+        public ListWrapper(List wrapped) {
+            myvalues = wrapped;
+        }
+
+        public int size() {
+            return myvalues.size();
+        }
+
+        public boolean isEmpty() {
+            return myvalues.isEmpty();
+        }
+
+        public boolean contains(Object o) {
+            return myvalues.contains(unwrap(o));
+        }
+
+        public Iterator<E> iterator() {
+            return listIterator();
+        }
+
+        public Object[] toArray() {
+            return toArray(new Object[myvalues.size()]);
+        }
+
+        // It's supposed to blow up if given invalid types
+        @SuppressWarnings({ "hiding", "unchecked" })
+        public <T> T[] toArray(T[] a) {
+            final Class<?> componentType = a.getClass().getComponentType();
+            T[] out = (T[]) java.lang.reflect.Array.newInstance(componentType, myvalues.size());
+
+            Arrays.setAll(out, (i) -> (T) createFieldValue(myvalues.get(i)));
+            return out;
+        }
+
+        public boolean add(E o) {
+            return myvalues.add(unwrap(o));
+        }
+
+        public boolean remove(Object o) {
+            return myvalues.remove(unwrap(o));
+        }
+
+        public boolean containsAll(Collection<?> c) {
+            for (Object o : c) {
+                if (!myvalues.contains(unwrap(o))) return false;
+            }
+            return true;
+        }
+
+        public boolean addAll(Collection<? extends E> c) {
+            boolean result = false;
+            for (Object o : c) {
+                result |= myvalues.add(unwrap(o));
+            }
+            return result;
+        }
+
+        public boolean addAll(int index, Collection<? extends E> c) {
+            for (Object o : c) {
+                myvalues.add(index++, unwrap(o));
+            }
+            return true;
+        }
+
+        public boolean removeAll(Collection<?> c) {
+            boolean result = false;
+            for (Object o : c) {
+                result |= myvalues.remove(unwrap(o));
+            }
+            return result;
+        }
+
+        public boolean retainAll(Collection<?> c) {
+            throw new UnsupportedOperationException("retainAll() not implemented for this type");
+        }
+
+        public void clear() {
+            myvalues.clear();
+        }
+
+        public E get(int index) {
+            Object o = myvalues.get(index);
+            return (E) (o == null ? null : createFieldValue(o));
+        }
+
+        public E set(int index, E element) {
+            Object o = myvalues.set(index, unwrap(element));
+            return (E) (o == null ? null : createFieldValue(o));
+        }
+
+        public void add(int index, E element) {
+            myvalues.add(index, unwrap(element));
+        }
+
+        public E remove(int index) {
+            Object o = myvalues.remove(index);
+            return (E) (o == null ? null : createFieldValue(o));
+        }
+
+        public int indexOf(Object o) {
+            return myvalues.indexOf(unwrap(o));
+        }
+
+        public int lastIndexOf(Object o) {
+            return myvalues.lastIndexOf(unwrap(o));
+        }
+
+        public ListIterator<E> listIterator() {
+            return listIterator(0);
+        }
+
+        public ListIterator<E> listIterator(final int index) {
+            return new ListIterator<E>() {
+                ListIterator it = myvalues.listIterator(index);
+
+                public boolean hasNext() {
+                    return it.hasNext();
+                }
+
+                public E next() {
+                    return (E) createFieldValue(it.next());
+                }
+
+                public boolean hasPrevious() {
+                    return it.hasPrevious();
+                }
+
+                public E previous() {
+                    return (E) createFieldValue(it.previous());
+                }
+
+                public int nextIndex() {
+                    return it.nextIndex();
+                }
+
+                public int previousIndex() {
+                    return it.previousIndex();
+                }
+
+                public void remove() {
+                    it.remove();
+                }
+
+                public void set(E o) {
+                    it.set(unwrap(o));
+                }
+
+                public void add(E o) {
+                    it.add(unwrap(o));
+                }
+            };
+        }
+
+        @SuppressWarnings("deprecation")
+        public List<E> subList(int fromIndex, int toIndex) {
+            return new ListWrapper<E>(myvalues.subList(fromIndex, toIndex));
+        }
+
+        public String toString() {
+            return myvalues.toString();
+        }
+
+        @Override
+        @SuppressWarnings("deprecation, unchecked")
+        public boolean equals(Object o) {
+            return this == o || o instanceof ListWrapper && myvalues.equals(((ListWrapper) o).myvalues);
+        }
+
+        @Override
+        public int hashCode() {
+            return myvalues.hashCode();
+        }
+    }
+
+    @Override
+    public void serialize(Field field, FieldWriter writer) {
+        writer.write(field, this);
+    }
+
+    @Override
+    public void deserialize(Field field, FieldReader reader) {
+        reader.read(field, this);
+    }
+
+    @Override
+    public int compareTo(FieldValue fieldValue) {
+        int comp = super.compareTo(fieldValue);
+
+        if (comp != 0) {
+            return comp;
+        }
+
+        //types are equal, this must be of this type
+        Array otherValue = (Array) fieldValue;
+        return CollectionComparator.compare(values, otherValue.values);
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/datatypes/ByteFieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/ByteFieldValue.java
new file mode 100644
index 00000000000..c6606dd9886
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/datatypes/ByteFieldValue.java
@@ -0,0 +1,154 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.datatypes;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.Field;
+import com.yahoo.document.PrimitiveDataType;
+import com.yahoo.document.serialization.FieldReader;
+import com.yahoo.document.serialization.FieldWriter;
+import com.yahoo.document.serialization.XmlSerializationHelper;
+import com.yahoo.document.serialization.XmlStream;
+import com.yahoo.vespa.objects.Ids;
+
+/**
+ * FieldValue which encapsulates a byte.
+ *
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class ByteFieldValue extends NumericFieldValue {
+    private static class Factory extends PrimitiveDataType.Factory {
+        public FieldValue create() {
+            return new ByteFieldValue();
+        }
+    }
+    public static PrimitiveDataType.Factory getFactory() { return new Factory(); }
+    public static final int classId = registerClass(Ids.document + 10, ByteFieldValue.class);
+    private byte value;
+
+    public ByteFieldValue() {
+        this((byte) 0);
+    }
+
+    public ByteFieldValue(byte value) {
+        this.value = value;
+    }
+
+    public ByteFieldValue(Byte value) {
+        this.value = value;
+    }
+
+    public ByteFieldValue(Integer value) {
+        this.value = (byte) value.intValue();
+    }
+
+    public ByteFieldValue(String s) { value = Byte.parseByte(s); }
+
+    @Override
+    public ByteFieldValue clone() {
+        ByteFieldValue val = (ByteFieldValue) super.clone();
+        val.value = value;
+        return val;
+
+    }
+
+    @Override
+    public Number getNumber() {
+        return value;
+    }
+
+    @Override
+    public void clear() {
+        value = (byte) 0;
+    }
+
+    @Override
+    public void assign(Object o) {
+        if (!checkAssign(o)) {
+            return;
+        }
+        if (o instanceof Number) {
+            value = ((Number) o).byteValue();
+        } else if (o instanceof NumericFieldValue) {
+            value = ((NumericFieldValue) o).getNumber().byteValue();
+        } else if (o instanceof String || o instanceof StringFieldValue) {
+            value = Byte.parseByte(o.toString());
+        } else {
+            throw new IllegalArgumentException("Class " + o.getClass() + " not applicable to an " + this.getClass() + " instance.");
+        }
+    }
+
+    public byte getByte() {
+        return value;
+    }
+
+    @Override
+    public Object getWrappedValue() {
+        return value;
+    }
+
+    @Override
+    public DataType getDataType() {
+        return DataType.BYTE;
+    }
+
+    @Override
+    public void printXml(XmlStream xml) {
+        XmlSerializationHelper.printByteXml(this, xml);
+    }
+
+    @Override
+    public String toString() {
+        return "" + value;
+    }
+
+    @Override
+    public int hashCode() {
+        int result = super.hashCode();
+        result = 31 * result + (int) value;
+        return result;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (!(o instanceof ByteFieldValue)) return false;
+        if (!super.equals(o)) return false;
+
+        ByteFieldValue that = (ByteFieldValue) o;
+        if (value != that.value) return false;
+        return true;
+    }
+
+    @Override
+    public void serialize(Field field, FieldWriter writer) {
+        writer.write(field, this);
+    }
+
+    /* (non-Javadoc)
+      * @see com.yahoo.document.datatypes.FieldValue#deserialize(com.yahoo.document.Field, com.yahoo.document.serialization.FieldReader)
+      */
+
+    @Override
+    public void deserialize(Field field, FieldReader reader) {
+        reader.read(field, this);
+    }
+
+    @Override
+    public int compareTo(FieldValue fieldValue) {
+        int comp = super.compareTo(fieldValue);
+
+        if (comp != 0) {
+            return comp;
+        }
+
+        //types are equal, this must be of this type
+        ByteFieldValue otherValue = (ByteFieldValue) fieldValue;
+        if (value < otherValue.value) {
+            return -1;
+        } else if (value > otherValue.value) {
+            return 1;
+        } else {
+            return 0;
+        }
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/datatypes/CollectionFieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/CollectionFieldValue.java
new file mode 100644
index 00000000000..aeeb4abf632
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/datatypes/CollectionFieldValue.java
@@ -0,0 +1,82 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.datatypes;
+
+import com.yahoo.document.CollectionDataType;
+
+import java.util.Collection;
+import java.util.Iterator;
+
+/**
+ * Date: Apr 16, 2008
+ *
+ * @author <a href="mailto:humbe@yahoo-inc.com">H&aring;kon Humberset</a>
+ */
+public abstract class CollectionFieldValue<T extends FieldValue> extends CompositeFieldValue {
+
+    CollectionFieldValue(CollectionDataType type) {
+        super(type);
+    }
+
+    @Override
+    public CollectionDataType getDataType() {
+        return (CollectionDataType) super.getDataType();
+    }
+
+    /**
+     * Utility function to wrap primitives.
+     *
+     * @see Array.ListWrapper
+     */
+    protected FieldValue createFieldValue(Object o) {
+        if (o instanceof FieldValue) {
+            if (!getDataType().getNestedType().isValueCompatible((FieldValue) o)) {
+                throw new IllegalArgumentException(
+                        "Incompatible data types. Got "
+                                + ((FieldValue)o).getDataType() + ", expected "
+                                + getDataType().getNestedType());
+            }
+            return (FieldValue) o;
+        } else {
+            FieldValue fval = getDataType().getNestedType().createFieldValue();
+            fval.assign(o);
+            return fval;
+        }
+    }
+
+    public void verifyElementCompatibility(T o) {
+        if (!getDataType().getNestedType().isValueCompatible(o)) {
+            throw new IllegalArgumentException(
+                    "Incompatible data types. Got "
+                    + o.getDataType() + ", expected "
+                    + getDataType().getNestedType());
+        }
+    }
+
+    public abstract Iterator<T> fieldValueIterator();
+
+    // Collection implementation
+
+    public abstract boolean add(T value);
+
+    public abstract boolean contains(Object o);
+
+    public abstract boolean isEmpty();
+
+    protected boolean isEmpty(Collection collection) {
+        return collection.isEmpty();
+    }
+
+    public abstract Iterator<T> iterator();
+
+    public abstract boolean removeValue(FieldValue o);
+
+    protected boolean removeValue(FieldValue o, Collection collection) {
+        int removedCount = 0;
+        while (collection.remove(o)) {
+            ++removedCount;
+        }
+        return (removedCount > 0);
+    }
+
+    public abstract int size();
+}
diff --git a/document/src/main/java/com/yahoo/document/datatypes/CompositeFieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/CompositeFieldValue.java
new file mode 100644
index 00000000000..a1f0f81eea7
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/datatypes/CompositeFieldValue.java
@@ -0,0 +1,39 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.datatypes;
+
+import com.yahoo.document.DataType;
+
+public abstract class CompositeFieldValue extends FieldValue {
+    private DataType dataType;
+
+    public CompositeFieldValue(DataType dataType) {
+        this.dataType = dataType;
+    }
+
+    @Override
+    public DataType getDataType() {
+        return dataType;
+    }
+
+    public void setDataType(DataType dataType) {
+        this.dataType = dataType;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (!(o instanceof CompositeFieldValue)) return false;
+        if (!super.equals(o)) return false;
+
+        CompositeFieldValue that = (CompositeFieldValue) o;
+        if (dataType != null ? !dataType.equals(that.dataType) : that.dataType != null) return false;
+        return true;
+    }
+
+    @Override
+    public int hashCode() {
+        int result = super.hashCode();
+        result = 31 * result + (dataType != null ? dataType.hashCode() : 0);
+        return result;
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/datatypes/DoubleFieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/DoubleFieldValue.java
new file mode 100644
index 00000000000..2d9d900a092
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/datatypes/DoubleFieldValue.java
@@ -0,0 +1,145 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.datatypes;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.Field;
+import com.yahoo.document.PrimitiveDataType;
+import com.yahoo.document.serialization.FieldReader;
+import com.yahoo.document.serialization.FieldWriter;
+import com.yahoo.document.serialization.XmlSerializationHelper;
+import com.yahoo.document.serialization.XmlStream;
+import com.yahoo.vespa.objects.Ids;
+
+/**
+ * FieldValue which encapsulates a double.
+ *
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public final class DoubleFieldValue extends NumericFieldValue {
+    private static class Factory extends PrimitiveDataType.Factory {
+        public FieldValue create() {
+            return new DoubleFieldValue();
+        }
+    }
+    public static PrimitiveDataType.Factory getFactory() { return new Factory(); }
+    public static final int classId = registerClass(Ids.document + 14, DoubleFieldValue.class);
+    private double value;
+
+    public DoubleFieldValue() {
+        this(0.0);
+    }
+
+    public DoubleFieldValue(double value) {
+        this.value = value;
+    }
+
+    public DoubleFieldValue(Double value) {
+        this.value = value;
+    }
+
+    public DoubleFieldValue(String s) { value = Double.parseDouble(s); }
+
+    @Override
+    public DoubleFieldValue clone() {
+        DoubleFieldValue val = (DoubleFieldValue) super.clone();
+        val.value = value;
+        return val;
+    }
+
+    @Override
+    public void clear() {
+        value = 0.0;
+    }
+
+    @Override
+    public Number getNumber() {
+        return value;
+    }
+
+    @Override
+    public void assign(Object obj) {
+        if (!checkAssign(obj)) {
+            return;
+        }
+        if (obj instanceof Number) {
+            value = ((Number) obj).doubleValue();
+        } else if (obj instanceof NumericFieldValue) {
+            value = (((NumericFieldValue) obj).getNumber().doubleValue());
+        } else if (obj instanceof String || obj instanceof StringFieldValue) {
+            value = Double.parseDouble(obj.toString());
+        } else {
+            throw new IllegalArgumentException("Class " + obj.getClass() + " not applicable to an " + this.getClass() + " instance.");
+        }
+    }
+
+    public double getDouble() {
+        return value;
+    }
+
+    @Override
+    public Object getWrappedValue() {
+        return value;
+    }
+
+    @Override
+    public DataType getDataType() {
+        return DataType.DOUBLE;
+    }
+
+    @Override
+    public void printXml(XmlStream xml) {
+        XmlSerializationHelper.printDoubleXml(this, xml);
+    }
+
+    @Override
+    public String toString() {
+        return "" + value;
+    }
+
+    @Override
+    public int hashCode() {
+        int result = super.hashCode();
+        long temp;
+        temp = value != +0.0d ? Double.doubleToLongBits(value) : 0L;
+        result = 31 * result + (int) (temp ^ (temp >>> 32));
+        return result;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (!(o instanceof DoubleFieldValue)) return false;
+        if (!super.equals(o)) return false;
+
+        DoubleFieldValue that = (DoubleFieldValue) o;
+        if (Double.compare(that.value, value) != 0) return false;
+        return true;
+    }
+
+    @Override
+    public void serialize(Field field, FieldWriter writer) {
+        writer.write(field, this);
+    }
+
+    /* (non-Javadoc)
+      * @see com.yahoo.document.datatypes.FieldValue#deserialize(com.yahoo.document.Field, com.yahoo.document.serialization.FieldReader)
+      */
+
+    @Override
+    public void deserialize(Field field, FieldReader reader) {
+        reader.read(field, this);
+    }
+
+    @Override
+    public int compareTo(FieldValue fieldValue) {
+        int comp = super.compareTo(fieldValue);
+
+        if (comp != 0) {
+            return comp;
+        }
+
+        //types are equal, this must be of this type
+        DoubleFieldValue otherValue = (DoubleFieldValue) fieldValue;
+        return Double.compare(value, otherValue.value);
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/datatypes/FieldPathIteratorHandler.java b/document/src/main/java/com/yahoo/document/datatypes/FieldPathIteratorHandler.java
new file mode 100644
index 00000000000..c8d007037f3
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/datatypes/FieldPathIteratorHandler.java
@@ -0,0 +1,103 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.datatypes;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.TreeMap;
+
+/**
+ * @author <a href="mailto:thomasg@yahoo-inc.com">Thomas Gundersen</a>
+ */
+public abstract class FieldPathIteratorHandler {
+
+    public static class IndexValue {
+
+        private int index;
+        private FieldValue key;
+
+        public int getIndex() {
+            return index;
+        }
+
+        public FieldValue getKey() {
+            return key;
+        }
+
+        public IndexValue() {
+            index = -1;
+            key = null;
+        }
+
+        public IndexValue(int index) {
+            this.index = index;
+            key = null;
+        }
+
+        public IndexValue(FieldValue key) {
+            index = -1;
+            this.key = key;
+        }
+
+        public String toString() {
+            if (key != null) {
+                return key.toString();
+            } else {
+                return "" + index;
+            }
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            IndexValue other = (IndexValue)o;
+
+            if (key != null) {
+                if (other.key != null && key.equals(other.key)) {
+                    return true;
+                }
+                return false;
+            }
+
+            return index == other.index;
+        }
+    };
+
+    public static class VariableMap extends TreeMap<String, IndexValue> {
+
+        @Override
+        public Object clone() {
+            Map<String, IndexValue> map = new VariableMap();
+            map.putAll(this);
+            return map;
+        }
+    }
+
+    private VariableMap variables = new VariableMap();
+
+    public void onPrimitive(FieldValue fv) {
+
+    }
+
+    public boolean onComplex(FieldValue fv) {
+        return true;
+    }
+
+    public ModificationStatus doModify(FieldValue fv) {
+        return ModificationStatus.NOT_MODIFIED;
+    }
+
+    public enum ModificationStatus {
+        MODIFIED, REMOVED, NOT_MODIFIED
+    }
+
+    public ModificationStatus modify(FieldValue fv) {
+        return doModify(fv);
+    }
+
+    public boolean createMissingPath() {
+        return false;
+    }
+
+    public VariableMap getVariables() {
+        return variables;
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/datatypes/FieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/FieldValue.java
new file mode 100644
index 00000000000..df10bca83a2
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/datatypes/FieldValue.java
@@ -0,0 +1,187 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.datatypes;
+
+import com.yahoo.document.ArrayDataType;
+import com.yahoo.document.DataType;
+import com.yahoo.document.Field;
+import com.yahoo.document.FieldPath;
+import com.yahoo.document.serialization.*;
+import com.yahoo.io.GrowableByteBuffer;
+import com.yahoo.vespa.objects.BufferSerializer;
+import com.yahoo.vespa.objects.Deserializer;
+import com.yahoo.vespa.objects.Identifiable;
+import com.yahoo.vespa.objects.Ids;
+import com.yahoo.vespa.objects.Serializer;
+import com.yahoo.document.config.DocumentmanagerConfig.Datatype.Structtype.Compresstype;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public abstract class FieldValue extends Identifiable implements Comparable<FieldValue> {
+
+    public static final int classId = registerClass(Ids.document + 9, FieldValue.class);
+
+    public abstract DataType getDataType();
+
+    public static FieldValue create(FieldReader reader, DataType type) {
+        FieldValue value = type.createFieldValue();
+        value.deserialize(reader);
+        return value;
+    }
+
+    /**
+     * Get XML representation of a single field and all its children, if any.
+     * @return XML representation of field in a &lt;value&gt; element
+     */
+    public String toXml() {
+        XmlStream xml = new XmlStream();
+        xml.setIndent("  ");
+        xml.beginTag("value");
+        printXml(xml);
+        xml.endTag();
+        return xml.toString();
+    }
+
+    /**
+     * Read data from the given buffer to create this field value. As some field values have their type self
+     * contained, we need the type manager object to be able to retrieve it.
+     */
+    final public void deserialize(FieldReader reader) {
+        deserialize(null, reader);
+    }
+
+    final public void serialize(GrowableByteBuffer buf) {
+        serialize(DocumentSerializerFactory.create42(buf));
+    }
+
+    public abstract void printXml(XmlStream xml);
+
+    public abstract void clear();
+
+    @Override
+    public FieldValue clone() {
+        return (FieldValue) super.clone();
+    }
+
+    boolean checkAssign(Object o) {
+        if (o == null) {
+            clear();
+            return false;
+        }
+
+        return true;
+    }
+
+    /**
+     * Assign this non-fieldvalue value to this field value. This is used to be able
+     * to assign ints to Integer field values and List to Array field values and such.
+     * <p>
+     * Override to accept the specific types that should be legal.
+     *
+     * @throws IllegalArgumentException If the object given is of wrong type for this field value.
+     */
+    public abstract void assign(Object o);
+
+    /**
+     * Used to retrieve wrapped type for simple types, such that you can use get methods to retrieve ints and floats
+     * directly instead of Int/Float field values. Complex types that can't be specified by simple java types just
+     * return themself.
+     */
+    public Object getWrappedValue() {
+        return this;
+    }
+
+    class RecursiveIteratorHandler extends FieldPathIteratorHandler {
+        FieldValue retVal = null;
+        boolean multiValue = false;
+
+        @Override
+        public boolean onComplex(FieldValue fv) {
+            onPrimitive(fv);
+            return false;
+        }
+
+        @Override
+        public void onPrimitive(FieldValue fv) {
+            if (retVal != null) {
+                if (multiValue) {
+                    ((Array) retVal).add(fv);
+                } else {
+                    Array afv = new Array(new ArrayDataType(retVal.getDataType()));
+                    afv.add(retVal);
+                    afv.add(fv);
+                    retVal = afv;
+                    multiValue = true;
+                }
+            } else {
+                retVal = fv;
+            }
+        }
+    }
+
+    /**
+     * Using the given field path, digs through the document and returns the matching field value.
+     * If the field path resolves to multiple values, returns an ArrayFieldValue containing the
+     * values.
+     */
+    public FieldValue getRecursiveValue(String path) {
+        return getRecursiveValue(getDataType().buildFieldPath(path));
+    }
+
+    public FieldValue getRecursiveValue(FieldPath path) {
+        RecursiveIteratorHandler handler = new RecursiveIteratorHandler();
+        iterateNested(path, 0, handler);
+        return handler.retVal;
+    }
+
+    @Override
+    public void onSerialize(Serializer target) {
+        if (target instanceof FieldWriter) {
+            serialize(null, (FieldWriter) target);
+        } else if (target instanceof BufferSerializer) {
+            serialize(null, DocumentSerializerFactory.create42(((BufferSerializer) target).getBuf()));
+        } else {
+            DocumentSerializer fw = DocumentSerializerFactory.create42(new GrowableByteBuffer());
+            serialize(null, fw);
+            target.put(null, fw.getBuf().getByteBuffer());
+        }
+    }
+
+    @Override
+    public void onDeserialize(Deserializer data) {
+        if (data instanceof FieldReader) {
+            deserialize(null, (FieldReader) data);
+        } else {
+            throw new IllegalArgumentException("I am not able to deserialize from " + data.getClass().getName());
+        }
+    }
+
+    /**
+     * Iterates through the document using the given fieldpath, calling callbacks in the given iterator
+     * handler.
+     */
+    FieldPathIteratorHandler.ModificationStatus iterateNested(FieldPath fieldPath, int pos, FieldPathIteratorHandler handler) {
+        if (pos >= fieldPath.size()) {
+            handler.onPrimitive(this);
+            return handler.modify(this);
+        } else {
+            throw new IllegalArgumentException("Primitive types can't be iterated through");
+        }
+    }
+
+    /**
+     * Write out field value to the specified writer
+     */
+    abstract public void serialize(Field field, FieldWriter writer);
+
+    /**
+     * Read a field value from the specified reader
+     */
+    abstract public void deserialize(Field field, FieldReader reader);
+
+    @Override
+    public int compareTo(FieldValue fieldValue) {
+        return getDataType().compareTo(fieldValue.getDataType());
+    }
+
+}
diff --git a/document/src/main/java/com/yahoo/document/datatypes/FloatFieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/FloatFieldValue.java
new file mode 100644
index 00000000000..a8c83f426ed
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/datatypes/FloatFieldValue.java
@@ -0,0 +1,144 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.datatypes;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.Field;
+import com.yahoo.document.PrimitiveDataType;
+import com.yahoo.document.serialization.FieldReader;
+import com.yahoo.document.serialization.FieldWriter;
+import com.yahoo.document.serialization.XmlSerializationHelper;
+import com.yahoo.document.serialization.XmlStream;
+import com.yahoo.vespa.objects.Ids;
+
+/**
+ * FieldValue which encapsulates a float.
+ *
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public final class FloatFieldValue extends NumericFieldValue {
+    private static class Factory extends PrimitiveDataType.Factory {
+        public FieldValue create() {
+            return new FloatFieldValue();
+        }
+    }
+    public static PrimitiveDataType.Factory getFactory() { return new Factory(); }
+    public static final int classId = registerClass(Ids.document + 13, FloatFieldValue.class);
+    private float value;
+
+    public FloatFieldValue() {
+        this((float) 0);
+    }
+
+    public FloatFieldValue(float value) {
+        this.value = value;
+    }
+
+    public FloatFieldValue(Float value) {
+        this.value = value;
+    }
+
+    public FloatFieldValue(String s) { value = Float.parseFloat(s); }
+
+    @Override
+    public FloatFieldValue clone() {
+        FloatFieldValue val = (FloatFieldValue) super.clone();
+        val.value = value;
+        return val;
+    }
+
+    @Override
+    public Number getNumber() {
+        return value;
+    }
+
+    @Override
+    public void clear() {
+        value = 0.0f;
+    }
+
+    @Override
+    public void assign(Object obj) {
+        if (!checkAssign(obj)) {
+            return;
+        }
+        if (obj instanceof Number) {
+            value = ((Number) obj).floatValue();
+        } else if (obj instanceof NumericFieldValue) {
+            value = (((NumericFieldValue) obj).getNumber().floatValue());
+        } else if (obj instanceof String || obj instanceof StringFieldValue) {
+            value = Float.parseFloat(obj.toString());
+        } else {
+            throw new IllegalArgumentException("Class " + obj.getClass() + " not applicable to an " + this.getClass() + " instance.");
+        }
+    }
+
+    public float getFloat() {
+        return value;
+    }
+
+    @Override
+    public Object getWrappedValue() {
+        return value;
+    }
+
+    @Override
+    public DataType getDataType() {
+        return DataType.FLOAT;
+    }
+
+    @Override
+    public void printXml(XmlStream xml) {
+        XmlSerializationHelper.printFloatXml(this, xml);
+    }
+
+    @Override
+    public String toString() {
+        return "" + value;
+    }
+
+    @Override
+    public int hashCode() {
+        int result = super.hashCode();
+        result = 31 * result + (value != +0.0f ? Float.floatToIntBits(value) : 0);
+        return result;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (!(o instanceof FloatFieldValue)) return false;
+        if (!super.equals(o)) return false;
+
+        FloatFieldValue that = (FloatFieldValue) o;
+        if (Float.compare(that.value, value) != 0) return false;
+        return true;
+    }
+
+    @Override
+    public void serialize(Field field, FieldWriter writer) {
+        writer.write(field, this);
+    }
+
+    /* (non-Javadoc)
+      * @see com.yahoo.document.datatypes.FieldValue#deserialize(com.yahoo.document.Field, com.yahoo.document.serialization.FieldReader)
+      */
+
+    @Override
+    public void deserialize(Field field, FieldReader reader) {
+        reader.read(field, this);
+    }
+
+    @Override
+    public int compareTo(FieldValue fieldValue) {
+        int comp = super.compareTo(fieldValue);
+
+        if (comp != 0) {
+            return comp;
+        }
+
+        //types are equal, this must be of this type
+        FloatFieldValue otherValue = (FloatFieldValue) fieldValue;
+        return Float.compare(value, otherValue.value);
+    }
+
+}
diff --git a/document/src/main/java/com/yahoo/document/datatypes/IntegerFieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/IntegerFieldValue.java
new file mode 100644
index 00000000000..4f54c4c6cb1
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/datatypes/IntegerFieldValue.java
@@ -0,0 +1,153 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.datatypes;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.Field;
+import com.yahoo.document.PrimitiveDataType;
+import com.yahoo.document.serialization.FieldReader;
+import com.yahoo.document.serialization.FieldWriter;
+import com.yahoo.document.serialization.XmlSerializationHelper;
+import com.yahoo.document.serialization.XmlStream;
+import com.yahoo.vespa.objects.Ids;
+
+/**
+ * FieldValue which encapsulates an int.
+ *
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public final class IntegerFieldValue extends NumericFieldValue {
+
+    private static class Factory extends PrimitiveDataType.Factory {
+        public FieldValue create() {
+            return new IntegerFieldValue();
+        }
+    }
+
+    public static PrimitiveDataType.Factory getFactory() { return new Factory(); }
+    public static final int classId = registerClass(Ids.document + 11, IntegerFieldValue.class);
+    private int value;
+
+    public IntegerFieldValue() {
+        this(0);
+    }
+
+    public IntegerFieldValue(int value) {
+        this.value = value;
+    }
+
+    public IntegerFieldValue(Integer value) {
+        this.value = value;
+    }
+
+    public IntegerFieldValue(String s) {
+        value = Integer.parseInt(s);
+    }
+
+    @Override
+    public IntegerFieldValue clone() {
+        IntegerFieldValue val = (IntegerFieldValue) super.clone();
+        val.value = value;
+        return val;
+    }
+
+    @Override
+    public Number getNumber() {
+        return value;
+    }
+
+    @Override
+    public void clear() {
+        value = 0;
+    }
+
+    @Override
+    public void assign(Object obj) {
+        if (!checkAssign(obj)) {
+            return;
+        }
+        if (obj instanceof Number) {
+            value = ((Number) obj).intValue();
+        } else if (obj instanceof NumericFieldValue) {
+            value = (((NumericFieldValue) obj).getNumber().intValue());
+        } else if (obj instanceof String || obj instanceof StringFieldValue) {
+            value = Integer.parseInt(obj.toString());
+        } else {
+            throw new IllegalArgumentException("Class " + obj.getClass() + " not applicable to an " + this.getClass() + " instance.");
+        }
+    }
+
+    public int getInteger() {
+        return value;
+    }
+
+    @Override
+    public Object getWrappedValue() {
+        return value;
+    }
+
+    @Override
+    public void printXml(XmlStream xml) {
+        XmlSerializationHelper.printIntegerXml(this, xml);
+    }
+
+    @Override
+    public String toString() {
+        return "" + value;
+    }
+
+    @Override
+    public int hashCode() {
+        int result = super.hashCode();
+        result = 31 * result + value;
+        return result;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (!(o instanceof IntegerFieldValue)) return false;
+        if (!super.equals(o)) return false;
+
+        IntegerFieldValue that = (IntegerFieldValue) o;
+        return (value == that.value);
+    }
+
+    @Override
+    public void serialize(Field field, FieldWriter writer) {
+        writer.write(field, this);
+    }
+
+    /* (non-Javadoc)
+      * @see com.yahoo.document.datatypes.FieldValue#deserialize(com.yahoo.document.Field, com.yahoo.document.serialization.FieldReader)
+      */
+
+    @Override
+    public void deserialize(Field field, FieldReader reader) {
+        reader.read(field, this);
+    }
+
+    @Override
+    public DataType getDataType() {
+        return DataType.INT;
+    }
+
+    @Override
+    public int compareTo(FieldValue fieldValue) {
+        int comp = super.compareTo(fieldValue);
+
+        if (comp != 0) {
+            return comp;
+        }
+
+        //types are equal, this must be of this type
+        IntegerFieldValue otherValue = (IntegerFieldValue) fieldValue;
+        if (value < otherValue.value) {
+            return -1;
+        } else if (value > otherValue.value) {
+            return 1;
+        } else {
+            return 0;
+        }
+    }
+
+}
diff --git a/document/src/main/java/com/yahoo/document/datatypes/LongFieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/LongFieldValue.java
new file mode 100644
index 00000000000..3705d89e146
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/datatypes/LongFieldValue.java
@@ -0,0 +1,151 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.datatypes;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.Field;
+import com.yahoo.document.PrimitiveDataType;
+import com.yahoo.document.serialization.FieldReader;
+import com.yahoo.document.serialization.FieldWriter;
+import com.yahoo.document.serialization.XmlSerializationHelper;
+import com.yahoo.document.serialization.XmlStream;
+import com.yahoo.vespa.objects.Ids;
+
+/**
+ * FieldValue which encapsulates a long.
+ *
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public final class LongFieldValue extends NumericFieldValue {
+    private static class Factory extends PrimitiveDataType.Factory {
+        public FieldValue create() {
+            return new LongFieldValue();
+        }
+    }
+    public static PrimitiveDataType.Factory getFactory() { return new Factory(); }
+    public static final int classId = registerClass(Ids.document + 12, LongFieldValue.class);
+    private long value;
+
+    public LongFieldValue() {
+        this(0l);
+    }
+
+    public LongFieldValue(long value) {
+        this.value = value;
+    }
+
+    public LongFieldValue(Long value) {
+        this.value = value;
+    }
+
+    public LongFieldValue(String s) {
+        value = Long.parseLong(s);
+    }
+
+    @Override
+    public LongFieldValue clone() {
+        LongFieldValue val = (LongFieldValue) super.clone();
+        val.value = value;
+        return val;
+    }
+
+    @Override
+    public void clear() {
+        value = 0l;
+    }
+
+    @Override
+    public Number getNumber() {
+        return value;
+    }
+
+    @Override
+    public void assign(Object obj) {
+        if (!checkAssign(obj)) {
+            return;
+        }
+        if (obj instanceof Number) {
+            value = ((Number) obj).longValue();
+        } else if (obj instanceof NumericFieldValue) {
+            value = (((NumericFieldValue) obj).getNumber().longValue());
+        } else if (obj instanceof String || obj instanceof StringFieldValue) {
+            value = Long.parseLong(obj.toString());
+        } else {
+            throw new IllegalArgumentException("Class " + obj.getClass() + " not applicable to an " + this.getClass() + " instance.");
+        }
+    }
+
+    public long getLong() {
+        return value;
+    }
+
+    @Override
+    public Object getWrappedValue() {
+        return value;
+    }
+
+    @Override
+    public DataType getDataType() {
+        return DataType.LONG;
+    }
+
+    @Override
+    public void printXml(XmlStream xml) {
+        XmlSerializationHelper.printLongXml(this, xml);
+    }
+
+    @Override
+    public String toString() {
+        return "" + value;
+    }
+
+    @Override
+    public int hashCode() {
+        int result = super.hashCode();
+        result = 31 * result + (int) (value ^ (value >>> 32));
+        return result;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (!(o instanceof LongFieldValue)) return false;
+        if (!super.equals(o)) return false;
+
+        LongFieldValue that = (LongFieldValue) o;
+        if (value != that.value) return false;
+        return true;
+    }
+
+    @Override
+    public void serialize(Field field, FieldWriter writer) {
+        writer.write(field, this);
+    }
+
+    /* (non-Javadoc)
+      * @see com.yahoo.document.datatypes.FieldValue#deserialize(com.yahoo.document.Field, com.yahoo.document.serialization.FieldReader)
+      */
+
+    @Override
+    public void deserialize(Field field, FieldReader reader) {
+        reader.read(field, this);
+    }
+
+    @Override
+    public int compareTo(FieldValue fieldValue) {
+        int comp = super.compareTo(fieldValue);
+
+        if (comp != 0) {
+            return comp;
+        }
+
+        //types are equal, this must be of this type
+        LongFieldValue otherValue = (LongFieldValue) fieldValue;
+        if (value < otherValue.value) {
+            return -1;
+        } else if (value > otherValue.value) {
+            return 1;
+        } else {
+            return 0;
+        }
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/datatypes/MapFieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/MapFieldValue.java
new file mode 100644
index 00000000000..261d40161d0
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/datatypes/MapFieldValue.java
@@ -0,0 +1,396 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.datatypes;
+
+import com.yahoo.collections.CollectionComparator;
+import com.yahoo.document.DataType;
+import com.yahoo.document.Field;
+import com.yahoo.document.FieldPath;
+import com.yahoo.document.MapDataType;
+import com.yahoo.document.serialization.FieldReader;
+import com.yahoo.document.serialization.FieldWriter;
+import com.yahoo.document.serialization.XmlSerializationHelper;
+import com.yahoo.document.serialization.XmlStream;
+import java.util.*;
+
+/**
+ * Vespa map. Backed by and and parametrized by FieldValue
+ *
+ * @author vegardh
+ */
+public class MapFieldValue<K extends FieldValue, V extends FieldValue> extends CompositeFieldValue implements java.util.Map<K,V> {
+
+    private java.util.Map<K,V> values;
+
+    public MapFieldValue(MapDataType type) {
+        this(type, 1);
+    }
+
+    public MapFieldValue(MapDataType type, int initialCapacity) {
+        super(type);
+        values = new HashMap<K, V>(initialCapacity);
+    }
+
+    @Override
+    public MapDataType getDataType() {
+        return (MapDataType)super.getDataType();
+    }
+
+    @Override
+    public void assign(Object o) {
+        if (!checkAssign(o)) {
+            return;
+        }
+
+        if (o instanceof MapFieldValue) {
+            if (o == this) return;
+            MapFieldValue a = (MapFieldValue) o;
+            values.clear();
+            putAll(a);
+        } else if (o instanceof Map) {
+            values = new MapWrapper((Map)o);
+        }
+        else {
+            throw new IllegalArgumentException("Class " + o.getClass() + " not applicable to an " + this.getClass() + " instance.");
+        }
+    }
+
+    @Override
+    public MapFieldValue clone() {
+        MapFieldValue copy = (MapFieldValue) super.clone();
+        copy.values = new HashMap<K, V>(values.size());
+        for (Map.Entry<K, V> entry : values.entrySet()) {
+            copy.values.put(entry.getKey().clone(), entry.getValue().clone());
+        }
+        return copy;
+    }
+
+    /**
+     * Checks if another object is equal to this set.
+     *
+     * @param o the object to check for equality with
+     * @return true if o is an instance of WeightedSet and the two encapsulated Maps are equal, false otherwise
+     */
+    public boolean equals(Object o) {
+        if (!(o instanceof MapFieldValue)) return false;
+        MapFieldValue otherSet = (MapFieldValue) o;
+        Map<K, V> map1 = values;
+        Map<K, V> map2 = otherSet.values;
+        return (super.equals(o) && map1.equals(map2));
+    }
+
+    @Override
+    public void clear() {
+        values.clear();
+    }
+
+    @Override
+    public void deserialize(Field field, FieldReader reader) {
+        reader.read(field, this);
+    }
+
+    @Override
+    public void printXml(XmlStream xml) {
+        XmlSerializationHelper.printMapXml(this, xml);
+    }
+
+    @Override
+    public void serialize(Field field, FieldWriter writer) {
+        writer.write(field, this);
+    }
+
+    @Override
+    public Object getWrappedValue() {
+        if (values instanceof MapFieldValue.MapWrapper) {
+            return ((MapFieldValue.MapWrapper) values).map;
+        }
+        Map tmpMap = new HashMap();
+        for (Entry<K, V> kvEntry : values.entrySet()) {
+            tmpMap.put(kvEntry.getKey().getWrappedValue(), kvEntry.getValue().getWrappedValue());
+        }
+        return tmpMap;
+    }
+
+    ///// java.util.Map methods
+
+    public boolean containsKey(Object key) {
+        return values.containsKey(key);
+    }
+
+    public boolean containsValue(Object value) {
+        return values.containsValue(value);
+    }
+
+    public Set<java.util.Map.Entry<K, V>> entrySet() {
+        return values.entrySet();
+    }
+
+    public V get(Object key) {
+        return values.get(key);
+    }
+
+    public Set<K> keySet() {
+        return values.keySet();
+    }
+
+    private void validateCompatibleTypes(DataType d, FieldValue v) {
+        if (!d.isValueCompatible(v)) {
+            throw new IllegalArgumentException(
+                    "Incompatible data types. Got " + v.getDataType()
+                    + ", expected " + d);
+        }
+    }
+
+    public V put(K key, V value) {
+        validateCompatibleTypes(getDataType().getKeyType(), key);
+        validateCompatibleTypes(getDataType().getValueType(), value);
+        return values.put(key, value);
+    }
+
+    public void putAll(java.util.Map<? extends K, ? extends V> m) {
+        for (K key : m.keySet()) {
+            validateCompatibleTypes(getDataType().getKeyType(), key);
+        }
+        for (V value : m.values()) {
+            validateCompatibleTypes(getDataType().getValueType(), value);
+        }
+        values.putAll(m);
+    }
+
+    public V remove(Object key) {
+        return values.remove(key);
+    }
+
+    public Collection<V> values() {
+        return values.values();
+    }
+
+    public boolean contains(Object o) {
+        return values.containsKey(o);
+    }
+
+    public boolean isEmpty() {
+        return values.isEmpty();
+    }
+
+    public int size() {
+        return values.size();
+    }
+
+    boolean checkAndRemove(FieldValue key, FieldPathIteratorHandler.ModificationStatus status, boolean wasModified, List<FieldValue> keysToRemove) {
+        if (status == FieldPathIteratorHandler.ModificationStatus.REMOVED) {
+            keysToRemove.add(key);
+            return true;
+        } else if (status == FieldPathIteratorHandler.ModificationStatus.MODIFIED) {
+            return true;
+        }
+
+        return wasModified;
+    }
+
+   FieldPathIteratorHandler.ModificationStatus iterateNested(FieldPath fieldPath, int pos, FieldPathIteratorHandler handler, FieldValue complexFieldValue) {
+        List<FieldValue> keysToRemove = new ArrayList<FieldValue>();
+        boolean wasModified = false;
+
+        if (pos < fieldPath.size()) {
+            switch (fieldPath.get(pos).getType()) {
+                case MAP_KEY:
+                {
+                    FieldValue val = values.get(fieldPath.get(pos).getLookupKey());
+                    if (val != null) {
+                        wasModified = checkAndRemove(fieldPath.get(pos).getLookupKey(), val.iterateNested(fieldPath, pos + 1, handler), wasModified, keysToRemove);
+                    } else if (handler.createMissingPath()) {
+                        val = getDataType().getValueType().createFieldValue();
+                        FieldPathIteratorHandler.ModificationStatus status = val.iterateNested(fieldPath, pos + 1, handler);
+                        if (status == FieldPathIteratorHandler.ModificationStatus.MODIFIED) {
+                            put((K)fieldPath.get(pos).getLookupKey(), (V)val);
+                            return status;
+                        }
+                    }
+                    break;
+                }
+                case MAP_ALL_KEYS:
+                    for (FieldValue f : values.keySet()) {
+                        wasModified = checkAndRemove(f, f.iterateNested(fieldPath, pos + 1, handler), wasModified, keysToRemove);
+                    }
+                    break;
+                case MAP_ALL_VALUES:
+                    for (Map.Entry<K, V> entry : values.entrySet()) {
+                        wasModified = checkAndRemove(entry.getKey(), entry.getValue().iterateNested(fieldPath, pos + 1, handler), wasModified, keysToRemove);
+                    }
+                    break;
+                case VARIABLE:
+                {
+                    FieldPathIteratorHandler.IndexValue idx = handler.getVariables().get(fieldPath.get(pos).getVariableName());
+                    if (idx != null) {
+                        FieldValue val = values.get(idx.getKey());
+                        if (val != null) {
+                            wasModified = checkAndRemove(idx.getKey(), val.iterateNested(fieldPath, pos + 1, handler), wasModified, keysToRemove);
+                        }
+                    } else {
+                        for (Map.Entry<K, V> entry : values.entrySet()) {
+                            handler.getVariables().put(fieldPath.get(pos).getVariableName(), new FieldPathIteratorHandler.IndexValue(entry.getKey()));
+                            wasModified = checkAndRemove(entry.getKey(), entry.getValue().iterateNested(fieldPath, pos + 1, handler), wasModified, keysToRemove);
+                        }
+                        handler.getVariables().remove(fieldPath.get(pos).getVariableName());
+                    }
+                    break;
+                }
+                default:
+                    for (Map.Entry<K, V> entry : values.entrySet()) {
+                        wasModified = checkAndRemove(entry.getKey(), entry.getKey().iterateNested(fieldPath, pos, handler), wasModified, keysToRemove);
+                    }
+                    break;
+            }
+        } else {
+            FieldPathIteratorHandler.ModificationStatus status = handler.modify(complexFieldValue);
+            if (status == FieldPathIteratorHandler.ModificationStatus.REMOVED) {
+                return status;
+            } else if (status == FieldPathIteratorHandler.ModificationStatus.MODIFIED) {
+                wasModified = true;
+            }
+
+            if (handler.onComplex(complexFieldValue)) {
+                for (Map.Entry<K, V> entry : values.entrySet()) {
+                    wasModified = checkAndRemove(entry.getKey(), entry.getKey().iterateNested(fieldPath, pos, handler), wasModified, keysToRemove);
+                }
+            }
+        }
+
+        for (FieldValue f : keysToRemove) {
+            values.remove(f);
+        }
+
+        return wasModified ? FieldPathIteratorHandler.ModificationStatus.MODIFIED : FieldPathIteratorHandler.ModificationStatus.NOT_MODIFIED;
+    }
+
+    @Override
+    FieldPathIteratorHandler.ModificationStatus iterateNested(FieldPath fieldPath, int pos, FieldPathIteratorHandler handler) {
+        return iterateNested(fieldPath, pos, handler, this);
+    }
+
+    @Override
+    public int compareTo(FieldValue fieldValue) {
+        int comp = super.compareTo(fieldValue);
+
+        if (comp != 0) {
+            return comp;
+        }
+        //types are equal, this must be of this type
+        MapFieldValue otherValue = (MapFieldValue) fieldValue;
+        comp = CollectionComparator.compare(values.keySet(), otherValue.values.keySet());
+
+        if (comp != 0) {
+            return comp;
+        }
+
+        return CollectionComparator.compare(values.values(), otherValue.values.values());
+    }
+
+    /**
+     * Map of field values backed by a normal map of Java objects
+     * @author vegardh
+     *
+     */
+    class MapWrapper implements Map<K,V> {
+
+        private Map<Object,Object> map; // Not field values, basic objects
+        private DataType keyTypeVespa = getDataType().getKeyType();
+        private DataType valTypeVespa = getDataType().getValueType();
+        public MapWrapper(Map map) {
+            this.map=map;
+        }
+
+        private Object unwrap(Object o) {
+            return (o instanceof FieldValue ? ((FieldValue) o).getWrappedValue() : o);
+        }
+
+        private K wrapKey(Object o) {
+            if (o==null) return null;
+            return (K)keyTypeVespa.createFieldValue(o);
+        }
+
+        private V wrapValue(Object o) {
+            if (o==null) return null;
+            return (V)valTypeVespa.createFieldValue(o);
+        }
+
+        @Override
+        public void clear() {
+            map.clear();
+        }
+
+        @Override
+        public boolean containsKey(Object key) {
+            return map.containsKey(unwrap(key));
+        }
+
+        @Override
+        public boolean containsValue(Object value) {
+            return map.containsValue(unwrap(value));
+        }
+
+        @Override
+        public Set<java.util.Map.Entry<K, V>> entrySet() {
+            Map<K, V> ret = new HashMap<K, V>();
+            for (Map.Entry e : map.entrySet()) {
+                ret.put(wrapKey(e.getKey()), wrapValue(e.getValue()));
+            }
+            return ret.entrySet();
+        }
+
+        @Override
+        public V get(Object key) {
+            Object o = map.get(unwrap(key));
+            return o == null ? null : wrapValue(o);
+        }
+
+        @Override
+        public boolean isEmpty() {
+            return map.isEmpty();
+        }
+
+        @Override
+        public Set<K> keySet() {
+            Set<K> ret = new HashSet<K>();
+            for (Map.Entry e : map.entrySet()) {
+                ret.add(wrapKey(e.getKey()));
+            }
+            return ret;
+        }
+
+        @Override
+        public V put(K key, V value) {
+            V old = get(key);
+            map.put(unwrap(key), unwrap(value));
+            return old;
+        }
+
+        @Override
+        public void putAll(Map<? extends K, ? extends V> m) {
+            for (Map.Entry<?, ?> e : m.entrySet()) {
+                map.put(unwrap(e.getKey()), unwrap(e.getValue()));
+            }
+        }
+
+        @Override
+        public V remove(Object key) {
+            return wrapValue(map.remove(unwrap(key)));
+        }
+
+        @Override
+        public int size() {
+            return map.size();
+        }
+
+        @Override
+        public Collection<V> values() {
+            Collection<V> ret = new ArrayList<V>();
+            for (Object v : map.values()) {
+                ret.add(wrapValue(v));
+            }
+            return ret;
+        }
+
+    }
+
+}
diff --git a/document/src/main/java/com/yahoo/document/datatypes/NumericFieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/NumericFieldValue.java
new file mode 100644
index 00000000000..776ca42c47d
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/datatypes/NumericFieldValue.java
@@ -0,0 +1,8 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.datatypes;
+
+public abstract class NumericFieldValue extends FieldValue {
+
+    public abstract Number getNumber();
+
+}
diff --git a/document/src/main/java/com/yahoo/document/datatypes/PredicateFieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/PredicateFieldValue.java
new file mode 100644
index 00000000000..4978acc19d2
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/datatypes/PredicateFieldValue.java
@@ -0,0 +1,136 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.datatypes;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.Field;
+import com.yahoo.document.PrimitiveDataType;
+import com.yahoo.document.predicate.Predicate;
+import com.yahoo.document.serialization.FieldReader;
+import com.yahoo.document.serialization.FieldWriter;
+import com.yahoo.document.serialization.XmlStream;
+
+import java.util.Objects;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a>
+ */
+public class PredicateFieldValue extends FieldValue {
+
+    private Predicate predicate;
+
+    public PredicateFieldValue() {
+        this((Predicate)null);
+    }
+
+    public PredicateFieldValue(Predicate predicate) {
+        this.predicate = predicate;
+    }
+
+    public PredicateFieldValue(String predicateString) {
+        this(Predicate.fromString(predicateString));
+    }
+
+    public Predicate getPredicate() {
+        return predicate;
+    }
+
+    public PredicateFieldValue setPredicate(Predicate predicate) {
+        this.predicate = predicate;
+        return this;
+    }
+
+    @Override
+    public DataType getDataType() {
+        return DataType.PREDICATE;
+    }
+
+    @Override
+    public void printXml(XmlStream xml) {
+        if (predicate == null) {
+            return;
+        }
+        xml.addContent(predicate.toString());
+    }
+
+    @Override
+    public void clear() {
+        predicate = null;
+    }
+
+    @Override
+    public void assign(Object o) {
+        if (o == null) {
+            predicate = null;
+        } else if (o instanceof Predicate) {
+            predicate = (Predicate)o;
+        } else if (o instanceof PredicateFieldValue) {
+            predicate = ((PredicateFieldValue)o).predicate;
+        } else {
+            throw new IllegalArgumentException("Expected " + getClass().getName() + ", got " +
+                                               o.getClass().getName() + ".");
+        }
+    }
+
+    @Override
+    public void serialize(Field field, FieldWriter writer) {
+        writer.write(field, this);
+    }
+
+    @Override
+    public void deserialize(Field field, FieldReader reader) {
+        reader.read(field, this);
+    }
+
+    @Override
+    public Object getWrappedValue() {
+        return predicate;
+    }
+
+    @Override
+    public PredicateFieldValue clone() {
+        PredicateFieldValue obj = (PredicateFieldValue)super.clone();
+        if (predicate != null) {
+            try {
+                obj.predicate = predicate.clone();
+            } catch (CloneNotSupportedException e) {
+                throw new AssertionError(e);
+            }
+        }
+        return obj;
+    }
+
+    @Override
+    public int hashCode() {
+        return predicate != null ? predicate.hashCode() : 31;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == this) {
+            return true;
+        }
+        if (!(obj instanceof PredicateFieldValue)) {
+            return false;
+        }
+        PredicateFieldValue rhs = (PredicateFieldValue)obj;
+        if (!Objects.equals(predicate, rhs.predicate)) {
+            return false;
+        }
+        return true;
+    }
+
+    @Override
+    public String toString() {
+        return String.valueOf(predicate);
+    }
+
+    public static PrimitiveDataType.Factory getFactory() {
+        return new PrimitiveDataType.Factory() {
+
+            @Override
+            public FieldValue create() {
+                return new PredicateFieldValue();
+            }
+        };
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/datatypes/Raw.java b/document/src/main/java/com/yahoo/document/datatypes/Raw.java
new file mode 100644
index 00000000000..7d4d2430984
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/datatypes/Raw.java
@@ -0,0 +1,146 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.datatypes;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.Field;
+import com.yahoo.document.PrimitiveDataType;
+import com.yahoo.document.serialization.FieldReader;
+import com.yahoo.document.serialization.FieldWriter;
+import com.yahoo.document.serialization.XmlSerializationHelper;
+import com.yahoo.document.serialization.XmlStream;
+import com.yahoo.vespa.objects.Ids;
+
+import java.io.ByteArrayOutputStream;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+
+/**
+ * FieldValue which encapsulates a Raw value
+ *
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public final class Raw extends FieldValue {
+    private static class Factory extends PrimitiveDataType.Factory {
+        public FieldValue create() {
+            return new Raw();
+        }
+    }
+    public static PrimitiveDataType.Factory getFactory() { return new Factory(); }
+    public static final int classId = registerClass(Ids.document + 16, Raw.class);
+    private ByteBuffer value;
+
+    public Raw() {
+        value = null;
+    }
+
+    public Raw(ByteBuffer value) {
+        this.value = value;
+    }
+
+    public Raw(byte [] buf) {
+        this.value = ByteBuffer.wrap(buf);
+        value.position(0);
+    }
+
+    public ByteBuffer getByteBuffer() {
+        return value;
+    }
+
+    @Override
+    public Raw clone() {
+        Raw raw = (Raw) super.clone();
+        if (value.hasArray()) {
+            raw.value = ByteBuffer.wrap(Arrays.copyOf(value.array(), value.array().length));
+            raw.value.position(value.position());
+        } else {
+            byte[] copyBuf = new byte[value.capacity()];
+            int origPos = value.position();
+            value.position(0);
+            value.get(copyBuf);
+            value.position(origPos);
+            raw.value = ByteBuffer.wrap(copyBuf);
+            raw.value.position(value.position());
+        }
+        return raw;
+    }
+
+    @Override
+    public Object getWrappedValue() {
+        return value;
+    }
+
+    @Override
+    public void clear() {
+        value = ByteBuffer.wrap(new byte[0]);
+    }
+
+    @Override
+    public void assign(Object o) {
+        if (!checkAssign(o)) {
+            return;
+        }
+        if (o instanceof Raw) {
+            value = ((Raw) o).value;
+        } else if (o instanceof ByteBuffer) {
+            value = (ByteBuffer) o;
+        } else if (o instanceof byte[]) {
+            ByteBuffer byteBufVal = ByteBuffer.wrap((byte[]) o);
+            byteBufVal.position(0);
+            value = byteBufVal;
+        } else {
+            throw new IllegalArgumentException("Class " + o.getClass() + " not applicable to an " + this.getClass() + " instance.");
+        }
+    }
+
+    @Override
+    public DataType getDataType() {
+        return DataType.RAW;
+    }
+
+    @Override
+    public void printXml(XmlStream xml) {
+        XmlSerializationHelper.printRawXml(this, xml);
+    }
+
+    @Override
+    public String toString() {
+        ByteBuffer buf = value.slice();
+        byte[] arr = new byte[buf.remaining()];
+        buf.get(arr);
+        return com.yahoo.io.HexDump.toHexString(arr);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (!(o instanceof Raw)) return false;
+        if (!super.equals(o)) return false;
+
+        Raw raw = (Raw) o;
+
+        if (value != null ? !value.equals(raw.value) : raw.value != null) return false;
+
+        return true;
+    }
+
+    @Override
+    public int hashCode() {
+        int result = super.hashCode();
+        result = 31 * result + (value != null ? value.hashCode() : 0);
+        return result;
+    }
+
+    @Override
+    public void serialize(Field field, FieldWriter writer) {
+        writer.write(field, this);
+    }
+
+    /* (non-Javadoc)
+      * @see com.yahoo.document.datatypes.FieldValue#deserialize(com.yahoo.document.Field, com.yahoo.document.serialization.FieldReader)
+      */
+
+    @Override
+    public void deserialize(Field field, FieldReader reader) {
+        reader.read(field, this);
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/datatypes/StringFieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/StringFieldValue.java
new file mode 100644
index 00000000000..29c5951f39b
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/datatypes/StringFieldValue.java
@@ -0,0 +1,447 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.datatypes;
+
+import com.google.common.collect.ImmutableList;
+import com.yahoo.collections.CollectionComparator;
+import com.yahoo.document.DataType;
+import com.yahoo.document.Field;
+import com.yahoo.document.PrimitiveDataType;
+import com.yahoo.document.annotation.SpanTree;
+import com.yahoo.document.serialization.FieldReader;
+import com.yahoo.document.serialization.FieldWriter;
+import com.yahoo.document.serialization.XmlSerializationHelper;
+import com.yahoo.document.serialization.XmlStream;
+import com.yahoo.vespa.objects.Ids;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * A StringFieldValue is a wrapper class that holds a String in {@link com.yahoo.document.Document}s and
+ * other {@link com.yahoo.document.datatypes.FieldValue}s.
+ *
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class StringFieldValue extends FieldValue {
+    private static class Factory extends PrimitiveDataType.Factory {
+        public FieldValue create() {
+            return new StringFieldValue();
+        }
+    }
+    public static PrimitiveDataType.Factory getFactory() { return new Factory(); }
+    public static final int classId = registerClass(Ids.document + 15, StringFieldValue.class);
+    private String value;
+    private Map<String, SpanTree> spanTrees = null;
+    private static final boolean[] allowedAsciiChars = new boolean[0x80];
+
+    static {
+        allowedAsciiChars[0x0] = false;
+        allowedAsciiChars[0x1] = false;
+        allowedAsciiChars[0x2] = false;
+        allowedAsciiChars[0x3] = false;
+        allowedAsciiChars[0x4] = false;
+        allowedAsciiChars[0x5] = false;
+        allowedAsciiChars[0x6] = false;
+        allowedAsciiChars[0x7] = false;
+        allowedAsciiChars[0x8] = false;
+        allowedAsciiChars[0x9] = true;  //tab
+        allowedAsciiChars[0xA] = true;  //nl
+        allowedAsciiChars[0xB] = false;
+        allowedAsciiChars[0xC] = false;
+        allowedAsciiChars[0xD] = true;  //cr
+        for (int i = 0xE; i < 0x20; i++) {
+            allowedAsciiChars[i] = false;
+        }
+        for (int i = 0x20; i < 0x7F; i++) {
+            allowedAsciiChars[i] = true;  //printable ascii chars
+        }
+        allowedAsciiChars[0x7F] = true;  //del - discouraged, but allowed
+    }
+
+
+    /** Creates a new StringFieldValue holding an empty String. */
+    public StringFieldValue() {
+        value = "";
+    }
+
+    /**
+     * Creates a new StringFieldValue with the given value.
+     *
+     * @param value the value to wrap.
+     */
+    public StringFieldValue(String value) {
+        if (value==null) throw new IllegalArgumentException("Value cannot be null");
+        setValue(value);
+    }
+
+    private void setValue(String value) {
+        for (int i = 0; i < value.length(); i++) {
+            char theChar = value.charAt(i);
+            int codePoint = value.codePointAt(i);
+            if (Character.isHighSurrogate(theChar)) {
+                //skip one char ahead, since codePointAt() consumes one more char in this case
+                ++i;
+            }
+
+            //See http://www.w3.org/TR/2006/REC-xml11-20060816/#charsets
+
+            if (codePoint < 0x80) {  //ascii
+                if (allowedAsciiChars[codePoint]) {
+                    continue;
+                } else {
+                    throw new IllegalArgumentException("StringFieldValue cannot contain code point 0x" + Integer.toHexString(codePoint).toUpperCase());
+                }
+            }
+
+            //source cited above notes that 0x7F-0x84 and 0x86-0x9F are discouraged, but they are still allowed.
+            //see http://www.w3.org/International/questions/qa-controls
+
+            if (codePoint < 0xFDD0) {
+                continue;
+            }
+            if (codePoint <= 0xFDDF) {
+                throw new IllegalArgumentException("StringFieldValue cannot contain code point 0x" + Integer.toHexString(codePoint).toUpperCase());
+            }
+
+            if (codePoint < 0x1FFFE) {
+                continue;
+            }
+            if (codePoint <= 0x1FFFF) {
+                throw new IllegalArgumentException("StringFieldValue cannot contain code point 0x" + Integer.toHexString(codePoint).toUpperCase());
+            }
+            if (codePoint < 0x2FFFE) {
+                continue;
+            }
+            if (codePoint <= 0x2FFFF) {
+                throw new IllegalArgumentException("StringFieldValue cannot contain code point 0x" + Integer.toHexString(codePoint).toUpperCase());
+            }
+            if (codePoint < 0x3FFFE) {
+                continue;
+            }
+            if (codePoint <= 0x3FFFF) {
+                throw new IllegalArgumentException("StringFieldValue cannot contain code point 0x" + Integer.toHexString(codePoint).toUpperCase());
+            }
+            if (codePoint < 0x4FFFE) {
+                continue;
+            }
+            if (codePoint <= 0x4FFFF) {
+                throw new IllegalArgumentException("StringFieldValue cannot contain code point 0x" + Integer.toHexString(codePoint).toUpperCase());
+            }
+            if (codePoint < 0x5FFFE) {
+                continue;
+            }
+            if (codePoint <= 0x5FFFF) {
+                throw new IllegalArgumentException("StringFieldValue cannot contain code point 0x" + Integer.toHexString(codePoint).toUpperCase());
+            }
+            if (codePoint < 0x6FFFE) {
+                continue;
+            }
+            if (codePoint <= 0x6FFFF) {
+                throw new IllegalArgumentException("StringFieldValue cannot contain code point 0x" + Integer.toHexString(codePoint).toUpperCase());
+            }
+            if (codePoint < 0x7FFFE) {
+                continue;
+            }
+            if (codePoint <= 0x7FFFF) {
+                throw new IllegalArgumentException("StringFieldValue cannot contain code point 0x" + Integer.toHexString(codePoint).toUpperCase());
+            }
+            if (codePoint < 0x8FFFE) {
+                continue;
+            }
+            if (codePoint <= 0x8FFFF) {
+                throw new IllegalArgumentException("StringFieldValue cannot contain code point 0x" + Integer.toHexString(codePoint).toUpperCase());
+            }
+            if (codePoint < 0x9FFFE) {
+                continue;
+            }
+            if (codePoint <= 0x9FFFF) {
+                throw new IllegalArgumentException("StringFieldValue cannot contain code point 0x" + Integer.toHexString(codePoint).toUpperCase());
+            }
+            if (codePoint < 0xAFFFE) {
+                continue;
+            }
+            if (codePoint <= 0xAFFFF) {
+                throw new IllegalArgumentException("StringFieldValue cannot contain code point 0x" + Integer.toHexString(codePoint).toUpperCase());
+            }
+            if (codePoint < 0xBFFFE) {
+                continue;
+            }
+            if (codePoint <= 0xBFFFF) {
+                throw new IllegalArgumentException("StringFieldValue cannot contain code point 0x" + Integer.toHexString(codePoint).toUpperCase());
+            }
+            if (codePoint < 0xCFFFE) {
+                continue;
+            }
+            if (codePoint <= 0xCFFFF) {
+                throw new IllegalArgumentException("StringFieldValue cannot contain code point 0x" + Integer.toHexString(codePoint).toUpperCase());
+            }
+            if (codePoint < 0xDFFFE) {
+                continue;
+            }
+            if (codePoint <= 0xDFFFF) {
+                throw new IllegalArgumentException("StringFieldValue cannot contain code point 0x" + Integer.toHexString(codePoint).toUpperCase());
+            }
+            if (codePoint < 0xEFFFE) {
+                continue;
+            }
+            if (codePoint <= 0xEFFFF) {
+                throw new IllegalArgumentException("StringFieldValue cannot contain code point 0x" + Integer.toHexString(codePoint).toUpperCase());
+            }
+            if (codePoint < 0xFFFFE) {
+                continue;
+            }
+            if (codePoint <= 0xFFFFF) {
+                throw new IllegalArgumentException("StringFieldValue cannot contain code point 0x" + Integer.toHexString(codePoint).toUpperCase());
+            }
+            if (codePoint < 0x10FFFE) {
+                continue;
+            }
+            if (codePoint <= 0x10FFFF) {
+                throw new IllegalArgumentException("StringFieldValue cannot contain code point 0x" + Integer.toHexString(codePoint).toUpperCase());
+            }
+        }
+        this.value = value;
+    }
+
+    /**
+     * Returns {@link com.yahoo.document.DataType}.STRING.
+     *
+     * @return DataType.STRING, always
+     * @see com.yahoo.document.DataType
+     */
+    @Override
+    public DataType getDataType() {
+        return DataType.STRING;
+    }
+
+    /**
+     * Clones this StringFieldValue and its span trees.
+     *
+     * @return a new deep-copied StringFieldValue
+     */
+    @Override
+    public StringFieldValue clone() {
+        StringFieldValue strfval = (StringFieldValue) super.clone();
+        if (spanTrees != null) {
+            strfval.spanTrees = new HashMap<String, SpanTree>(spanTrees.size());
+            for (Map.Entry<String, SpanTree> entry : spanTrees.entrySet()) {
+                strfval.spanTrees.put(entry.getKey(), new SpanTree(entry.getValue()));
+            }
+        }
+        return strfval;
+    }
+
+    /** Sets the wrapped String to be an empty String, and clears all span trees. */
+    @Override
+    public void clear() {
+        value = "";
+        if (spanTrees != null) {
+            spanTrees.clear();
+            spanTrees = null;
+        }
+    }
+
+    /**
+     * Sets a new value for this StringFieldValue.&nbsp;NOTE that doing so will clear all span trees from this value,
+     * since they most certainly will not make sense for a new string value.
+     *
+     * @param o the new String to assign to this. An argument of null is equal to calling clear().
+     */
+    @Override
+    public void assign(Object o) {
+        if (spanTrees != null) {
+            spanTrees.clear();
+            spanTrees = null;
+        }
+
+        if (!checkAssign(o)) {
+            return;
+        }
+        if (o instanceof StringFieldValue) {
+            spanTrees=((StringFieldValue)o).spanTrees;
+        }
+        if (o instanceof String) {
+            setValue((String) o);
+        } else if (o instanceof StringFieldValue || o instanceof NumericFieldValue) {
+            setValue(o.toString());
+        } else {
+            throw new IllegalArgumentException("Class " + o.getClass() + " not applicable to an " + this.getClass() + " instance.");
+        }
+    }
+
+    /**
+     * Returns an unmodifiable Collection of the span trees with annotations over this String, if any.
+     *
+     * @return an unmodifiable Collection of the span trees with annotations over this String, or an empty Collection
+     */
+    public Collection<SpanTree> getSpanTrees() {
+        if (spanTrees == null) {
+            return ImmutableList.of();
+        }
+        return ImmutableList.copyOf(spanTrees.values());
+    }
+
+    /**
+     *
+     * @return The map of spantrees. Might be null.
+     */
+    public final Map<String, SpanTree> getSpanTreeMap() {
+        return spanTrees;
+    }
+
+    /**
+     * Returns the span tree associated with the given name, or null if this does not exist.
+     *
+     * @param name the name of the span tree to return
+     * @return the span tree associated with the given name, or null if this does not exist.
+     */
+    public SpanTree getSpanTree(String name) {
+        if (spanTrees == null) {
+            return null;
+        }
+        return spanTrees.get(name);
+    }
+
+    /**
+     * Sets the span tree with annotations over this String.
+     *
+     * @param spanTree the span tree with annotations over this String
+     * @return the input spanTree for chaining
+     * @throws IllegalArgumentException if a span tree with the given name already exists.
+     */
+    public SpanTree setSpanTree(SpanTree spanTree) {
+        if (spanTrees == null) {
+            spanTrees = new HashMap(1);
+        }
+        if (spanTrees.containsKey(spanTree.getName())) {
+            throw new IllegalArgumentException("Span tree " + spanTree.getName() + " already exists.");
+        }
+        spanTrees.put(spanTree.getName(), spanTree);
+        spanTree.setStringFieldValue(this);
+        return spanTree;
+    }
+
+    /**
+     * Removes the span tree associated with the given name.
+     *
+     * @param name the name of the span tree to remove
+     * @return the span tree previously associated with the given name, or null if it did not exist.
+     */
+    public SpanTree removeSpanTree(String name) {
+        if (spanTrees == null) {
+            return null;
+        }
+        SpanTree tree = spanTrees.remove(name);
+        if (tree != null) {
+            tree.setStringFieldValue(null);
+        }
+        return tree;
+    }
+
+    /**
+     * Returns the String value wrapped by this StringFieldValue.
+     *
+     * @return the String value wrapped by this StringFieldValue.
+     */
+    public String getString() {
+        return value;
+    }
+
+    /**
+     * Returns the String value wrapped by this StringFieldValue.
+     *
+     * @return the String value wrapped by this StringFieldValue.
+     */
+    @Override
+    public Object getWrappedValue() {
+        return value;
+    }
+
+    /**
+     * Prints XML in Vespa Document XML format for this StringFieldValue.
+     *
+     * @param xml the stream to print to.
+     */
+    @Override
+    public void printXml(XmlStream xml) {
+        XmlSerializationHelper.printStringXml(this, xml);
+        //TODO: add spanTree printing
+    }
+
+    /**
+     * Returns the String value wrapped by this StringFieldValue.
+     *
+     * @return the String value wrapped by this StringFieldValue.
+     */
+    @Override
+    public String toString() {
+        return value;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (!(o instanceof StringFieldValue)) return false;
+        if (!super.equals(o)) return false;
+        StringFieldValue that = (StringFieldValue) o;
+        if ((spanTrees != null) ? !spanTrees.equals(that.spanTrees) : that.spanTrees != null) return false;
+        if ((value != null) ? !value.equals(that.value) : that.value != null) return false;
+        return true;
+    }
+
+    @Override
+    public int hashCode() {
+        return (value != null) ? value.hashCode() : super.hashCode();
+    }
+
+    @Override
+    public void serialize(Field field, FieldWriter writer) {
+        writer.write(field, this);
+    }
+
+    @Override
+    public void deserialize(Field field, FieldReader reader) {
+        reader.read(field, this);
+    }
+
+    @Override
+    public int compareTo(FieldValue fieldValue) {
+        int comp = super.compareTo(fieldValue);
+
+        if (comp != 0) {
+            return comp;
+        }
+
+        //types are equal, this must be of this type
+        StringFieldValue otherValue = (StringFieldValue) fieldValue;
+        comp = value.compareTo(otherValue.value);
+
+        if (comp != 0) {
+            return comp;
+        }
+
+        if (spanTrees == null) {
+            comp = (otherValue.spanTrees == null) ? 0 : -1;
+        } else {
+            if (otherValue.spanTrees == null) {
+                comp = 1;
+            } else {
+                comp = CollectionComparator.compare(spanTrees.keySet(), otherValue.spanTrees.keySet());
+                if (comp != 0) {
+                    return comp;
+                }
+                comp = CollectionComparator.compare(spanTrees.values(), otherValue.spanTrees.values());
+            }
+        }
+        return comp;
+    }
+
+    /**
+     * Only for use by deserializer to avoid the cost of verifying input.
+     */
+    public void setUnChecked(String s) {
+        value = s;
+    }
+
+}
diff --git a/document/src/main/java/com/yahoo/document/datatypes/Struct.java b/document/src/main/java/com/yahoo/document/datatypes/Struct.java
new file mode 100644
index 00000000000..5a01dc33aa1
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/datatypes/Struct.java
@@ -0,0 +1,391 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.datatypes;
+
+import com.yahoo.collections.Hashlet;
+import com.yahoo.document.*;
+import com.yahoo.document.serialization.FieldReader;
+import com.yahoo.document.serialization.FieldWriter;
+import com.yahoo.document.serialization.XmlSerializationHelper;
+import com.yahoo.document.serialization.XmlStream;
+import com.yahoo.vespa.objects.Ids;
+
+import java.util.*;
+
+/**
+ * Date: Apr 15, 2008
+ *
+ * @author humbe
+ */
+public class Struct extends StructuredFieldValue {
+
+    public static final int classId = registerClass(Ids.document + 33, Struct.class);
+    private Hashlet<Integer, FieldValue> values = new Hashlet<>();
+    private int [] order = null;
+
+    private int version;
+
+    private int [] getInOrder() {
+        if (order == null) {
+            order = new int[values.size()];
+            for (int i = 0; i < values.size(); i++) {
+                order[i] = values.key(i);
+            }
+            Arrays.sort(order);
+        }
+        return order;
+    }
+
+    private void invalidateOrder() {
+        order = null;
+    }
+
+    public Struct(DataType type) {
+        super((StructDataType) type);
+        this.version = Document.SERIALIZED_VERSION;
+    }
+
+    @Override
+    public StructDataType getDataType() {
+        return (StructDataType)super.getDataType();
+    }
+
+    public void setVersion(int version) {
+        this.version = version;
+    }
+
+    public int getVersion() {
+        return this.version;
+    }
+
+    public com.yahoo.compress.CompressionType getCompressionType() {
+        if (getDataType().getCompressionConfig() == null) {
+            return com.yahoo.compress.CompressionType.NONE;
+        }
+        return getDataType().getCompressionConfig().type;
+    }
+
+    public int getCompressionLevel() {
+        if ( getDataType().getCompressionConfig() == null) {
+            return 9;
+        }
+        return getDataType().getCompressionConfig().compressionLevel;
+    }
+
+    public float getCompressionThreshold() {
+        if (getDataType().getCompressionConfig() == null) {
+            return .95f;
+        }
+        return getDataType().getCompressionConfig().threshold;
+    }
+
+    @Override
+    public Struct clone() {
+        Struct struct = (Struct) super.clone();
+        struct.values = new Hashlet<>();
+        struct.values.reserve(values.size());
+        for (int i = 0; i < values.size(); i++) {
+            struct.values.put(values.key(i), values.value(i).clone());
+        }
+        return struct;
+    }
+
+    @Override
+    public void clear() {
+        values = new Hashlet<>();
+        invalidateOrder();
+    }
+
+    @Override
+    public Iterator<Map.Entry<Field, FieldValue>> iterator() {
+        return new FieldSet().iterator();
+    }
+
+    public Set<Map.Entry<Field, FieldValue>> getFields() {
+        return new FieldSet();
+    }
+
+    @Override
+    public void printXml(XmlStream xml) {
+        if (getDataType().equals(PositionDataType.INSTANCE)) {
+            try {
+                PositionDataType.renderXml(this, xml);
+                return;
+            } catch (RuntimeException e) {
+                // fallthrough to handling below
+            }
+        }
+        XmlSerializationHelper.printStructXml(this, xml);
+    }
+
+    @Override
+    public FieldValue getFieldValue(Field field) {
+        return values.get(field.getId());
+    }
+
+
+    @Override
+    public Field getField(String fieldName) {
+        return getDataType().getField(fieldName);
+    }
+
+    @Override
+    public int getFieldCount() {
+        return values.size();
+    }
+
+    @Override
+    protected void doSetFieldValue(Field field, FieldValue value) {
+        if (field == null) {
+            throw new IllegalArgumentException("Invalid null field pointer");
+        }
+        Field myField = getDataType().getField(field.getId());
+        if (myField==null) {
+            throw new IllegalArgumentException("No such field in "+getDataType()+" : "+field.getName());
+        }
+        if (!myField
+                   .getDataType().isValueCompatible(value)) {
+            throw new IllegalArgumentException(
+                    "Incompatible data types. Got " + value.getDataType()
+                    + ", expected "
+                    + myField.getDataType());
+        }
+
+        if (myField.getId()
+                != field.getId()) {
+            throw new IllegalArgumentException(
+                    "Inconsistent field: " + field);
+        }
+
+        int index = values.getIndexOfKey(field.getId());
+        if (index == -1) {
+            values.put(field.getId(), value);
+            invalidateOrder();
+        } else {
+            values.setValue(index, value);
+        }
+    }
+
+    @Override
+    public FieldValue removeFieldValue(Field field) {
+        FieldValue found = values.get(field.getId());
+        if (found != null) {
+            Hashlet<Integer, FieldValue> copy = new Hashlet<>();
+            copy.reserve(values.size() - 1);
+            for (int i=0; i < values.size(); i++) {
+                if (values.key(i) != field.getId()) {
+                    copy.put(values.key(i), values.value(i));
+                }
+            }
+            values = copy;
+            invalidateOrder();
+        }
+        return found;
+    }
+
+    @Override
+    public void assign(Object o) {
+        if ((o instanceof Struct) && ((Struct) o).getDataType().equals(getDataType())) {
+            clear();
+            Iterator<Map.Entry<Field,FieldValue>> otherValues = ((Struct) o).iterator();
+            while (otherValues.hasNext()) {
+                Map.Entry<Field, FieldValue> otherEntry = otherValues.next();
+                setFieldValue(otherEntry.getKey(), otherEntry.getValue());
+            }
+        } else {
+            throw new IllegalArgumentException("Type " + o.getClass() + " can not specify a " + getClass() + " instance");
+        }
+    }
+
+    /**
+     * Clears this and assigns from the given {@link StructuredFieldValue}
+     */
+    public void assignFrom(StructuredFieldValue sfv) {
+        clear();
+        Iterator<Map.Entry<Field,FieldValue>> otherValues = sfv.iterator();
+        while (otherValues.hasNext()) {
+            Map.Entry<Field, FieldValue> otherEntry = otherValues.next();
+            setFieldValue(otherEntry.getKey(), otherEntry.getValue());
+        }
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (!(o instanceof Struct)) return false;
+        if (!super.equals(o)) return false;
+
+        Struct struct = (Struct) o;
+
+        return values.equals(struct.values);
+    }
+
+    @Override
+    public int hashCode() {
+        int result = super.hashCode();
+        result = 31 * result + values.hashCode();
+        return result;
+    }
+
+    @Override
+    public String toString() {
+        StringBuilder retVal = new StringBuilder();
+        retVal.append("Struct (").append(getDataType()).append("): ");
+        int [] increasing = getInOrder();
+        for (int i = 0; i < increasing.length; i++) {
+            int id = increasing[i];
+            retVal.append(getDataType().getField(id)).append("=").append(values.get(id)).append(", ");
+        }
+        return retVal.toString();
+    }
+
+    @Override
+    public void serialize(Field field, FieldWriter writer) {
+        writer.write(field, this);
+    }
+
+    @Override
+    public int compareTo(FieldValue obj) {
+        int cmp = super.compareTo(obj);
+        if (cmp != 0) {
+            return cmp;
+        }
+        Struct rhs = (Struct)obj;
+        cmp = values.size() - rhs.values.size();
+        if (cmp != 0) {
+            return cmp;
+        }
+        StructDataType type = getDataType();
+        for (Field field : type.getFields()) {
+            FieldValue lhsField = getFieldValue(field);
+            FieldValue rhsField = rhs.getFieldValue(field);
+            if (lhsField != null && rhsField != null) {
+                cmp = lhsField.compareTo(rhsField);
+                if (cmp != 0) {
+                    return cmp;
+                }
+            } else if (lhsField != null || rhsField != null) {
+                return (lhsField != null ? -1 : 1);
+            }
+        }
+        return 0;
+    }
+
+    /*
+     * (non-Javadoc)
+     * @see com.yahoo.document.datatypes.FieldValue#deserialize(com.yahoo.document.Field, com.yahoo.document.serialization.FieldReader)
+     */
+    @Override
+    public void deserialize(Field field, FieldReader reader) {
+        reader.read(field, this);
+    }
+
+    private class FieldEntry implements Map.Entry<Field, FieldValue> {
+        private int id;
+
+        private FieldEntry(int id) {
+            this.id = id;
+        }
+
+        public Field getKey() {
+            return getDataType().getField(id);
+        }
+
+        public FieldValue getValue() {
+            return values.get(id);
+        }
+
+        public FieldValue setValue(FieldValue value) {
+            if (value == null) {
+                throw new NullPointerException("Null values in Struct not supported, use removeFieldValue() to remove value instead.");
+            }
+
+            int index = values.getIndexOfKey(id);
+            FieldValue retVal = null;
+            if (index == -1) {
+                values.put(id, value);
+                invalidateOrder();
+            } else {
+                retVal = values.value(index);
+                values.setValue(index, value);
+            }
+
+            return retVal;
+        }
+
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (!(o instanceof FieldEntry)) return false;
+
+            FieldEntry that = (FieldEntry) o;
+            return (id == that.id);
+        }
+
+        public int hashCode() {
+            return id;
+        }
+    }
+
+    private class FieldSet extends AbstractSet<Map.Entry<Field, FieldValue>> {
+        @Override
+        public int size() {
+            return values.size();
+        }
+
+        @Override
+        public Iterator<Map.Entry<Field, FieldValue>> iterator() {
+            return new FieldSetIterator();
+        }
+
+
+    }
+
+    private class FieldSetIterator implements Iterator<Map.Entry<Field, FieldValue>> {
+        private int position = 0;
+        private int [] increasing = getInOrder();
+
+        public boolean hasNext() {
+            return (position < increasing.length);
+        }
+
+        public Map.Entry<Field, FieldValue> next() {
+            if (position >= increasing.length) {
+                throw new NoSuchElementException("No more elements in collection");
+            }
+            FieldEntry retval = new FieldEntry(increasing[position]);
+            position++;
+            return retval;
+        }
+
+        public void remove() {
+            throw new UnsupportedOperationException("The set of fields and values of this struct is unmodifiable when accessed through this method.");
+        }
+    }
+
+    public static <T> T getFieldValue(FieldValue struct, DataType structType, String fieldName, Class<T> fieldType) {
+        if (!(struct instanceof Struct)) {
+            return null;
+        }
+        if (!struct.getDataType().equals(structType)) {
+            return null;
+        }
+        FieldValue fieldValue = ((Struct)struct).getFieldValue(fieldName);
+        if (!fieldType.isInstance(fieldValue)) {
+            return null;
+        }
+        return fieldType.cast(fieldValue);
+    }
+
+    public static <T> T getFieldValue(FieldValue struct, DataType structType, Field field, Class<T> fieldType) {
+        if (!(struct instanceof Struct)) {
+            return null;
+        }
+        if (!struct.getDataType().equals(structType)) {
+            return null;
+        }
+        FieldValue fieldValue = ((Struct)struct).getFieldValue(field);
+        if (!fieldType.isInstance(fieldValue)) {
+            return null;
+        }
+        return fieldType.cast(fieldValue);
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/datatypes/StructuredFieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/StructuredFieldValue.java
new file mode 100644
index 00000000000..b4585a2188d
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/datatypes/StructuredFieldValue.java
@@ -0,0 +1,235 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.datatypes;
+
+import com.yahoo.document.*;
+import com.yahoo.vespa.objects.Ids;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * @author <a href="mailto:humbe@yahoo-inc.com">H&aring;kon Humberset</a>
+ */
+public abstract class StructuredFieldValue extends CompositeFieldValue {
+
+    public static final int classId = registerClass(Ids.document + 32, StructuredFieldValue.class);
+
+    protected StructuredFieldValue(StructuredDataType type) {
+        super(type);
+    }
+
+    @Override
+    public StructuredDataType getDataType() {
+        return (StructuredDataType)super.getDataType();
+    }
+
+    /**
+     * Returns the named field object, or null if that field does not exist.
+     *
+     * @param fieldName The name of the field to return.
+     * @return The corresponding field, or null.
+     */
+    public abstract Field getField(String fieldName);
+
+    /**
+     * Returns the value of the given field. If the field does not exist, this method returns null.
+     *
+     * @param field The field whose value to return.
+     * @return The value of the field, or null.
+     */
+    public abstract FieldValue getFieldValue(Field field);
+
+    /**
+     * Convenience method to return the value of a named field. This is the same as calling {@link #getField(String)},
+     * and using the returned value to call {@link #getFieldValue(Field)}. If the named field does not exist, this
+     * method returns null.
+     *
+     * @param fieldName The name of the field whose value to return.
+     * @return The value of the field, or null.
+     */
+    public FieldValue getFieldValue(String fieldName) {
+        Field field = getField(fieldName);
+        if (field == null) {
+            return null;
+        }
+        return getFieldValue(field);
+    }
+
+    /**
+     * Sets the value of the given field. The type of the value must match the type of this field, i.e.
+     * <pre>field.getDataType().getValueClass().isAssignableFrom(value.getClass())</pre> must be true.
+     *
+     * @param field The field whose value to set.
+     * @param value The value to set.
+     * @return The previous value of the field, or null.
+     * @throws IllegalArgumentException If the value is not compatible with the field.
+     */
+    public FieldValue setFieldValue(Field field, FieldValue value) {
+        if (value == null) {
+            return removeFieldValue(field);
+        }
+        DataType type = field.getDataType();
+        if (!type.getValueClass().isAssignableFrom(value.getClass())) {
+            FieldValue tmp = type.createFieldValue();
+            tmp.assign(value);
+            value = tmp;
+        }
+        FieldValue ret = getFieldValue(field);
+        doSetFieldValue(field, value);
+        return ret;
+    }
+
+    protected abstract void doSetFieldValue(Field field, FieldValue value);
+
+    /**
+     * Convenience method to set the value of a named field. This is the same as calling {@link #getField(String)}, and
+     * using the returned value to call {@link #setFieldValue(Field, FieldValue)}. If the named field does not exist,
+     * this method returns null.
+     *
+     * @param fieldName The name of the field whose value to set.
+     * @param value     The value to set.
+     * @return The previous value of the field, or null.
+     */
+    public FieldValue setFieldValue(String fieldName, FieldValue value) {
+        Field field = getField(fieldName);
+        if (field == null) {
+            return null;
+        }
+        return setFieldValue(field, value);
+    }
+
+    public final FieldValue setFieldValue(Field field, String value) {
+        return setFieldValue(field, new StringFieldValue(value));
+    }
+
+    public final FieldValue setFieldValue(Field field, Double value) {
+        return setFieldValue(field, new DoubleFieldValue(value));
+    }
+
+    public final FieldValue setFieldValue(Field field, Integer value) {
+        return setFieldValue(field, new IntegerFieldValue(value));
+    }
+
+    public final FieldValue setFieldValue(Field field, Long value) {
+        return setFieldValue(field, new LongFieldValue(value));
+    }
+
+    public final FieldValue setFieldValue(Field field, Byte value) {
+        return setFieldValue(field, new ByteFieldValue(value));
+    }
+
+    public final FieldValue setFieldValue(String field, String value) {
+        return setFieldValue(field, new StringFieldValue(value));
+    }
+
+    public final FieldValue setFieldValue(String field, Double value) {
+        return setFieldValue(field, new DoubleFieldValue(value));
+    }
+
+    public final FieldValue setFieldValue(String field, Integer value) {
+        return setFieldValue(field, new IntegerFieldValue(value));
+    }
+
+    public final FieldValue setFieldValue(String field, Long value) {
+        return setFieldValue(field, new LongFieldValue(value));
+    }
+
+    public final FieldValue setFieldValue(String field, Byte value) {
+        return setFieldValue(field, new ByteFieldValue(value));
+    }
+    /**
+     * Removes and returns a field value.
+     *
+     * @param field The field whose value to remove.
+     * @return The previous value of the field, or null.
+     */
+    public abstract FieldValue removeFieldValue(Field field);
+
+    /**
+     * Convenience method to remove the value of a named field. This is the same as calling {@link #getField(String)},
+     * and using the returned value to call {@link #removeFieldValue(Field)}. If the named field does not exist, this
+     * method returns null.
+     *
+     * @param fieldName The name of the field whose value to remove.
+     * @return The previous value of the field, or null.
+     */
+    public FieldValue removeFieldValue(String fieldName) {
+        Field field = getField(fieldName);
+        if (field == null) {
+            return null;
+        }
+        return removeFieldValue(field);
+    }
+
+    public abstract void clear();
+
+    public abstract int getFieldCount();
+
+    public abstract Iterator<Map.Entry<Field, FieldValue>> iterator();
+
+    @Override
+    public FieldPathIteratorHandler.ModificationStatus iterateNested(FieldPath fieldPath, int pos,
+                                                                     FieldPathIteratorHandler handler) {
+        if (pos < fieldPath.size()) {
+            if (fieldPath.get(pos).getType() == FieldPathEntry.Type.STRUCT_FIELD) {
+                FieldValue fieldVal = getFieldValue(fieldPath.get(pos).getFieldRef());
+                if (fieldVal != null) {
+                    FieldPathIteratorHandler.ModificationStatus status = fieldVal.iterateNested(fieldPath, pos + 1, handler);
+                    if (status == FieldPathIteratorHandler.ModificationStatus.REMOVED) {
+                        removeFieldValue(fieldPath.get(pos).getFieldRef());
+                        return FieldPathIteratorHandler.ModificationStatus.MODIFIED;
+                    } else {
+                        if (isGenerated()) {
+                            // If this is a generated doc, the operations on the FieldValue in iterateNested do not write through to the doc,
+                            // so set the field again here. Should be a cleaner way to do this.
+                            setFieldValue(fieldPath.get(pos).getFieldRef(), fieldVal);
+                        }
+                        return status;
+                    }
+                } else if (handler.createMissingPath()) {
+                    FieldValue newVal = fieldPath.get(pos).getFieldRef().getDataType().createFieldValue();
+                    FieldPathIteratorHandler.ModificationStatus status = newVal.iterateNested(fieldPath, pos + 1, handler);
+                    if (status == FieldPathIteratorHandler.ModificationStatus.MODIFIED) {
+                        setFieldValue(fieldPath.get(pos).getFieldRef(), newVal);
+                        return status;
+                    }
+                }
+                return FieldPathIteratorHandler.ModificationStatus.NOT_MODIFIED;
+            }
+            throw new IllegalArgumentException("Illegal field path " + fieldPath.get(pos) + " for struct value");
+        } else {
+            FieldPathIteratorHandler.ModificationStatus status = handler.modify(this);
+            if (status == FieldPathIteratorHandler.ModificationStatus.REMOVED) {
+                return status;
+            }
+            if (handler.onComplex(this)) {
+                List<Field> fieldsToRemove = new ArrayList<Field>();
+                for (Iterator<Map.Entry<Field, FieldValue>> iter = iterator(); iter.hasNext();) {
+                    Map.Entry<Field, FieldValue> entry = iter.next();
+                    FieldPathIteratorHandler.ModificationStatus currStatus = entry.getValue().iterateNested(fieldPath, pos, handler);
+                    if (currStatus == FieldPathIteratorHandler.ModificationStatus.REMOVED) {
+                        fieldsToRemove.add(entry.getKey());
+                        status = FieldPathIteratorHandler.ModificationStatus.MODIFIED;
+                    } else if (currStatus == FieldPathIteratorHandler.ModificationStatus.MODIFIED) {
+                        status = currStatus;
+                    }
+                }
+                for (Field field : fieldsToRemove) {
+                    removeFieldValue(field);
+                }
+            }
+            return status;
+        }
+    }
+
+    /**
+     * Generated Document subclasses should override this and return true. This is used instead of using class.getAnnotation(Generated.class), because that is so slow.
+     * @return true if in a concrete subtype of Document
+     */
+    protected boolean isGenerated() {
+        return false;
+    }
+
+}
diff --git a/document/src/main/java/com/yahoo/document/datatypes/TensorFieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/TensorFieldValue.java
new file mode 100644
index 00000000000..82e3ff36c1c
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/datatypes/TensorFieldValue.java
@@ -0,0 +1,100 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.datatypes;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.Field;
+import com.yahoo.document.PrimitiveDataType;
+import com.yahoo.document.serialization.FieldReader;
+import com.yahoo.document.serialization.FieldWriter;
+import com.yahoo.document.serialization.XmlStream;
+import com.yahoo.tensor.Tensor;
+
+import java.util.Objects;
+import java.util.Optional;
+
+/**
+ * Field value class that wraps a tensor.
+ *
+ * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a>
+ */
+public class TensorFieldValue extends FieldValue {
+
+    private Optional<Tensor> tensor;
+
+    public TensorFieldValue() {
+        tensor = Optional.empty();
+    }
+
+    public TensorFieldValue(Tensor tensor) {
+        this.tensor = Optional.of(tensor);
+    }
+
+    public Optional<Tensor> getTensor() {
+        return tensor;
+    }
+
+    @Override
+    public DataType getDataType() {
+        return DataType.TENSOR;
+    }
+
+    @Override
+    public void printXml(XmlStream xml) {
+        // TODO (geirst)
+    }
+
+    @Override
+    public void clear() {
+        tensor = Optional.empty();
+    }
+
+    @Override
+    public void assign(Object o) {
+        if (o == null) {
+            tensor = Optional.empty();
+        } else if (o instanceof Tensor) {
+            tensor = Optional.of((Tensor)o);
+        } else if (o instanceof TensorFieldValue) {
+            tensor = ((TensorFieldValue)o).getTensor();
+        } else {
+            throw new IllegalArgumentException("Expected class '" + getClass().getName() + "', got '" +
+                    o.getClass().getName() + "'.");
+        }
+    }
+
+    @Override
+    public void serialize(Field field, FieldWriter writer) {
+        writer.write(field, this);
+    }
+
+    @Override
+    public void deserialize(Field field, FieldReader reader) {
+        reader.read(field, this);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (!(o instanceof TensorFieldValue)) {
+            return false;
+        }
+        TensorFieldValue rhs = (TensorFieldValue)o;
+        if (!Objects.equals(tensor, rhs.tensor)) {
+            return false;
+        }
+        return true;
+    }
+
+    public static PrimitiveDataType.Factory getFactory() {
+        return new PrimitiveDataType.Factory() {
+
+            @Override
+            public FieldValue create() {
+                return new TensorFieldValue();
+            }
+        };
+    }
+}
+
diff --git a/document/src/main/java/com/yahoo/document/datatypes/UriFieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/UriFieldValue.java
new file mode 100644
index 00000000000..a1e0b12b6a0
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/datatypes/UriFieldValue.java
@@ -0,0 +1,49 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.datatypes;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.Field;
+import com.yahoo.document.PrimitiveDataType;
+import com.yahoo.document.serialization.FieldReader;
+import com.yahoo.net.Url;
+
+import java.net.URI;
+
+/**
+ * Created with IntelliJ IDEA.
+ * User: magnarn
+ * Date: 11/2/12
+ * Time: 2:37 PM
+ */
+public class UriFieldValue extends StringFieldValue {
+    public static class Factory extends PrimitiveDataType.Factory {
+        public FieldValue create() {
+            return new UriFieldValue();
+        }
+    }
+    public UriFieldValue() { super(); }
+
+    public UriFieldValue(String value) {
+        super(value);
+        Url.fromString(value);  // Throws if value is invalid.
+    }
+
+    @Override
+    public void assign(Object obj) {
+        if (obj instanceof URI) {
+            obj = obj.toString();
+        }
+        super.assign(obj);
+    }
+
+    @Override
+    public DataType getDataType() {
+        return DataType.URI;
+    }
+
+    @Override
+    public void deserialize(Field field, FieldReader reader) {
+        super.deserialize(field, reader);
+        Url.fromString(toString());  // Throws if value is invalid.
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/datatypes/WeightedSet.java b/document/src/main/java/com/yahoo/document/datatypes/WeightedSet.java
new file mode 100644
index 00000000000..5e56f247a7f
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/datatypes/WeightedSet.java
@@ -0,0 +1,418 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.datatypes;
+
+import com.yahoo.collections.CollectionComparator;
+import com.yahoo.document.*;
+import com.yahoo.document.serialization.FieldReader;
+import com.yahoo.document.serialization.FieldWriter;
+import com.yahoo.document.serialization.XmlSerializationHelper;
+import com.yahoo.document.serialization.XmlStream;
+
+import java.util.*;
+
+/**
+ * A weighted set, a unique set of keys with an associated integer weight. This class
+ * uses an encapsulated Map (actually a LinkedHashMap) that associates each key
+ * with its weight (value).
+ *
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public final class WeightedSet<K extends FieldValue> extends CollectionFieldValue<K> implements Map<K, Integer> {
+
+    private MapFieldValue<K, IntegerFieldValue> map;
+
+    /**
+     * Creates a new WeightedSet.
+     *
+     * @param type the data type for the field that this weighted set is associated with
+     */
+    public WeightedSet(DataType type) {
+        this(type, 1);
+    }
+
+    /**
+     * Creates a new weighted set with a given initial capacity.
+     *
+     * @param initialCapacity the initial capacity to use for the encapsulated Map
+     */
+    public WeightedSet(DataType type, int initialCapacity) {
+        super((WeightedSetDataType) type);
+        clearAndReserve(initialCapacity);
+    }
+
+    @Override
+    public WeightedSetDataType getDataType() {
+        return (WeightedSetDataType) super.getDataType();
+    }
+
+    @Override
+    public Iterator<K> fieldValueIterator() {
+        return map.keySet().iterator();
+    }
+
+    @Override
+    public void assign(Object o) {
+        if (!checkAssign(o)) {
+            return;
+        }
+
+        if (o instanceof WeightedSet) {
+            WeightedSet wset = (WeightedSet) o;
+            if (getDataType().equals(wset.getDataType())) {
+                map.assign(wset.map);
+            } else {
+                throw new IllegalArgumentException("Cannot assign a weighted set of type " + wset.getDataType()
+                                                   + " to a weighted set of type " + getDataType());
+            }
+        } else if (o instanceof Map) {
+            map = new WeightedSetWrapper((Map)o, map.getDataType());
+        } else {
+            throw new IllegalArgumentException("Class " + o.getClass() + " not applicable to an " + this.getClass() + " instance.");
+        }
+    }
+
+    @Override
+    public WeightedSet clone() {
+        WeightedSet<K> newSet = (WeightedSet<K>) super.clone();
+        newSet.map = (MapFieldValue<K, IntegerFieldValue>) map.clone();
+        return newSet;
+    }
+
+
+    @Override
+    public void printXml(XmlStream xml) {
+        XmlSerializationHelper.printWeightedSetXml(this, xml);
+    }
+
+    /**
+     * Returns the number of key-weight pairs in this set.
+     *
+     * @return the number of key-weight pairs in this set
+     */
+    public int size() {
+        return map.size();
+    }
+
+    public boolean add(K value) {
+        put(value, 1);
+        return true;
+    }
+
+    @Override
+    public Object getWrappedValue() {
+        if (map instanceof WeightedSet.WeightedSetWrapper) {
+            return ((WeightedSet.WeightedSetWrapper) map).map;
+        }
+        return map.getWrappedValue();
+    }
+
+    @Override
+    public boolean contains(Object o) {
+        return map.keySet().contains(o);
+    }
+
+    /**
+     * Checks if this set is empty.
+     *
+     * @return true if the set is empty
+     */
+    public boolean isEmpty() {
+        return map.isEmpty();
+    }
+
+    @Override
+    public Iterator<K> iterator() {
+        return map.keySet().iterator();
+    }
+
+    @Override
+    public boolean removeValue(FieldValue o) {
+        return super.removeValue(o, map.keySet());
+    }
+
+    /**
+     * Checks whether this set contains the specified key.
+     *
+     * @param key the key to search for
+     * @return true if this set contains this key
+     */
+    public boolean containsKey(Object key) {
+        return map.containsKey(key);
+    }
+
+    public boolean containsValue(Object value) {
+        return map.containsValue(value);
+    }
+
+    /**
+     * Returns the weight associated with the specified key.
+     *
+     * @param key the key to return the weight for
+     * @return the weight associated with the specified key, or null (if not found)
+     */
+    public Integer get(Object key) {
+        if (!(key instanceof FieldValue)) {
+            throw new IllegalArgumentException("Only FieldValues are allowed as keys.");
+        }
+        IntegerFieldValue ifv = map.get(key);
+        return ifv != null ? ifv.getInteger() : null;
+    }
+
+    /**
+     * Add a key with an associated weight to this set. If the key is already present in this set, the previous
+     * association is replaced. Checks to validate that all keys are of the same type.
+     *
+     * @param key    the key to add
+     * @param weight the weight to associate with this key
+     * @return the weight that was previously associated with this key, or null (if there was no previous key)
+     */
+    public Integer put(K key, Integer weight) {
+        verifyElementCompatibility(key);
+        IntegerFieldValue ifv =  putUnChecked(key, new IntegerFieldValue(weight));
+        return ifv != null ? ifv.getInteger() : null;
+    }
+
+    /**
+     * Add a key with an associated weight to this set. If the key is already present in this set, the previous
+     * association is replaced.
+     *
+     * @param key    the key to add
+     * @param weight the weight to associate with this key
+     * @return the weight that was previously associated with this key, or null (if there was no previous key)
+     */
+    public IntegerFieldValue putUnChecked(K key, IntegerFieldValue weight) {
+        return map.put(key, weight);
+    }
+
+    /**
+     * Remove a key-weight association from this set.
+     *
+     * @param key the key to remove
+     * @return the weight that was previously associated with this key, or null (if there was no previous key)
+     */
+    public Integer remove(Object key) {
+        IntegerFieldValue ifv = map.remove(key);
+        return ifv != null ? ifv.getInteger() : null;
+    }
+
+    public void putAll(Map<? extends K, ? extends Integer> t) {
+        for (Entry<? extends K, ? extends Integer> entry : t.entrySet()) {
+            put(entry.getKey(), entry.getValue());
+        }
+    }
+
+    /** Remove all key-weight associations in this set. */
+    public void clear() {
+        map.clear();
+    }
+
+    /**
+     * Reserve space for this amount of keys in order to avoid resizing
+     */
+    public void clearAndReserve(int count) {
+        map = new MapFieldValue<>(new MapDataType(getDataType().getNestedType(), DataType.INT), count);
+    }
+
+    Map<K, Integer> getPrimitiveMap() {
+        Map<K, Integer> retVal = new LinkedHashMap<>();
+        for (Entry<K, IntegerFieldValue> entry : map.entrySet()) {
+            retVal.put(entry.getKey(), entry.getValue().getInteger());
+        }
+        return retVal;
+    }
+
+    public Collection<Integer> values() {
+        return getPrimitiveMap().values();
+    }
+
+    public Set<K> keySet() {
+        return map.keySet();
+    }
+
+    public Set<Entry<K, Integer>> entrySet() {
+        return getPrimitiveMap().entrySet();
+    }
+
+    /**
+     * Checks if another object is equal to this set.
+     *
+     * @param o the object to check for equality with
+     * @return true if o is an instance of WeightedSet and the two encapsulated Maps are equal, false otherwise
+     */
+    public boolean equals(Object o) {
+        if (!(o instanceof WeightedSet)) return false;
+        WeightedSet otherSet = (WeightedSet) o;
+        return (super.equals(o) && map.equals(otherSet.map));
+    }
+
+    /**
+     * Uses hashCode() from the encapsulated Map.
+     *
+     * @return the hash code of this set
+     */
+    public int hashCode() {
+        return map.hashCode();
+    }
+
+    /**
+     * Uses toString() from the encapsulated Map.
+     *
+     * @return the toString() of this set
+     */
+    public String toString() {
+        StringBuilder sb = new StringBuilder();
+        sb.append("WeightedSet(").append(getDataType());
+        for (Map.Entry entry : map.entrySet()) {
+            sb.append("\n  key:   ").append(entry.getKey().getClass()).append(": ").append(entry.getKey());
+            sb.append("\n  value: ").append(entry.getValue().getClass()).append(": ").append(entry.getValue());
+        }
+        return sb.append("\n)").toString();
+    }
+
+    @Override
+    public void serialize(Field field, FieldWriter writer) {
+        writer.write(field, this);
+    }
+
+    @Override
+    public void deserialize(Field field, FieldReader reader) {
+        reader.read(field, this);
+    }
+
+    @Override
+    FieldPathIteratorHandler.ModificationStatus iterateNested(FieldPath fieldPath, int pos, FieldPathIteratorHandler handler) {
+        FieldPathIteratorHandler.ModificationStatus status = map.iterateNested(fieldPath, pos, handler, this);
+        return status;
+    }
+
+    @Override
+    public int compareTo(FieldValue fieldValue) {
+        int comp = super.compareTo(fieldValue);
+
+        if (comp != 0) {
+            return comp;
+        }
+
+        //types are equal, this must be of this type
+        WeightedSet otherValue = (WeightedSet) fieldValue;
+        comp = CollectionComparator.compare(map.keySet(), otherValue.map.keySet());
+
+        if (comp != 0) {
+            return comp;
+        }
+
+        return CollectionComparator.compare(map.values(), otherValue.map.values());
+    }
+
+
+    /**
+     * Weighted set MapFieldValue, backed by map of native Java types.
+     * Note: The key type of this is FieldValue, not K.
+     * @author vegardh
+     *
+     */
+    class WeightedSetWrapper extends MapFieldValue<K, IntegerFieldValue> {
+        Map<Object, Integer> map = new HashMap<Object, Integer>();
+        private DataType keyTypeVespa = getDataType().getKeyType();
+        private DataType valTypeVespa = DataType.INT;
+
+        public WeightedSetWrapper(Map map, MapDataType dt) {
+            super(dt);
+            this.map=map;
+        }
+
+        private Object unwrap(Object o) {
+            return (o instanceof FieldValue ? ((FieldValue) o).getWrappedValue() : o);
+        }
+
+        @SuppressWarnings("unchecked")
+        private K wrapKey(Object o) {
+            if (o==null) return null;
+            return (K) keyTypeVespa.createFieldValue(o);
+        }
+
+        private IntegerFieldValue wrapValue(Object o) {
+            if (o==null) return null;
+            return (IntegerFieldValue)valTypeVespa.createFieldValue(o);
+        }
+
+        @Override
+        public void clear() {
+            map.clear();
+        }
+
+        @Override
+        public boolean containsKey(Object key) {
+            return map.containsKey(unwrap(key));
+        }
+
+        @Override
+        public boolean containsValue(Object value) {
+            return map.containsValue(unwrap(value));
+        }
+
+        @Override
+        public Set<java.util.Map.Entry<K, IntegerFieldValue>> entrySet() {
+            Map<K, IntegerFieldValue> ret = new HashMap<>();
+            for (Map.Entry e : map.entrySet()) {
+                ret.put(wrapKey(e.getKey()), wrapValue(e.getValue()));
+            }
+            return ret.entrySet();
+        }
+
+        @Override
+        public IntegerFieldValue get(Object key) {
+            Object o = map.get(unwrap(key));
+            return o == null ? null : wrapValue(o);
+        }
+
+        @Override
+        public boolean isEmpty() {
+            return map.isEmpty();
+        }
+
+        @Override
+        public Set<K> keySet() {
+            Set<K> ret = new HashSet<>();
+            for (Map.Entry e : map.entrySet()) {
+                ret.add(wrapKey(e.getKey()));
+            }
+            return ret;
+        }
+
+        @Override
+        public IntegerFieldValue put(FieldValue key, IntegerFieldValue value) {
+            IntegerFieldValue old = get(key);
+            map.put(unwrap(key), (Integer) unwrap(value));
+            return old;
+        }
+
+        @Override
+        public void putAll(Map<? extends K, ? extends IntegerFieldValue> m) {
+            for (Map.Entry<?, ?> e : m.entrySet()) {
+                map.put(unwrap(e.getKey()), (Integer) unwrap(e.getValue()));
+            }
+        }
+
+        @Override
+        public IntegerFieldValue remove(Object key) {
+            return wrapValue(map.remove(unwrap(key)));
+        }
+
+        @Override
+        public int size() {
+            return map.size();
+        }
+
+        @Override
+        public Collection<IntegerFieldValue> values() {
+            Collection<IntegerFieldValue> ret = new ArrayList<>();
+            for (Object v : map.values()) {
+                ret.add(wrapValue(v));
+            }
+            return ret;
+        }
+
+    }
+
+}
diff --git a/document/src/main/java/com/yahoo/document/datatypes/package-info.java b/document/src/main/java/com/yahoo/document/datatypes/package-info.java
new file mode 100644
index 00000000000..67ed3c15393
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/datatypes/package-info.java
@@ -0,0 +1,7 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+@PublicApi
+package com.yahoo.document.datatypes;
+
+import com.yahoo.api.annotations.PublicApi;
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/document/src/main/java/com/yahoo/document/declaration/.gitignore b/document/src/main/java/com/yahoo/document/declaration/.gitignore
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/declaration/.gitignore
diff --git a/document/src/main/java/com/yahoo/document/fieldpathupdate/AddFieldPathUpdate.java b/document/src/main/java/com/yahoo/document/fieldpathupdate/AddFieldPathUpdate.java
new file mode 100644
index 00000000000..28673adbd88
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/fieldpathupdate/AddFieldPathUpdate.java
@@ -0,0 +1,123 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.fieldpathupdate;
+
+import com.yahoo.document.Document;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.datatypes.Array;
+import com.yahoo.document.datatypes.CollectionFieldValue;
+import com.yahoo.document.datatypes.FieldPathIteratorHandler;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.serialization.DocumentUpdateReader;
+import com.yahoo.document.serialization.VespaDocumentSerializerHead;
+
+/**
+ * @author <a href="mailto:thomasg@yahoo-inc.com">Thomas Gundersen</a>
+ */
+public class AddFieldPathUpdate extends FieldPathUpdate {
+    class IteratorHandler extends FieldPathIteratorHandler {
+        Array newValues;
+
+        IteratorHandler(Array newValues) {
+            this.newValues = newValues;
+        }
+
+        @SuppressWarnings({ "unchecked" })
+        @Override
+        public ModificationStatus doModify(FieldValue fv) {
+            for (Object newValue : newValues.getValues()) {
+                ((CollectionFieldValue)fv).add((FieldValue) newValue);
+            }
+            return ModificationStatus.MODIFIED;
+        }
+
+        @Override
+        public boolean createMissingPath() {
+            return true;
+        }
+
+        @Override
+        public boolean onComplex(FieldValue fv) {
+            return false;
+        }
+
+        public Array getNewValues() {
+            return newValues;
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+
+            IteratorHandler that = (IteratorHandler) o;
+
+            if (!newValues.equals(that.newValues)) return false;
+
+            return true;
+        }
+
+        @Override
+        public int hashCode() {
+            return newValues.hashCode();
+        }
+    }
+
+    IteratorHandler handler;
+
+    public AddFieldPathUpdate(DocumentType type, String fieldPath, String whereClause, Array newValues) {
+        super(FieldPathUpdate.Type.ADD, type, fieldPath, whereClause);
+        setNewValues(newValues);
+    }
+
+    public AddFieldPathUpdate(DocumentType type, String fieldPath, Array newValues) {
+        super(FieldPathUpdate.Type.ADD, type, fieldPath, null);
+        setNewValues(newValues);
+    }
+
+    public AddFieldPathUpdate(DocumentType type, DocumentUpdateReader reader) {
+        super(FieldPathUpdate.Type.ADD, type, reader);
+        reader.read(this);
+    }
+
+    public void setNewValues(Array value) {
+        handler = new IteratorHandler(value);
+    }
+
+    public Array getNewValues() {
+        return handler.getNewValues();
+    }
+
+    public FieldPathIteratorHandler getIteratorHandler(Document doc) {
+        return handler;
+    }
+
+    @Override
+    public void serialize(VespaDocumentSerializerHead data) {
+        data.write(this);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        if (!super.equals(o)) return false;
+
+        AddFieldPathUpdate that = (AddFieldPathUpdate) o;
+
+        if (!handler.equals(that.handler)) return false;
+
+        return true;
+    }
+
+    @Override
+    public int hashCode() {
+        int result = super.hashCode();
+        result = 31 * result + handler.hashCode();
+        return result;
+    }
+
+    @Override
+    public String toString() {
+        return "Add: " + super.toString() + " : " + handler.getNewValues();
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/fieldpathupdate/AssignFieldPathUpdate.java b/document/src/main/java/com/yahoo/document/fieldpathupdate/AssignFieldPathUpdate.java
new file mode 100644
index 00000000000..b799a56197f
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/fieldpathupdate/AssignFieldPathUpdate.java
@@ -0,0 +1,281 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.fieldpathupdate;
+
+import com.yahoo.document.Document;
+import com.yahoo.document.DocumentCalculator;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.datatypes.FieldPathIteratorHandler;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.NumericFieldValue;
+import com.yahoo.document.select.parser.ParseException;
+import com.yahoo.document.serialization.DocumentUpdateReader;
+import com.yahoo.document.serialization.VespaDocumentSerializerHead;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * @author <a href="mailto:thomasg@yahoo-inc.com">Thomas Gundersen</a>
+ */
+public class AssignFieldPathUpdate extends FieldPathUpdate {
+    class SimpleAssignIteratorHandler extends FieldPathIteratorHandler {
+        FieldValue newValue;
+        boolean removeIfZero;
+        boolean createMissingPath;
+
+        SimpleAssignIteratorHandler(FieldValue newValue, boolean removeIfZero, boolean createMissingPath) {
+            this.newValue = newValue;
+            this.removeIfZero = removeIfZero;
+            this.createMissingPath = createMissingPath;
+        }
+
+        @Override
+        public ModificationStatus doModify(FieldValue fv) {
+            if (!fv.getDataType().equals(newValue.getDataType())) {
+                throw new IllegalArgumentException("Trying to assign " + newValue + " of type " + newValue.getDataType() + " to an instance of " + fv.getDataType());
+            } else {
+                if (removeIfZero && (newValue instanceof NumericFieldValue) && ((NumericFieldValue)newValue).getNumber().longValue() == 0) {
+                    return ModificationStatus.REMOVED;
+                }
+                fv.assign(newValue);
+            }
+            return ModificationStatus.MODIFIED;
+        }
+
+        @Override
+        public boolean createMissingPath() {
+            return createMissingPath;
+        }
+
+        @Override
+        public boolean onComplex(FieldValue fv) {
+            return false;
+        }
+    }
+
+    class MathAssignIteratorHandler extends FieldPathIteratorHandler {
+        DocumentCalculator calc;
+        Document doc;
+        boolean removeIfZero;
+        boolean createMissingPath;
+
+        MathAssignIteratorHandler(String expression, Document doc, boolean removeIfZero, boolean createMissingPath) throws ParseException {
+            this.calc = new DocumentCalculator(expression);
+            this.doc = doc;
+            this.removeIfZero = removeIfZero;
+            this.createMissingPath = createMissingPath;
+        }
+
+        @Override
+        public ModificationStatus doModify(FieldValue fv) {
+            if (fv instanceof NumericFieldValue) {
+                Map<String, Object> vars = new HashMap<String, Object>();
+                for (Map.Entry<String, IndexValue> entry : getVariables().entrySet()) {
+                    if (entry.getValue().getKey() != null && entry.getValue().getKey() instanceof NumericFieldValue) {
+                        vars.put(entry.getKey(), ((NumericFieldValue)entry.getValue().getKey()).getNumber());
+                    } else {
+                        vars.put(entry.getKey(), entry.getValue().getIndex());
+                    }
+                }
+                vars.put("value", ((NumericFieldValue)fv).getNumber());
+
+                try {
+                    Number d = calc.evaluate(doc, vars);
+                    if (removeIfZero && d.longValue() == 0) {
+                        return ModificationStatus.REMOVED;
+                    } else {
+                        fv.assign(calc.evaluate(doc, vars));
+                    }
+                } catch (IllegalArgumentException e) {
+                    // Ignore divide by zero
+                    return ModificationStatus.NOT_MODIFIED;
+                }
+            } else {
+                throw new IllegalArgumentException("Trying to perform arithmetic on " + fv + " of type " + fv.getDataType());
+            }
+            return ModificationStatus.MODIFIED;
+        }
+
+        @Override
+        public boolean createMissingPath() {
+            return createMissingPath;
+        }
+
+        @Override
+        public boolean onComplex(FieldValue fv) {
+            return false;
+        }
+    }
+
+    FieldValue fieldValue = null;
+    String expression = null;
+    boolean createMissingPath = true;
+    boolean removeIfZero = false;
+
+    // Flag bits
+    public static final int ARITHMETIC_EXPRESSION = 1;
+    public static final int REMOVE_IF_ZERO = 2;
+    public static final int CREATE_MISSING_PATH = 4;
+
+    /**
+     * Creates an assignment update that overwrites the old value with the given new value.
+     *
+     * @param type The document type the assignment works on.
+     * @param fieldPath The field path of the field to be overwritten.
+     * @param whereClause A document selection string that selects documents and variables to be updated.
+     * @param newValue The new value of the assignment.
+     */
+    public AssignFieldPathUpdate(DocumentType type, String fieldPath, String whereClause, FieldValue newValue) {
+        super(FieldPathUpdate.Type.ASSIGN, type, fieldPath, whereClause);
+        setNewValue(newValue);
+    }
+
+    public AssignFieldPathUpdate(DocumentType type, String fieldPath, FieldValue newValue) {
+        super(FieldPathUpdate.Type.ASSIGN, type, fieldPath, null);
+        setNewValue(newValue);
+    }
+
+    /**
+     * Creates an assign statement based on a mathematical expression.
+     *
+     * @param type The document type the assignment works on.
+     * @param fieldPath The field path of the field to be overwritten.
+     * @param whereClause A document selection string that selects documents and variables to be updated.
+     * @param expression The mathematical expression to apply. Use $value to signify the previous value of the field.
+     */
+    public AssignFieldPathUpdate(DocumentType type, String fieldPath, String whereClause, String expression) {
+        super(FieldPathUpdate.Type.ASSIGN, type, fieldPath, whereClause);
+        setExpression(expression);
+    }
+
+    /**
+     * Creates an assign update from a serialized object.
+     *
+     * @param type The document type the assignment will work on.
+     * @param reader A reader that can deserialize something into this object.
+     */
+    public AssignFieldPathUpdate(DocumentType type, DocumentUpdateReader reader) {
+        super(FieldPathUpdate.Type.ASSIGN, type, reader);
+        reader.read(this);
+    }
+
+    /**
+     * Turns this assignment into a literal one.
+     *
+     * @param value The new value to assign to the document.
+     */
+    public void setNewValue(FieldValue value) {
+        fieldValue = value;
+        expression = null;
+    }
+
+    /**
+     *
+     * @return Returns the value to assign, or null if this is a mathematical expression.
+     */
+    public FieldValue getNewValue() {
+        return fieldValue;
+    }
+
+    /**
+     * Turns this assignment into a mathematical expression assignment.
+     *
+     * @param value The expression to use for assignment.
+     */
+    public void setExpression(String value) {
+        expression = value;
+        fieldValue = null;
+    }
+
+    /**
+     *
+     * @return Returns the arithmetic expression to assign, or null if this is not a mathematical expression.
+     */
+    public String getExpression() {
+        return expression;
+    }
+
+    /**
+     * If set to true, and the new value assigned evaluates to a numeric value of 0, removes the value instead of setting it.
+     * Default is false.
+     */
+    public void setRemoveIfZero(boolean removeIfZero) {
+        this.removeIfZero = removeIfZero;
+    }
+
+    /**
+     * If set to true, and any part of the field path specified does not exist (except for array indexes), we create the path as necessary.
+     * Default is true.
+     */
+    public void setCreateMissingPath(boolean createMissingPath) {
+        this.createMissingPath = createMissingPath;
+    }
+
+    /**
+     *
+     * @return Returns true if this assignment is an arithmetic operation.
+     */
+    public boolean isArithmetic() {
+        return expression != null;
+    }
+
+    FieldPathIteratorHandler getIteratorHandler(Document doc) {
+        if (expression != null) {
+            try {
+                return new MathAssignIteratorHandler(expression, doc, removeIfZero, createMissingPath);
+            } catch (ParseException e) {
+                return null;
+            }
+        } else {
+            return new SimpleAssignIteratorHandler(fieldValue, removeIfZero, createMissingPath);
+        }
+    }
+
+    @Override
+    public void serialize(VespaDocumentSerializerHead data) {
+        data.write(this);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        if (!super.equals(o)) return false;
+
+        AssignFieldPathUpdate that = (AssignFieldPathUpdate) o;
+
+        if (createMissingPath != that.createMissingPath) return false;
+        if (removeIfZero != that.removeIfZero) return false;
+        if (expression != null ? !expression.equals(that.expression) : that.expression != null) return false;
+        if (fieldValue != null ? !fieldValue.equals(that.fieldValue) : that.fieldValue != null) return false;
+
+        return true;
+    }
+
+    @Override
+    public int hashCode() {
+        int result = super.hashCode();
+        result = 31 * result + (fieldValue != null ? fieldValue.hashCode() : 0);
+        result = 31 * result + (expression != null ? expression.hashCode() : 0);
+        result = 31 * result + (createMissingPath ? 1 : 0);
+        result = 31 * result + (removeIfZero ? 1 : 0);
+        return result;
+    }
+
+    @Override
+    public String toString() {
+        return "Assign: " + super.toString() + " : " + (isArithmetic() ? getExpression() : getNewValue().toString());
+    }
+
+    public boolean getCreateMissingPath() {
+        return createMissingPath;
+    }
+
+    public boolean getRemoveIfZero() {
+        return removeIfZero;
+    }
+
+    public FieldValue getFieldValue() {
+        return fieldValue;
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/fieldpathupdate/FieldPathUpdate.java b/document/src/main/java/com/yahoo/document/fieldpathupdate/FieldPathUpdate.java
new file mode 100644
index 00000000000..318b696ce7a
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/fieldpathupdate/FieldPathUpdate.java
@@ -0,0 +1,172 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.fieldpathupdate;
+
+import com.yahoo.document.Document;
+import com.yahoo.document.DocumentPut;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.FieldPath;
+import com.yahoo.document.datatypes.FieldPathIteratorHandler;
+import com.yahoo.document.select.DocumentSelector;
+import com.yahoo.document.select.Result;
+import com.yahoo.document.select.ResultList;
+import com.yahoo.document.select.parser.ParseException;
+import com.yahoo.document.serialization.DocumentUpdateReader;
+import com.yahoo.document.serialization.VespaDocumentSerializerHead;
+
+/**
+ * @author <a href="mailto:thomasg@yahoo-inc.com">Thomas Gundersen</a>
+ */
+public abstract class FieldPathUpdate {
+
+    public static enum Type {
+        ASSIGN(0),
+        REMOVE(1),
+        ADD(2);
+
+        private final int code;
+
+        private Type(int code) {
+            this.code = code;
+        }
+
+        public static Type valueOf(int code) {
+            for (Type type : values()) {
+                if (type.code == code) {
+                    return type;
+                }
+            }
+            throw new IllegalArgumentException("Field path update type " + code + " not supported.");
+        }
+
+        public int getCode() {
+            return code;
+        }
+    }
+
+    private FieldPath fieldPath;
+    private DocumentSelector selector;
+    private String originalFieldPath;
+    private String whereClause;
+    private Type updType;
+    private DocumentType docType;
+
+    public FieldPathUpdate(Type updType, DocumentType docType, String fieldPath, String whereClause) {
+        this.updType = updType;
+        this.docType = docType;
+
+        try {
+            setWhereClause(whereClause);
+        } catch (ParseException e) {
+            throw new IllegalArgumentException(e);
+        }
+        setFieldPath(fieldPath);
+    }
+
+    public FieldPathUpdate(Type updType, DocumentType docType, DocumentUpdateReader reader) {
+        this.updType = updType;
+        this.docType = docType;
+        reader.read(this);
+    }
+
+    public Type getUpdateType() {
+        return updType;
+    }
+
+    public DocumentType getDocumentType() {
+        return docType;
+    }
+
+    public void setFieldPath(String fieldPath) {
+        originalFieldPath = fieldPath;
+        this.fieldPath = docType.buildFieldPath(fieldPath);
+    }
+
+    public FieldPath getFieldPath() {
+        return fieldPath;
+    }
+
+    public String getOriginalFieldPath() {
+        return originalFieldPath;
+    }
+
+    public void setWhereClause(String whereClause) throws ParseException {
+        this.whereClause = whereClause;
+        selector = null;
+        if (whereClause != null && !whereClause.equals("")) {
+            selector = new DocumentSelector(whereClause);
+        }
+    }
+
+    public DocumentSelector getWhereClause() {
+        return selector;
+    }
+
+    public String getOriginalWhereClause() {
+        return whereClause;
+    }
+
+    public void applyTo(Document doc) {
+        if (selector == null) {
+            FieldPathIteratorHandler handler = getIteratorHandler(doc);
+            doc.iterateNested(fieldPath, 0, handler);
+        } else {
+            ResultList results = selector.getMatchingResultList(new DocumentPut(doc));
+
+            for (ResultList.ResultPair rp : results.getResults()) {
+                if (rp.getResult() == Result.TRUE) {
+                    FieldPathIteratorHandler handler = getIteratorHandler(doc);
+                    handler.getVariables().clear();
+                    handler.getVariables().putAll(rp.getVariables());
+
+                    doc.iterateNested(fieldPath, 0, handler);
+                }
+            }
+        }
+    }
+
+    public void serialize(VespaDocumentSerializerHead data) {
+        data.write(this);
+    }
+
+    public static FieldPathUpdate create(Type type, DocumentType docType, DocumentUpdateReader reader) throws ParseException {
+        switch (type) {
+        case ASSIGN:
+            return new AssignFieldPathUpdate(docType, reader);
+        case ADD:
+            return new AddFieldPathUpdate(docType, reader);
+        case REMOVE:
+            return new RemoveFieldPathUpdate(docType, reader);
+        }
+        throw new IllegalArgumentException("Field path update type '" + type + "' not supported.");
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        FieldPathUpdate that = (FieldPathUpdate) o;
+
+        if (docType != null ? !docType.equals(that.docType) : that.docType != null) return false;
+        if (originalFieldPath != null ? !originalFieldPath.equals(that.originalFieldPath) : that.originalFieldPath != null)
+            return false;
+        if (whereClause != null ? !whereClause.equals(that.whereClause) : that.whereClause != null) return false;
+
+        return true;
+    }
+
+    @Override
+    public int hashCode() {
+        int result = originalFieldPath != null ? originalFieldPath.hashCode() : 0;
+        result = 31 * result + (whereClause != null ? whereClause.hashCode() : 0);
+        result = 31 * result + (docType != null ? docType.hashCode() : 0);
+        return result;
+    }
+
+    @Override
+    public String toString() {
+        return "fieldpath=\"" + originalFieldPath + "\"" + (whereClause != null ? " where=\"" + whereClause + "\"" : "");
+    }
+
+    abstract FieldPathIteratorHandler getIteratorHandler(Document doc);
+}
diff --git a/document/src/main/java/com/yahoo/document/fieldpathupdate/RemoveFieldPathUpdate.java b/document/src/main/java/com/yahoo/document/fieldpathupdate/RemoveFieldPathUpdate.java
new file mode 100644
index 00000000000..fc4a5b39f92
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/fieldpathupdate/RemoveFieldPathUpdate.java
@@ -0,0 +1,56 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.fieldpathupdate;
+
+import com.yahoo.document.Document;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.datatypes.FieldPathIteratorHandler;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.select.parser.ParseException;
+import com.yahoo.document.serialization.DocumentUpdateReader;
+
+/**
+ * @author <a href="mailto:thomasg@yahoo-inc.com">Thomas Gundersen</a>
+ */
+public class RemoveFieldPathUpdate extends FieldPathUpdate {
+    class IteratorHandler extends FieldPathIteratorHandler {
+        IteratorHandler() {
+        }
+
+        @Override
+        public ModificationStatus doModify(FieldValue fv) {
+            return ModificationStatus.REMOVED;
+        }
+
+        @Override
+        public boolean onComplex(FieldValue fv) {
+            return false;
+        }
+    }
+
+    IteratorHandler handler;
+
+    public RemoveFieldPathUpdate(DocumentType type, String fieldPath, String whereClause) {
+        super(FieldPathUpdate.Type.REMOVE, type, fieldPath, whereClause);
+        handler = new IteratorHandler();
+    }
+
+    public RemoveFieldPathUpdate(DocumentType type, String fieldPath) {
+        super(FieldPathUpdate.Type.REMOVE, type, fieldPath, null);
+        handler = new IteratorHandler();
+    }
+
+    public RemoveFieldPathUpdate(DocumentType type, DocumentUpdateReader reader) {
+        super(FieldPathUpdate.Type.REMOVE, type, reader);
+        reader.read(this);
+        handler = new IteratorHandler();
+    }
+
+    FieldPathIteratorHandler getIteratorHandler(Document doc) {
+        return handler;
+    }
+
+    @Override
+    public String toString() {
+        return "Remove: " + super.toString();
+    }
+}
+\ No newline at end of file
diff --git a/document/src/main/java/com/yahoo/document/fieldpathupdate/package-info.java b/document/src/main/java/com/yahoo/document/fieldpathupdate/package-info.java
new file mode 100644
index 00000000000..8e9f4069386
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/fieldpathupdate/package-info.java
@@ -0,0 +1,7 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+@PublicApi
+package com.yahoo.document.fieldpathupdate;
+
+import com.yahoo.api.annotations.PublicApi;
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/document/src/main/java/com/yahoo/document/fieldset/AllFields.java b/document/src/main/java/com/yahoo/document/fieldset/AllFields.java
new file mode 100644
index 00000000000..3dff7c1e4e6
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/fieldset/AllFields.java
@@ -0,0 +1,21 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.fieldset;
+
+/**
+ * Created with IntelliJ IDEA.
+ * User: thomasg
+ * Date: 4/25/12
+ * Time: 3:18 PM
+ * To change this template use File | Settings | File Templates.
+ */
+public class AllFields implements FieldSet {
+    @Override
+    public boolean contains(FieldSet o) {
+        return true;
+    }
+
+    @Override
+    public FieldSet clone() throws CloneNotSupportedException {
+        return new AllFields();
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/fieldset/BodyFields.java b/document/src/main/java/com/yahoo/document/fieldset/BodyFields.java
new file mode 100644
index 00000000000..50a72fa2fde
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/fieldset/BodyFields.java
@@ -0,0 +1,42 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.fieldset;
+
+import com.yahoo.document.Field;
+
+/**
+ * Created with IntelliJ IDEA.
+ * User: thomasg
+ * Date: 4/25/12
+ * Time: 3:18 PM
+ * To change this template use File | Settings | File Templates.
+ */
+public class BodyFields implements FieldSet {
+    @Override
+    public boolean contains(FieldSet o) {
+        if (o instanceof BodyFields || o instanceof DocIdOnly || o instanceof NoFields) {
+            return true;
+        }
+
+        if (o instanceof Field) {
+            return !((Field) o).isHeader();
+        }
+
+        if (o instanceof FieldCollection) {
+            FieldCollection c = (FieldCollection)o;
+            for (Field f : c) {
+                if (f.isHeader()) {
+                    return false;
+                }
+            }
+
+            return true;
+        } else {
+            return false;
+        }
+    }
+
+    @Override
+    public FieldSet clone() throws CloneNotSupportedException {
+        return new BodyFields();
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/fieldset/DocIdOnly.java b/document/src/main/java/com/yahoo/document/fieldset/DocIdOnly.java
new file mode 100644
index 00000000000..96deedf34f0
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/fieldset/DocIdOnly.java
@@ -0,0 +1,21 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.fieldset;
+
+/**
+ * Created with IntelliJ IDEA.
+ * User: thomasg
+ * Date: 4/25/12
+ * Time: 3:21 PM
+ * To change this template use File | Settings | File Templates.
+ */
+public class DocIdOnly implements FieldSet {
+    @Override
+    public boolean contains(FieldSet o) {
+        return (o instanceof DocIdOnly || o instanceof NoFields);
+    }
+
+    @Override
+    public FieldSet clone() throws CloneNotSupportedException {
+        return new DocIdOnly();
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/fieldset/FieldCollection.java b/document/src/main/java/com/yahoo/document/fieldset/FieldCollection.java
new file mode 100644
index 00000000000..bec47bb9c2e
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/fieldset/FieldCollection.java
@@ -0,0 +1,49 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.fieldset;
+
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.Field;
+
+import java.util.ArrayList;
+
+public class FieldCollection extends ArrayList<Field> implements FieldSet {
+    DocumentType docType;
+
+    public FieldCollection(DocumentType type) {
+        docType = type;
+    }
+
+    public DocumentType getDocumentType() {
+        return docType;
+    }
+
+    @Override
+    public boolean contains(FieldSet o) {
+        if (o instanceof DocIdOnly || o instanceof NoFields) {
+            return true;
+        }
+
+        if (o instanceof Field) {
+            return super.contains(o);
+        } else if (o instanceof FieldCollection) {
+            FieldCollection c = (FieldCollection)o;
+
+            for (Field f : c) {
+                if (!super.contains(f)) {
+                    return false;
+                }
+            }
+
+            return true;
+        } else {
+            return false;
+        }
+    }
+
+    @Override
+    public FieldSet clone() {
+        FieldCollection c = new FieldCollection(docType);
+        c.addAll(this);
+        return c;
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/fieldset/FieldSet.java b/document/src/main/java/com/yahoo/document/fieldset/FieldSet.java
new file mode 100644
index 00000000000..0d0b9244e35
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/fieldset/FieldSet.java
@@ -0,0 +1,19 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.fieldset;
+
+import com.yahoo.document.Document;
+import com.yahoo.document.Field;
+import com.yahoo.document.datatypes.FieldValue;
+
+import java.lang.Object;
+import java.util.Iterator;
+import java.util.Map;
+
+/**
+ * TODO: Move to Java and implement.
+ */
+public interface FieldSet {
+    public boolean contains(FieldSet o);
+
+    public FieldSet clone() throws CloneNotSupportedException;
+}
diff --git a/document/src/main/java/com/yahoo/document/fieldset/FieldSetRepo.java b/document/src/main/java/com/yahoo/document/fieldset/FieldSetRepo.java
new file mode 100644
index 00000000000..c0da7fd9bc7
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/fieldset/FieldSetRepo.java
@@ -0,0 +1,141 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.fieldset;
+
+import com.yahoo.document.Document;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.DocumentTypeManager;
+import com.yahoo.document.Field;
+import com.yahoo.document.datatypes.FieldValue;
+
+import java.lang.String;
+import java.util.*;
+
+/**
+ * TODO: Move to document and implement
+ */
+public class FieldSetRepo {
+
+    FieldSet parseSpecialValues(String name)
+    {
+        if (name.equals("[id]")) { return new DocIdOnly(); }
+        else if (name.equals("[all]")) { return (new AllFields()); }
+        else if (name.equals("[none]")) { return (new NoFields()); }
+        else if (name.equals("[header]")) { return (new HeaderFields()); }
+        else if (name.equals("[docid]")) { return (new DocIdOnly()); }
+        else if (name.equals("[body]")) { return (new BodyFields()); }
+        else {
+            throw new IllegalArgumentException(
+                    "The only special names (enclosed in '[]') allowed are " +
+                    "id, all, none, header, body");
+        }
+    }
+
+    FieldSet parseFieldCollection(DocumentTypeManager docMan, String docType, String fieldNames) {
+        DocumentType type = docMan.getDocumentType(docType);
+        if (type == null) {
+         throw new IllegalArgumentException("Unknown document type " + docType);
+        }
+
+        StringTokenizer tokenizer = new StringTokenizer(fieldNames, ",");
+        FieldCollection collection = new FieldCollection(type);
+
+        for (; tokenizer.hasMoreTokens(); ) {
+            String token = tokenizer.nextToken();
+            Field f = type.getField(token);
+            if (f == null) {
+                throw new IllegalArgumentException("No such field " + token);
+            }
+            collection.add(f);
+        }
+
+        return collection;
+    }
+
+    public FieldSet parse(DocumentTypeManager docMan, String fieldSet) {
+        if (fieldSet.length() == 0) {
+            throw new IllegalArgumentException("Illegal field set value \"\"");
+        }
+
+        if (fieldSet.startsWith("[")) {
+            return parseSpecialValues(fieldSet);
+        }
+
+        StringTokenizer tokenizer = new StringTokenizer(fieldSet, ":");
+        if (tokenizer.countTokens() != 2) {
+            throw new IllegalArgumentException(
+                    "The field set list must consist of a document type, " +
+                    "then a colon (:), then a comma-separated list of field names");
+        }
+
+        String type = tokenizer.nextToken();
+        String fields = tokenizer.nextToken();
+
+        return parseFieldCollection(docMan, type, fields);
+    }
+
+    public String serialize(FieldSet fieldSet) {
+        if (fieldSet instanceof Field) {
+            return ((Field)fieldSet).getName();
+        } else if (fieldSet instanceof FieldCollection) {
+            FieldCollection c = ((FieldCollection)fieldSet);
+
+            StringBuffer buffer = new StringBuffer();
+            for (Field f : c) {
+                if (buffer.length() == 0) {
+                    buffer.append(c.getDocumentType().getName());
+                    buffer.append(":");
+                } else {
+                    buffer.append(",");
+                }
+                buffer.append(f.getName());
+            }
+
+            return buffer.toString();
+        } else if (fieldSet instanceof AllFields) {
+            return "[all]";
+        } else if (fieldSet instanceof NoFields) {
+            return "[none]";
+        } else if (fieldSet instanceof BodyFields) {
+            return "[body]";
+        } else if (fieldSet instanceof HeaderFields) {
+            return "[header]";
+        } else if (fieldSet instanceof DocIdOnly) {
+            return "[docid]";
+        } else {
+            throw new IllegalArgumentException("Unknown field set type " + fieldSet);
+        }
+    }
+
+
+    /**
+     * Copies fields from one document to another based on whether the fields match the given
+     * fieldset.
+     */
+    public void copyFields(Document source, Document target, FieldSet fieldSet) {
+        for (Iterator<Map.Entry<Field, FieldValue>> i = source.iterator(); i.hasNext();) {
+            Map.Entry<Field, FieldValue> v = i.next();
+
+            if (fieldSet.contains(v.getKey())) {
+                target.setFieldValue(v.getKey(), v.getValue());
+            }
+        }
+    }
+
+    /**
+     * Strips all fields not wanted by the given field set from the document.
+     */
+    public void stripFields(Document target, FieldSet fieldSet) {
+        List<Field> toStrip = new ArrayList<Field>();
+        for (Iterator<Map.Entry<Field, FieldValue>> i = target.iterator(); i.hasNext();) {
+            Map.Entry<Field, FieldValue> v = i.next();
+
+            if (!fieldSet.contains(v.getKey())) {
+                toStrip.add(v.getKey());
+            }
+        }
+
+        for (Field f : toStrip) {
+            target.removeFieldValue(f);
+        }
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/fieldset/HeaderFields.java b/document/src/main/java/com/yahoo/document/fieldset/HeaderFields.java
new file mode 100644
index 00000000000..a9e9375f9ac
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/fieldset/HeaderFields.java
@@ -0,0 +1,42 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.fieldset;
+
+import com.yahoo.document.Field;
+
+/**
+ * Created with IntelliJ IDEA.
+ * User: thomasg
+ * Date: 4/25/12
+ * Time: 3:18 PM
+ * To change this template use File | Settings | File Templates.
+ */
+public class HeaderFields implements FieldSet {
+    @Override
+    public boolean contains(FieldSet o) {
+        if (o instanceof HeaderFields || o instanceof DocIdOnly || o instanceof NoFields) {
+            return true;
+        }
+
+        if (o instanceof Field) {
+            return ((Field)o).isHeader();
+        }
+
+        if (o instanceof FieldCollection) {
+            FieldCollection c = (FieldCollection)o;
+            for (Field f : c) {
+                if (!f.isHeader()) {
+                    return false;
+                }
+            }
+
+            return true;
+        } else {
+            return false;
+        }
+    }
+
+    @Override
+    public FieldSet clone() throws CloneNotSupportedException {
+        return new HeaderFields();
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/fieldset/NoFields.java b/document/src/main/java/com/yahoo/document/fieldset/NoFields.java
new file mode 100644
index 00000000000..43d9412f94d
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/fieldset/NoFields.java
@@ -0,0 +1,21 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.fieldset;
+
+/**
+ * Created with IntelliJ IDEA.
+ * User: thomasg
+ * Date: 4/25/12
+ * Time: 3:18 PM
+ * To change this template use File | Settings | File Templates.
+ */
+public class NoFields implements FieldSet {
+    @Override
+    public boolean contains(FieldSet o) {
+        return (o instanceof NoFields);
+    }
+
+    @Override
+    public FieldSet clone() throws CloneNotSupportedException {
+        return new NoFields();
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/fieldset/package-info.java b/document/src/main/java/com/yahoo/document/fieldset/package-info.java
new file mode 100644
index 00000000000..33534f3396c
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/fieldset/package-info.java
@@ -0,0 +1,7 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+@PublicApi
+package com.yahoo.document.fieldset;
+
+import com.yahoo.api.annotations.PublicApi;
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/document/src/main/java/com/yahoo/document/idstring/DocIdString.java b/document/src/main/java/com/yahoo/document/idstring/DocIdString.java
new file mode 100644
index 00000000000..85ed7451fbe
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/idstring/DocIdString.java
@@ -0,0 +1,47 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.idstring;
+
+import com.yahoo.collections.MD5;
+import com.yahoo.text.Utf8;
+
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+
+/**
+ * Representation of doc scheme in document IDs.
+ *
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class DocIdString extends IdString {
+    /**
+     * Create a doc scheme object.
+     * <code>doc:&lt;namespace&gt;:&lt;namespaceSpecific&gt;</code>
+     *
+     * @param namespace The namespace of this document id.
+     * @param namespaceSpecific The namespace specific part.
+     */
+    public DocIdString(String namespace, String namespaceSpecific) {
+        super(Scheme.doc, namespace, namespaceSpecific);
+    }
+
+    /**
+     * Get the location of this document id. The location is used for distribution
+     * in clusters. For the doc scheme, the location is a hash of the whole id.
+     *
+     * @return The 64 bit location.
+     */
+    public long getLocation() {
+        long result = 0;
+        byte[] md5sum = MD5.md5.get().digest(Utf8.toBytes(toString()));
+        for (int i=0; i<8; ++i) {
+            result |= (md5sum[i] & 0xFFl) << (8*i);
+        }
+
+        return result;
+    }
+
+    /** Get the scheme specific part. Which is non-existing for doc scheme. */
+    public String getSchemeSpecific() {
+        return "";
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/idstring/GroupDocIdString.java b/document/src/main/java/com/yahoo/document/idstring/GroupDocIdString.java
new file mode 100644
index 00000000000..aed7c78d0ef
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/idstring/GroupDocIdString.java
@@ -0,0 +1,64 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.idstring;
+
+import com.yahoo.collections.MD5;
+import com.yahoo.text.Utf8;
+
+import java.security.MessageDigest;
+
+/**
+ * Representation of groupdoc scheme in document IDs.
+ *
+ * @author <a href="mailto:humbe@yahoo-inc.com">H&aring;kon Humberset</a>
+ */
+public class GroupDocIdString extends IdString {
+    String group;
+
+    /**
+     * Create a groupdoc scheme object.
+     * <code>groupdoc:&lt;namespace&gt;:&lt;group&gt;:&lt;namespaceSpecific&gt;</code>
+     *
+     * @param namespace The namespace of this document id.
+     * @param group The groupname of this groupdoc id.
+     * @param namespaceSpecific The namespace specific part.
+     */
+    public GroupDocIdString(String namespace, String group, String namespaceSpecific) {
+        super(Scheme.groupdoc, namespace, namespaceSpecific);
+        this.group = group;
+    }
+
+    /**
+     * Get the location of this document id. The location is used for distribution
+     * in clusters. For the groupdoc scheme, the location is a hash of the groupname.
+     *
+     * @return The 64 bit location.
+     */
+    public long getLocation() {
+        long result = 0;
+        try{
+            byte[] md5sum = MD5.md5.get().digest(Utf8.toBytes(group));
+            for (int i=0; i<8; ++i) {
+                result |= (md5sum[i] & 0xFFl) << (8*i);
+            }
+        } catch (Exception e) {
+            e.printStackTrace(); // TODO: FIXME!
+        }
+        return result;
+    }
+
+    /** Get the scheme specific part. Which is for a groupdoc, is the groupdoc and a colon. */
+    public String getSchemeSpecific() {
+        return group + ":";
+    }
+
+    @Override
+    public boolean hasGroup() {
+        return true;
+    }
+
+    /** @return Get the groupname of this id. */
+    @Override
+    public String getGroup() {
+        return group;
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/idstring/IdIdString.java b/document/src/main/java/com/yahoo/document/idstring/IdIdString.java
new file mode 100644
index 00000000000..6fd5c578ee8
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/idstring/IdIdString.java
@@ -0,0 +1,132 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.idstring;
+
+import com.yahoo.collections.MD5;
+import com.yahoo.text.Utf8;
+
+/**
+ * Created with IntelliJ IDEA.
+ * User: magnarn
+ * Date: 10/15/12
+ * Time: 11:02 AM
+ */
+public class IdIdString extends IdString {
+    private String type;
+    private String group;
+    private long location;
+    private boolean hasGroup;
+    private boolean hasNumber;
+
+    public static String replaceType(String id, String typeName) {
+        int typeStartPos = id.indexOf(":", 3) + 1;
+        int typeEndPos = id.indexOf(":", typeStartPos);
+        return id.substring(0, typeStartPos) + typeName + id.substring(typeEndPos);
+    }
+
+
+    private static long makeLocation(String s) {
+        long result = 0;
+        byte[] md5sum = MD5.md5.get().digest(Utf8.toBytes(s));
+        for (int i=0; i<8; ++i) {
+            result |= (md5sum[i] & 0xFFl) << (8*i);
+        }
+
+        return result;
+    }
+
+    /**
+     * Create an id scheme object.
+     * <code>doc:&lt;namespace&gt;:&lt;documentType&gt;:&lt;key-value-pairs&gt;:&lt;namespaceSpecific&gt;</code>
+     *
+     * @param namespace The namespace of this document id.
+     * @param type The type of this document id.
+     * @param keyValues The key/value pairs of this document id.
+     * @param localId The namespace specific part.
+     */
+    public IdIdString(String namespace, String type, String keyValues, String localId) {
+        super(Scheme.id, namespace, localId);
+        this.type = type;
+        boolean hasSetLocation = false;
+        for(String pair : keyValues.split(",")) {
+            int pos = pair.indexOf('=');
+            if (pos == -1) {
+                if (pair.equals("")) {  // empty pair is ok
+                    continue;
+                }
+                throw new IllegalArgumentException("Illegal key-value pair '" + pair + "'");
+            }
+            String key = pair.substring(0, pos);
+            String value = pair.substring(pos + 1);
+            switch(key) {
+                case "n":
+                    if (hasSetLocation) {
+                        throw new IllegalArgumentException("Illegal key combination in " + keyValues);
+                    }
+                    location = Long.parseLong(value);
+                    hasSetLocation = true;
+                    hasNumber = true;
+                    break;
+                case "g":
+                    if (hasSetLocation) {
+                        throw new IllegalArgumentException("Illegal key combination in " + keyValues);
+                    }
+                    location = makeLocation(value);
+                    hasSetLocation = true;
+                    hasGroup = true;
+                    group = value;
+                    break;
+                default:
+                    throw new IllegalArgumentException("Illegal key '" + key + "'");
+            }
+        }
+        if (!hasSetLocation) {
+            location = makeLocation(localId);
+        }
+    }
+
+    @Override
+    public long getLocation() {
+        return location;
+    }
+
+    @Override
+    public String getSchemeSpecific() {
+        if (hasGroup) {
+            return type + ":g=" + group + ":";
+        } else if (hasNumber) {
+            return type + ":n=" + location + ":";
+        } else {
+          return type + "::";
+        }
+    }
+
+    @Override
+    public boolean hasDocType() {
+        return true;
+    }
+
+    @Override
+    public String getDocType() {
+        return type;
+    }
+
+    @Override
+    public boolean hasGroup() {
+        return hasGroup;
+    }
+
+    @Override
+    public String getGroup() {
+        return group;
+    }
+
+    @Override
+    public boolean hasNumber() {
+        return hasNumber;
+    }
+
+    @Override
+    public long getNumber() {
+        return location;
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/idstring/IdString.java b/document/src/main/java/com/yahoo/document/idstring/IdString.java
new file mode 100644
index 00000000000..55fd601dd0d
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/idstring/IdString.java
@@ -0,0 +1,219 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.idstring;
+
+import com.yahoo.text.Utf8String;
+
+import java.math.BigInteger;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+
+/**
+ * To be used with DocumentId constructor.
+ *
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public abstract class IdString {
+
+    public boolean hasDocType() {
+        return false;
+    }
+
+    public String getDocType() {
+        return "";
+    }
+
+    public boolean hasGroup() {
+        return false;
+    }
+
+    public boolean hasNumber() {
+        return false;
+    }
+
+    public long getNumber() {
+        return 0;
+    }
+
+    public String getGroup() {
+        return "";
+    }
+
+    public class GidModifier {
+        public int usedBits;
+        public long value;
+    }
+
+    public enum Scheme { doc, userdoc, groupdoc, orderdoc, id }
+    final Scheme scheme;
+    final String namespace;
+    final String namespaceSpecific;
+    Utf8String cache;
+
+    public static int[] generateOrderDocParams(String scheme) {
+        int parenPos = scheme.indexOf("(");
+        int endParenPos = scheme.indexOf(")");
+
+        if (parenPos == -1 || endParenPos == -1) {
+            throw new IllegalArgumentException("Unparseable scheme " + scheme + ": Must be on the form orderdoc(width, division)");
+        }
+
+        String params = scheme.substring(parenPos + 1, endParenPos);
+        String[] vals = params.split(",");
+
+        if (vals.length != 2) {
+            throw new IllegalArgumentException("Unparseable scheme " + scheme + ": Must be on the form orderdoc(width, division)");
+        }
+
+        int[] retVal = new int[2];
+
+        try {
+            retVal[0] = Integer.parseInt(vals[0]);
+            retVal[1] = Integer.parseInt(vals[1]);
+            return retVal;
+        } catch (Exception e) {
+            throw new IllegalArgumentException("Unparseable scheme " + scheme + ": Must be on the form orderdoc(width, division)");
+        }
+    }
+
+    public static IdString createIdString(String id) {
+        String namespace;
+        long userId;
+        String group;
+        long ordering;
+
+        int schemePos = id.indexOf(":");
+        if (schemePos < 0) {
+            throw new IllegalArgumentException("Unparseable id '" + id + "': Scheme missing");
+        }
+
+        //Find scheme
+        String schemeStr = id.substring(0, schemePos);
+        int currPos = schemePos + 1;
+
+        //Find namespace
+        int colonPos = id.indexOf(":", currPos);
+        if (colonPos < 0) {
+            throw new IllegalArgumentException("Unparseable id '" + id + "': Namespace missing");
+        } else {
+            namespace = id.substring(currPos, colonPos);
+
+            if (namespace.length() == 0) {
+                throw new IllegalArgumentException("Unparseable id '" + id + "': Namespace must be non-empty");
+            }
+
+            currPos = colonPos + 1;
+        }
+
+        if (schemeStr.equals("id")) {
+            colonPos = id.indexOf(":", currPos);
+            if (colonPos < 0) {
+                throw new IllegalArgumentException("Unparseable id '" + id + "': Document type missing");
+            }
+            String type = id.substring(currPos, colonPos);
+            currPos = colonPos + 1;
+            colonPos = id.indexOf(":", currPos);
+            if (colonPos < 0) {
+                throw new IllegalArgumentException("Unparseable id '" + id + "': Key/value section missing");
+            }
+            String keyValues = id.substring(currPos, colonPos);
+
+            currPos = colonPos + 1;
+            return new IdIdString(namespace, type, keyValues, id.substring(currPos));
+
+        } if (schemeStr.equals("doc")) {
+            return new DocIdString(namespace, id.substring(currPos));
+        } else if (schemeStr.equals("userdoc")) {
+            colonPos = id.indexOf(":", currPos);
+            if (colonPos < 0) {
+                throw new IllegalArgumentException("Unparseable id '" + id + "': User id missing");
+            }
+
+            try {
+                userId = new BigInteger(id.substring(currPos, colonPos)).longValue();
+            } catch (IllegalArgumentException iae) {
+                throw new IllegalArgumentException("Unparseable id '" + id + "': " + iae.getMessage(), iae.getCause());
+            }
+
+            currPos = colonPos + 1;
+            return new UserDocIdString(namespace, userId, id.substring(currPos));
+        } else if (schemeStr.equals("groupdoc")) {
+            colonPos = id.indexOf(":", currPos);
+
+            if (colonPos < 0) {
+                throw new IllegalArgumentException("Unparseable id '" + id + "': Group id missing");
+            }
+
+            group = id.substring(currPos, colonPos);
+            currPos = colonPos + 1;
+            return new GroupDocIdString(namespace, group, id.substring(currPos));
+        } else if (schemeStr.indexOf("orderdoc") == 0) {
+            int[] params = generateOrderDocParams(schemeStr);
+
+            colonPos = id.indexOf(":", currPos);
+
+            if (colonPos < 0) {
+                throw new IllegalArgumentException("Unparseable id '" + id + "': Group id missing");
+            }
+
+            group = id.substring(currPos, colonPos);
+
+            currPos = colonPos + 1;
+
+            colonPos = id.indexOf(":", currPos);
+            if (colonPos < 0) {
+                throw new IllegalArgumentException("Unparseable id '" + id + "': Ordering missing");
+            }
+
+            try {
+                ordering = Long.parseLong(id.substring(currPos, colonPos));
+            } catch (IllegalArgumentException e) {
+                throw new IllegalArgumentException("Unparseable id '" + id + "': " + e.getMessage(), e.getCause());
+            }
+
+            currPos = colonPos + 1;
+            return new OrderDocIdString(namespace, group, params[0], params[1], ordering, id.substring(currPos));
+        } else {
+            throw new IllegalArgumentException("Unknown id scheme '" + schemeStr + "'");
+        }
+    }
+
+    protected IdString(Scheme scheme, String namespace, String namespaceSpecific) {
+        this.scheme = scheme;
+        this.namespace = namespace;
+        this.namespaceSpecific = namespaceSpecific;
+    }
+
+    public Scheme getType() { return scheme; }
+
+    public String getNamespace() { return namespace; }
+    public String getNamespaceSpecific() { return namespaceSpecific; }
+    public abstract long getLocation();
+    public String getSchemeParameters() { return ""; }
+    public abstract String getSchemeSpecific();
+    public GidModifier getGidModifier() { return null; }
+
+    public boolean equals(Object o) {
+        return (o instanceof IdString && o.toString().equals(toString()));
+    }
+
+    public int hashCode() {
+        return toString().hashCode();
+    }
+
+    private Utf8String createToString() {
+        return new Utf8String(scheme.toString() + getSchemeParameters() + ':' + namespace + ':' + getSchemeSpecific() + namespaceSpecific);
+    }
+    public String toString() {
+        if (cache == null) {
+            cache = createToString();
+        }
+        return cache.toString();
+    }
+    public Utf8String toUtf8() {
+        if (cache == null) {
+            cache = createToString();
+        }
+        return cache;
+    }
+
+}
diff --git a/document/src/main/java/com/yahoo/document/idstring/OrderDocIdString.java b/document/src/main/java/com/yahoo/document/idstring/OrderDocIdString.java
new file mode 100644
index 00000000000..111be0110b5
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/idstring/OrderDocIdString.java
@@ -0,0 +1,116 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.idstring;
+
+import com.yahoo.collections.MD5;
+import com.yahoo.text.Utf8;
+
+import java.security.MessageDigest;
+
+/**
+ * Representation of groupdoc scheme in document IDs.
+ *
+ * @author <a href="mailto:humbe@yahoo-inc.com">H&aring;kon Humberset</a>
+ */
+public class OrderDocIdString extends IdString {
+    String group;
+    int widthBits;
+    int divisionBits;
+    long ordering;
+    long location;
+
+    /**
+     * Create a groupdoc scheme object.
+     * <code>groupdoc:&lt;namespace&gt;:&lt;group&gt;:&lt;namespaceSpecific&gt;</code>
+     *
+     * @param namespace The namespace of this document id.
+     * @param group The groupname of this groupdoc id.
+     * @param widthBits The number of bits used for the width of the data set
+     * @param divisionBits The number of bits used for the smalles partitioning of the data set
+     * @param ordering A value used to order documents of this type.
+     * @param namespaceSpecific The namespace specific part.
+     */
+    public OrderDocIdString(String namespace, String group, int widthBits, int divisionBits, long ordering, String namespaceSpecific) {
+        super(Scheme.orderdoc, namespace, namespaceSpecific);
+        this.group = group;
+        this.widthBits = widthBits;
+        this.divisionBits = divisionBits;
+        this.ordering = ordering;
+
+        try {
+            this.location = Long.parseLong(group);
+        } catch (Exception foo) {
+            location = 0;
+            byte[] md5sum = MD5.md5.get().digest(Utf8.toBytes(group));
+            for (int i=0; i<8; ++i) {
+                location |= (md5sum[i] & 0xFFl) << (8*i);
+            }
+        }
+    }
+
+    /**
+     * Get the location of this document id. The location is used for distribution
+     * in clusters. For the orderdoc scheme, the location is a hash of the groupname or just the number specified.
+     *
+     * @return The 64 bit location.
+     */
+    public long getLocation() {
+        return location;
+    }
+
+    public String getSchemeParameters() {
+        return "(" + widthBits + "," + divisionBits + ")";
+    }
+
+    /** Get the scheme specific part. */
+    public String getSchemeSpecific() {
+        return group + ":" + ordering + ":";
+    }
+
+    public GidModifier getGidModifier() {
+        GidModifier gm = new GidModifier();
+        gm.usedBits = widthBits - divisionBits;
+        long gidBits = (ordering << (64 - widthBits));
+        gidBits = Long.reverse(gidBits);
+        long gidMask = (0xFFFFFFFFFFFFFFFFl >>> (64 - gm.usedBits));
+        gidBits &= gidMask;
+        gm.value = gidBits;
+        return gm;
+    }
+
+    @Override
+    public boolean hasGroup() {
+        return true;
+    }
+
+    /** @return Get the groupname of this id. */
+    @Override
+    public String getGroup() {
+        return group;
+    }
+
+    @Override
+    public boolean hasNumber() {
+        return true;
+    }
+
+    @Override
+    public long getNumber() {
+        return location;
+    }
+
+    public long getUserId() {
+        return location;
+    }
+
+    public int getWidthBits() {
+        return widthBits;
+    }
+
+    public int getDivisionBits() {
+        return divisionBits;
+    }
+
+    public long getOrdering() {
+        return ordering;
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/idstring/UserDocIdString.java b/document/src/main/java/com/yahoo/document/idstring/UserDocIdString.java
new file mode 100644
index 00000000000..631c199e6bf
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/idstring/UserDocIdString.java
@@ -0,0 +1,59 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.idstring;
+
+import java.math.BigInteger;
+
+/**
+ * Representation of userdoc scheme in document IDs. A user id is any 64 bit
+ * number. Note that internally, these are handled as unsigned values.
+ *
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class UserDocIdString extends IdString {
+    long userId;
+
+    /**
+     * Create a userdoc scheme object.
+     * <code>userdoc:&lt;namespace&gt;:&lt;userid&gt;:&lt;namespaceSpecific&gt;</code>
+     *
+     * @param namespace The namespace of this document id.
+     * @param userId 64 bit user id of this userdoc id.
+     * @param namespaceSpecific The namespace specific part.
+     */
+    public UserDocIdString(String namespace, long userId, String namespaceSpecific) {
+        super(Scheme.userdoc, namespace, namespaceSpecific);
+        this.userId = userId;
+    }
+
+    @Override
+    public boolean hasNumber() {
+        return true;
+    }
+
+    @Override
+    public long getNumber() {
+        return userId;
+    }
+
+    /**
+     * Get the location of this document id. The location is used for distribution
+     * in clusters. For the userdoc scheme, the location equals the user id.
+     *
+     * @return The 64 bit location.
+     */
+    public long getLocation() { return userId; }
+
+    /** Get the scheme specific part. Which for a userdoc, is the userid and a colon. */
+    public String getSchemeSpecific() {
+        BigInteger uid = BigInteger.ZERO;
+        for (int i=0; i<64; i++) {
+            if ((userId >>> i & 0x1) == 1) {
+                uid = uid.setBit(i);
+            }
+        }
+        return uid.toString() + ":";
+    }
+
+    /** @return Get the user id of this id. */
+    public long getUserId() { return userId; }
+}
diff --git a/document/src/main/java/com/yahoo/document/idstring/package-info.java b/document/src/main/java/com/yahoo/document/idstring/package-info.java
new file mode 100644
index 00000000000..4ddd539ccd5
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/idstring/package-info.java
@@ -0,0 +1,7 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+@PublicApi
+package com.yahoo.document.idstring;
+
+import com.yahoo.api.annotations.PublicApi;
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/document/src/main/java/com/yahoo/document/json/JsonFeedReader.java b/document/src/main/java/com/yahoo/document/json/JsonFeedReader.java
new file mode 100644
index 00000000000..5d329554192
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/json/JsonFeedReader.java
@@ -0,0 +1,58 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.json;
+
+import java.io.InputStream;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.yahoo.document.DocumentOperation;
+import com.yahoo.document.DocumentPut;
+import com.yahoo.document.DocumentRemove;
+import com.yahoo.document.DocumentTypeManager;
+import com.yahoo.document.DocumentUpdate;
+import com.yahoo.vespaxmlparser.FeedReader;
+import com.yahoo.vespaxmlparser.VespaXMLFeedReader.Operation;
+
+
+/**
+ * Facade between JsonReader and the FeedReader API.
+ *
+ * <p>
+ * The feed reader will take ownership of the input stream and close it when the
+ * last parseable document has been read.
+ *
+ * @author steinar
+ */
+public class JsonFeedReader implements FeedReader {
+    private final JsonReader reader;
+    private InputStream stream;
+    private static final JsonFactory jsonFactory = new JsonFactory();
+
+    public JsonFeedReader(InputStream stream, DocumentTypeManager docMan) {
+        reader = new JsonReader(docMan, stream, jsonFactory);
+        this.stream = stream;
+    }
+
+    @Override
+    public void read(Operation operation) throws Exception {
+        DocumentOperation documentOperation = reader.next();
+
+        if (documentOperation == null) {
+            stream.close();
+            operation.setInvalid();
+            return;
+        }
+
+        if (documentOperation instanceof DocumentUpdate) {
+            operation.setDocumentUpdate((DocumentUpdate) documentOperation);
+        } else if (documentOperation instanceof DocumentRemove) {
+            operation.setRemove(documentOperation.getId());
+        } else if (documentOperation instanceof DocumentPut) {
+            operation.setDocument(((DocumentPut) documentOperation).getDocument());
+        } else {
+            throw new IllegalStateException("Got unknown class from JSON reader: " + documentOperation.getClass().getName());
+        }
+
+        operation.setCondition(documentOperation.getCondition());
+    }
+
+}
diff --git a/document/src/main/java/com/yahoo/document/json/JsonReader.java b/document/src/main/java/com/yahoo/document/json/JsonReader.java
new file mode 100644
index 00000000000..e5402d617bd
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/json/JsonReader.java
@@ -0,0 +1,773 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.json;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonToken;
+import com.google.common.annotations.Beta;
+import com.google.common.base.Preconditions;
+import com.yahoo.document.ArrayDataType;
+import com.yahoo.document.CollectionDataType;
+import com.yahoo.document.DataType;
+import com.yahoo.document.Document;
+import com.yahoo.document.DocumentId;
+import com.yahoo.document.DocumentOperation;
+import com.yahoo.document.DocumentPut;
+import com.yahoo.document.DocumentRemove;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.DocumentTypeManager;
+import com.yahoo.document.DocumentUpdate;
+import com.yahoo.document.Field;
+import com.yahoo.document.MapDataType;
+import com.yahoo.document.PositionDataType;
+import com.yahoo.document.TestAndSetCondition;
+import com.yahoo.document.WeightedSetDataType;
+import com.yahoo.document.datatypes.CollectionFieldValue;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.IntegerFieldValue;
+import com.yahoo.document.datatypes.MapFieldValue;
+import com.yahoo.document.datatypes.StructuredFieldValue;
+import com.yahoo.document.datatypes.TensorFieldValue;
+import com.yahoo.document.datatypes.WeightedSet;
+import com.yahoo.document.json.TokenBuffer.Token;
+import com.yahoo.document.update.FieldUpdate;
+import com.yahoo.document.update.MapValueUpdate;
+import com.yahoo.document.update.ValueUpdate;
+import com.yahoo.tensor.MapTensorBuilder;
+import org.apache.commons.codec.binary.Base64;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * Initialize Vespa documents/updates/removes from an InputStream containing a
+ * valid JSON representation of a feed.
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ * @since 5.1.25
+ */
+@Beta
+public class JsonReader {
+
+    private enum FieldOperation {
+        ADD, REMOVE
+    }
+
+    static final String MAP_KEY = "key";
+    static final String MAP_VALUE = "value";
+    static final String FIELDS = "fields";
+    static final String REMOVE = "remove";
+    static final String UPDATE_INCREMENT = "increment";
+    static final String UPDATE_DECREMENT = "decrement";
+    static final String UPDATE_MULTIPLY = "multiply";
+    static final String UPDATE_DIVIDE = "divide";
+    static final String TENSOR_DIMENSIONS = "dimensions";
+    static final String TENSOR_CELLS = "cells";
+    static final String TENSOR_ADDRESS = "address";
+    static final String TENSOR_VALUE = "value";
+
+    private static final String UPDATE = "update";
+    private static final String PUT = "put";
+    private static final String ID = "id";
+    private static final String CONDITION = "condition";
+    private static final String CREATE_IF_NON_EXISTENT = "create";
+    private static final String UPDATE_ASSIGN = "assign";
+    private static final String UPDATE_REMOVE = "remove";
+    private static final String UPDATE_MATCH = "match";
+    private static final String UPDATE_ADD = "add";
+    private static final String UPDATE_ELEMENT = "element";
+
+    private final JsonParser parser;
+    private TokenBuffer buffer = new TokenBuffer();
+    private final DocumentTypeManager typeManager;
+    private ReaderState state = ReaderState.AT_START;
+
+    static class DocumentParseInfo {
+        public DocumentId documentId;
+        public Optional<Boolean> create = Optional.empty();
+        Optional<String> condition = Optional.empty();
+        SupportedOperation operationType = null;
+    }
+
+    enum SupportedOperation {
+        PUT, UPDATE, REMOVE
+    }
+
+    enum ReaderState {
+        AT_START, READING, END_OF_FEED
+    }
+
+    public JsonReader(DocumentTypeManager typeManager, InputStream input, JsonFactory parserFactory) {
+        this.typeManager = typeManager;
+
+        try {
+            parser = parserFactory.createParser(input);
+        } catch (IOException e) {
+            state = ReaderState.END_OF_FEED;
+            throw new RuntimeException(e);
+        }
+    }
+
+    /**
+     * Reads a single operation. The operation is not expected to be part of an array. It only reads FIELDS.
+     * @param operationType the type of operation (update or put)
+     * @param docIdString document ID.
+     * @return the document
+     */
+    public DocumentOperation readSingleDocument(SupportedOperation operationType, String docIdString) {
+        DocumentId docId = new DocumentId(docIdString);
+        DocumentParseInfo documentParseInfo = parseToDocumentsFieldsAndInsertFieldsIntoBuffer(docId);
+        documentParseInfo.operationType = operationType;
+        DocumentOperation operation = createDocumentOperation(documentParseInfo);
+        operation.setCondition(TestAndSetCondition.fromConditionString(documentParseInfo.condition));
+        return operation;
+    }
+
+    public DocumentOperation next() {
+        switch (state) {
+            case AT_START:
+                JsonToken t = nextToken();
+                expectArrayStart(t);
+                state = ReaderState.READING;
+                break;
+            case END_OF_FEED:
+                return null;
+            case READING:
+                break;
+        }
+
+        Optional<DocumentParseInfo> documentParseInfo = parseDocument();
+
+        if (! documentParseInfo.isPresent()) {
+            state = ReaderState.END_OF_FEED;
+            return null;
+        }
+        DocumentOperation operation = createDocumentOperation(documentParseInfo.get());
+        operation.setCondition(TestAndSetCondition.fromConditionString(documentParseInfo.get().condition));
+        return operation;
+    }
+
+    private DocumentOperation createDocumentOperation(DocumentParseInfo documentParseInfo) {
+        DocumentType documentType = getDocumentTypeFromString(documentParseInfo.documentId.getDocType(), typeManager);
+        final DocumentOperation documentOperation;
+        try {
+            switch (documentParseInfo.operationType) {
+                case PUT:
+                    documentOperation = new DocumentPut(new Document(documentType, documentParseInfo.documentId));
+                    readPut((DocumentPut) documentOperation);
+                    verifyEndState();
+                    break;
+                case REMOVE:
+                    documentOperation = new DocumentRemove(documentParseInfo.documentId);
+                    break;
+                case UPDATE:
+                    documentOperation = new DocumentUpdate(documentType, documentParseInfo.documentId);
+                    readUpdate((DocumentUpdate) documentOperation);
+                    verifyEndState();
+                    break;
+                default:
+                    throw new IllegalStateException("Implementation out of sync with itself. This is a bug.");
+            }
+        } catch (JsonReaderException e) {
+            throw JsonReaderException.addDocId(e, documentParseInfo.documentId);
+        }
+        if (documentParseInfo.create.isPresent()) {
+            if (!(documentOperation instanceof DocumentUpdate)) {
+                throw new RuntimeException("Could not set create flag on non update operation.");
+            }
+            DocumentUpdate update = (DocumentUpdate) documentOperation;
+            update.setCreateIfNonExistent(documentParseInfo.create.get());
+        }
+        return documentOperation;
+    }
+
+    void readUpdate(DocumentUpdate next) {
+        if (buffer.size() == 0) {
+            bufferFields(nextToken());
+        }
+        populateUpdateFromBuffer(next);
+    }
+
+    void readPut(DocumentPut put) {
+        if (buffer.size() == 0) {
+            bufferFields(nextToken());
+        }
+        JsonToken t = buffer.currentToken();
+        try {
+            populateComposite(put.getDocument(), t);
+        } catch (JsonReaderException e) {
+            throw JsonReaderException.addDocId(e, put.getId());
+        }
+    }
+
+    private DocumentParseInfo parseToDocumentsFieldsAndInsertFieldsIntoBuffer(DocumentId documentId) {
+        long indentLevel = 0;
+        DocumentParseInfo documentParseInfo = new DocumentParseInfo();
+        documentParseInfo.documentId = documentId;
+        while (true) {
+            // we should now be at the start of a feed operation or at the end of the feed
+            JsonToken t = nextToken();
+            if (t == null) {
+                throw new IllegalArgumentException("Could not read document, no document?");
+            }
+            switch (t) {
+                case START_OBJECT:
+                    indentLevel++;
+                    break;
+                case END_OBJECT:
+                    indentLevel--;
+                    break;
+                case START_ARRAY:
+                    indentLevel+=10000L;
+                    break;
+                case END_ARRAY:
+                    indentLevel-=10000L;
+                    break;
+            }
+            if (indentLevel == 1 && (t == JsonToken.VALUE_TRUE || t == JsonToken.VALUE_FALSE)) {
+                try {
+                    if (CREATE_IF_NON_EXISTENT.equals(parser.getCurrentName())) {
+                        documentParseInfo.create = Optional.ofNullable(parser.getBooleanValue());
+                        continue;
+                    }
+                } catch (IOException e) {
+                    throw new RuntimeException("Got IO exception while parsing document", e);
+                }
+            }
+            if (indentLevel == 2L && t == JsonToken.START_OBJECT) {
+
+                try {
+                    if (!FIELDS.equals(parser.getCurrentName())) {
+                        continue;
+                    }
+                } catch (IOException e) {
+                    throw new RuntimeException("Got IO exception while parsing document", e);
+                }
+                bufferFields(t);
+                break;
+            }
+        }
+        return documentParseInfo;
+    }
+
+    private void verifyEndState() {
+        Preconditions.checkState(buffer.nesting() == 0, "Nesting not zero at end of operation");
+        expectObjectEnd(buffer.currentToken());
+        Preconditions.checkState(buffer.next() == null, "Dangling data at end of operation");
+        Preconditions.checkState(buffer.size() == 0, "Dangling data at end of operation");
+    }
+
+    private void populateUpdateFromBuffer(DocumentUpdate update) {
+        expectObjectStart(buffer.currentToken());
+        int localNesting = buffer.nesting();
+        JsonToken t = buffer.next();
+
+        while (localNesting <= buffer.nesting()) {
+            expectObjectStart(t);
+            String fieldName = buffer.currentName();
+            Field field = update.getType().getField(fieldName);
+            addFieldUpdates(update, field);
+            t = buffer.next();
+        }
+    }
+
+    private void addFieldUpdates(DocumentUpdate update, Field field) {
+        validateFieldUpdates(update, field);
+        int localNesting = buffer.nesting();
+        FieldUpdate fieldUpdate = FieldUpdate.create(field);
+
+        buffer.next();
+        while (localNesting <= buffer.nesting()) {
+            switch (buffer.currentName()) {
+            case UPDATE_REMOVE:
+                createAddsOrRemoves(field, fieldUpdate, FieldOperation.REMOVE);
+                break;
+            case UPDATE_ADD:
+                createAddsOrRemoves(field, fieldUpdate, FieldOperation.ADD);
+                break;
+            case UPDATE_MATCH:
+                fieldUpdate.addValueUpdate(createMapUpdate(field));
+                break;
+            default:
+                String action = buffer.currentName();
+                fieldUpdate.addValueUpdate(readSingleUpdate(field.getDataType(), action));
+            }
+            buffer.next();
+        }
+        update.addFieldUpdate(fieldUpdate);
+    }
+
+    private static void validateFieldUpdates(DocumentUpdate update, Field field) {
+        if (field.getDataType() == DataType.TENSOR) {
+            throw new IllegalArgumentException("Updates to fields of type TENSOR is not yet supported ("
+                    + "id='" + update.getId().toString() + "', field='" + field.getName() + "')");
+        }
+    }
+
+    @SuppressWarnings("rawtypes")
+    private ValueUpdate createMapUpdate(Field field) {
+        buffer.next();
+        MapValueUpdate m = (MapValueUpdate) createMapUpdate(field.getDataType(), null, null);
+        buffer.next();
+        // must generate the field value in parallell with the actual
+        return m;
+
+    }
+
+    @SuppressWarnings({ "rawtypes", "unchecked" })
+    private ValueUpdate createMapUpdate(DataType currentLevel, FieldValue keyParent, FieldValue topLevelKey) {
+        TokenBuffer.Token element = buffer.prefetchScalar(UPDATE_ELEMENT);
+        if (UPDATE_ELEMENT.equals(buffer.currentName())) {
+            buffer.next();
+        }
+
+        FieldValue key = keyTypeForMapUpdate(element, currentLevel);
+        if (keyParent != null) {
+            ((CollectionFieldValue) keyParent).add(key);
+        }
+        // structure is: [(match + element)*, (element + action)]
+        // match will always have element, and either match or action
+        if (!UPDATE_MATCH.equals(buffer.currentName())) {
+            // we have reached an action...
+            if (topLevelKey == null) {
+                return ValueUpdate.createMap(key, readSingleUpdate(valueTypeForMapUpdate(currentLevel), buffer.currentName()));
+            } else {
+                return ValueUpdate.createMap(topLevelKey, readSingleUpdate(valueTypeForMapUpdate(currentLevel), buffer.currentName()));
+            }
+        } else {
+            // next level of matching
+            if (topLevelKey == null) {
+                return createMapUpdate(valueTypeForMapUpdate(currentLevel), key, key);
+            } else {
+                return createMapUpdate(valueTypeForMapUpdate(currentLevel), key, topLevelKey);
+            }
+        }
+    }
+
+    private DataType valueTypeForMapUpdate(DataType parentType) {
+        if (parentType instanceof WeightedSetDataType) {
+            return DataType.INT;
+        } else if (parentType instanceof CollectionDataType) {
+            return ((CollectionDataType) parentType).getNestedType();
+        } else if (parentType instanceof MapDataType) {
+            return ((MapDataType) parentType).getValueType();
+        } else {
+            throw new UnsupportedOperationException("Unexpected parent type: " + parentType);
+        }
+    }
+
+    private FieldValue keyTypeForMapUpdate(Token element, DataType expectedType) {
+        FieldValue v;
+        if (expectedType instanceof ArrayDataType) {
+            v = new IntegerFieldValue(Integer.valueOf(element.text));
+        } else if (expectedType instanceof WeightedSetDataType) {
+            v = ((WeightedSetDataType) expectedType).getNestedType().createFieldValue(element.text);
+        } else if (expectedType instanceof MapDataType) {
+            v = ((MapDataType) expectedType).getKeyType().createFieldValue(element.text);
+        } else {
+            throw new IllegalArgumentException("Container type " + expectedType + " not supported for match update.");
+        }
+        return v;
+    }
+
+    @SuppressWarnings("rawtypes")
+    private ValueUpdate readSingleUpdate(DataType expectedType, String action) {
+        ValueUpdate update;
+
+        switch (action) {
+            case UPDATE_ASSIGN:
+                update = (buffer.currentToken() == JsonToken.VALUE_NULL)
+                        ? ValueUpdate.createClear()
+                        : ValueUpdate.createAssign(readSingleValue(buffer.currentToken(), expectedType));
+                break;
+                // double is silly, but it's what is used internally anyway
+            case UPDATE_INCREMENT:
+                update = ValueUpdate.createIncrement(Double.valueOf(buffer.currentText()));
+                break;
+            case UPDATE_DECREMENT:
+                update = ValueUpdate.createDecrement(Double.valueOf(buffer.currentText()));
+                break;
+            case UPDATE_MULTIPLY:
+                update = ValueUpdate.createMultiply(Double.valueOf(buffer.currentText()));
+                break;
+            case UPDATE_DIVIDE:
+                update = ValueUpdate.createDivide(Double.valueOf(buffer.currentText()));
+                break;
+            default:
+                throw new IllegalArgumentException("Operation \"" + buffer.currentName() + "\" not implemented.");
+        }
+        return update;
+    }
+
+    // yes, this suppresswarnings ugliness is by intention, the code relies on
+    // the contracts in the builders
+    @SuppressWarnings({ "cast", "rawtypes", "unchecked" })
+    private void createAddsOrRemoves(Field field, FieldUpdate update, FieldOperation op) {
+        FieldValue container = field.getDataType().createFieldValue();
+        FieldUpdate singleUpdate;
+        int initNesting = buffer.nesting();
+        JsonToken token;
+
+        Preconditions.checkState(buffer.currentToken().isStructStart(), "Expected start of composite, got %s", buffer.currentToken());
+        if (container instanceof CollectionFieldValue) {
+            token = buffer.next();
+            DataType valueType = ((CollectionFieldValue) container).getDataType().getNestedType();
+            if (container instanceof WeightedSet) {
+                // these are objects with string keys (which are the nested
+                // types) and values which are the weight
+                WeightedSet weightedSet = (WeightedSet) container;
+                fillWeightedSetUpdate(initNesting, valueType, weightedSet);
+                if (op == FieldOperation.REMOVE) {
+                    singleUpdate = FieldUpdate.createRemoveAll(field, weightedSet);
+                } else {
+                    singleUpdate = FieldUpdate.createAddAll(field, weightedSet);
+
+                }
+            } else {
+                List<FieldValue> arrayContents = new ArrayList<>();
+                token = fillArrayUpdate(initNesting, token, valueType, arrayContents);
+                if (token != JsonToken.END_ARRAY) {
+                    throw new IllegalStateException("Expected END_ARRAY. Got '" + token + "'.");
+                }
+                if (op == FieldOperation.REMOVE) {
+                    singleUpdate = FieldUpdate.createRemoveAll(field, arrayContents);
+                } else {
+                    singleUpdate = FieldUpdate.createAddAll(field, arrayContents);
+                }
+            }
+        } else {
+            throw new UnsupportedOperationException(
+                    "Trying to add or remove from a field of a type the reader does not know how to handle: "
+                            + container.getClass().getName());
+        }
+        expectCompositeEnd(buffer.currentToken());
+        update.addAll(singleUpdate);
+    }
+
+    private JsonToken fillArrayUpdate(int initNesting, JsonToken initToken, DataType valueType, List<FieldValue> arrayContents) {
+        JsonToken token = initToken;
+        while (buffer.nesting() >= initNesting) {
+            arrayContents.add(readSingleValue(token, valueType));
+            token = buffer.next();
+        }
+        return token;
+    }
+
+    private void fillWeightedSetUpdate(int initNesting, DataType valueType, @SuppressWarnings("rawtypes") WeightedSet weightedSet) {
+        iterateThroughWeightedSet(initNesting, valueType, weightedSet);
+    }
+
+    @SuppressWarnings({ "rawtypes", "unchecked" })
+    private void iterateThroughWeightedSet(int initNesting, DataType valueType, WeightedSet weightedSet) {
+        while (buffer.nesting() >= initNesting) {
+            // XXX the keys are defined in the spec to always be represented as strings
+            FieldValue v = valueType.createFieldValue(buffer.currentName());
+            weightedSet.put(v, Integer.valueOf(buffer.currentText()));
+            buffer.next();
+        }
+    }
+
+    // TODO populateComposite is extremely similar to add/remove, refactor
+    // yes, this suppresswarnings ugliness is by intention, the code relies on the contracts in the builders
+    @SuppressWarnings({ "cast", "rawtypes" })
+    private void populateComposite(FieldValue parent, JsonToken token) {
+        if ((token != JsonToken.START_OBJECT) && (token != JsonToken.START_ARRAY)) {
+            throw new IllegalArgumentException("Expected '[' or '{'. Got '" + token + "'.");
+        }
+        if (parent instanceof CollectionFieldValue) {
+            DataType valueType = ((CollectionFieldValue) parent).getDataType().getNestedType();
+            if (parent instanceof WeightedSet) {
+                fillWeightedSet(valueType, (WeightedSet) parent);
+            } else {
+                fillArray((CollectionFieldValue) parent, valueType);
+            }
+        } else if (parent instanceof MapFieldValue) {
+            fillMap((MapFieldValue) parent);
+        } else if (parent instanceof StructuredFieldValue) {
+            fillStruct((StructuredFieldValue) parent);
+        } else if (parent instanceof TensorFieldValue) {
+            fillTensor((TensorFieldValue) parent);
+        } else {
+            throw new IllegalStateException("Has created a composite field"
+                    + " value the reader does not know how to handle: "
+                    + parent.getClass().getName() + " This is a bug. token = " + token);
+        }
+        expectCompositeEnd(buffer.currentToken());
+    }
+
+    private void expectCompositeEnd(JsonToken token) {
+        Preconditions.checkState(token.isStructEnd(), "Expected end of composite, got %s", token);
+    }
+
+    private void fillStruct(StructuredFieldValue parent) {
+        // do note the order of initializing initNesting and token is relevant for empty docs
+        int initNesting = buffer.nesting();
+        JsonToken token = buffer.next();
+
+        while (buffer.nesting() >= initNesting) {
+            Field f = getField(parent);
+            try {
+                FieldValue v = readSingleValue(token, f.getDataType());
+                parent.setFieldValue(f, v);
+                token = buffer.next();
+            } catch (IllegalArgumentException e) {
+                throw new JsonReaderException(f, e);
+            }
+        }
+    }
+
+    private Field getField(StructuredFieldValue parent) {
+        Field f = parent.getField(buffer.currentName());
+        if (f == null) {
+            throw new NullPointerException("Could not get field \"" + buffer.currentName() +
+                    "\" in the structure of type \"" + parent.getDataType().getDataTypeName() + "\".");
+        }
+        return f;
+    }
+
+    @SuppressWarnings({ "rawtypes", "cast", "unchecked" })
+    private void fillMap(MapFieldValue parent) {
+        JsonToken token = buffer.currentToken();
+        int initNesting = buffer.nesting();
+        expectArrayStart(token);
+        token = buffer.next();
+        DataType keyType = parent.getDataType().getKeyType();
+        DataType valueType = parent.getDataType().getValueType();
+        while (buffer.nesting() >= initNesting) {
+            FieldValue key = null;
+            FieldValue value = null;
+            expectObjectStart(token);
+            token = buffer.next();
+            for (int i = 0; i < 2; ++i) {
+                if (MAP_KEY.equals(buffer.currentName())) {
+                    key = readSingleValue(token, keyType);
+                } else if (MAP_VALUE.equals(buffer.currentName())) {
+                    value = readSingleValue(token, valueType);
+                }
+                token = buffer.next();
+            }
+            Preconditions.checkState(key != null && value != null, "Missing key or value for map entry.");
+            parent.put(key, value);
+
+            expectObjectEnd(token);
+            token = buffer.next(); // array end or next entry
+        }
+    }
+
+    private void expectArrayStart(JsonToken token) {
+        Preconditions.checkState(token == JsonToken.START_ARRAY, "Expected start of array, got %s", token);
+    }
+
+    private void expectObjectStart(JsonToken token) {
+        Preconditions.checkState(token == JsonToken.START_OBJECT, "Expected start of JSON object, got %s", token);
+    }
+
+    private void expectObjectEnd(JsonToken token) {
+        Preconditions.checkState(token == JsonToken.END_OBJECT, "Expected end of JSON object, got %s", token);
+    }
+
+    @SuppressWarnings({ "unchecked", "rawtypes" })
+    private void fillArray(CollectionFieldValue parent, DataType valueType) {
+        int initNesting = buffer.nesting();
+        expectArrayStart(buffer.currentToken());
+        JsonToken token = buffer.next();
+        while (buffer.nesting() >= initNesting) {
+            parent.add(readSingleValue(token, valueType));
+            token = buffer.next();
+        }
+    }
+
+    private void fillWeightedSet(DataType valueType,
+            @SuppressWarnings("rawtypes") WeightedSet weightedSet) {
+        int initNesting = buffer.nesting();
+        expectObjectStart(buffer.currentToken());
+        buffer.next();
+        iterateThroughWeightedSet(initNesting, valueType, weightedSet);
+    }
+
+    private void fillTensor(TensorFieldValue tensorFieldValue) {
+        expectObjectStart(buffer.currentToken());
+        int initNesting = buffer.nesting();
+        MapTensorBuilder tensorBuilder = new MapTensorBuilder();
+        for (buffer.next(); buffer.nesting() >= initNesting; buffer.next()) {
+            if (TENSOR_DIMENSIONS.equals(buffer.currentName())) {
+                readTensorDimensions(tensorBuilder);
+            } else if (TENSOR_CELLS.equals(buffer.currentName())) {
+                readTensorCells(tensorBuilder);
+            }
+        }
+        expectObjectEnd(buffer.currentToken());
+        tensorFieldValue.assign(tensorBuilder.build());
+    }
+
+    private void readTensorDimensions(MapTensorBuilder tensorBuilder) {
+        expectArrayStart(buffer.currentToken());
+        int initNesting = buffer.nesting();
+        for (buffer.next(); buffer.nesting() >= initNesting; buffer.next()) {
+            if (buffer.currentToken().isScalarValue()) {
+                String dimension = buffer.currentText();
+                tensorBuilder.dimension(dimension);
+            }
+        }
+        expectCompositeEnd(buffer.currentToken());
+    }
+
+    private void readTensorCells(MapTensorBuilder tensorBuilder) {
+        expectArrayStart(buffer.currentToken());
+        int initNesting = buffer.nesting();
+        for (buffer.next(); buffer.nesting() >= initNesting; buffer.next()) {
+            readTensorCell(tensorBuilder.cell());
+        }
+        expectCompositeEnd(buffer.currentToken());
+    }
+
+    private void readTensorCell(MapTensorBuilder.CellBuilder cellBuilder) {
+        expectObjectStart(buffer.currentToken());
+        int initNesting = buffer.nesting();
+        double cellValue = 0.0;
+        for (buffer.next(); buffer.nesting() >= initNesting; buffer.next()) {
+            String currentName = buffer.currentName();
+            if (TENSOR_ADDRESS.equals(currentName)) {
+                readTensorAddress(cellBuilder);
+            } else if (TENSOR_VALUE.equals(currentName)) {
+                cellValue = Double.valueOf(buffer.currentText());
+            }
+        }
+        expectObjectEnd(buffer.currentToken());
+        cellBuilder.value(cellValue);
+    }
+
+    private void readTensorAddress(MapTensorBuilder.CellBuilder cellBuilder) {
+        expectObjectStart(buffer.currentToken());
+        int initNesting = buffer.nesting();
+        for (buffer.next(); buffer.nesting() >= initNesting; buffer.next()) {
+            String dimension = buffer.currentName();
+            String label = buffer.currentText();
+            cellBuilder.label(dimension, label);
+        }
+        expectObjectEnd(buffer.currentToken());
+    }
+
+    private FieldValue readSingleValue(JsonToken t, DataType expectedType) {
+        if (t.isScalarValue()) {
+            return readAtomic(expectedType);
+        } else {
+            FieldValue v = expectedType.createFieldValue();
+            populateComposite(v, t);
+            return v;
+        }
+    }
+
+    private FieldValue readAtomic(DataType expectedType) {
+        if (expectedType.equals(DataType.RAW)) {
+            return expectedType.createFieldValue(new Base64().decode(buffer.currentText()));
+        } else if (expectedType.equals(PositionDataType.INSTANCE)) {
+            return PositionDataType.fromString(buffer.currentText());
+        } else {
+            return expectedType.createFieldValue(buffer.currentText());
+        }
+    }
+
+    private void bufferFields(JsonToken current) {
+        buffer.bufferObject(current, parser);
+    }
+
+    private boolean jsonTokenIsBooleanOrString(JsonToken jsonToken) {
+        return jsonToken == JsonToken.VALUE_STRING || jsonToken == JsonToken.VALUE_TRUE || jsonToken == JsonToken.VALUE_FALSE;
+    }
+
+    Optional<DocumentParseInfo> parseDocument() {
+        Optional<Boolean> create = Optional.empty();
+        // we should now be at the start of a feed operation or at the end of the feed
+        JsonToken token = nextToken();
+        if (token == JsonToken.END_ARRAY) {
+            return Optional.empty(); // end of feed
+        }
+        expectObjectStart(token);
+
+        DocumentParseInfo documentParseInfo = new DocumentParseInfo();
+
+        while (true) {
+            try {
+                token = nextToken();
+                if ((token == JsonToken.VALUE_TRUE || token == JsonToken.VALUE_FALSE) &&
+                        CREATE_IF_NON_EXISTENT.equals(parser.getCurrentName())) {
+                    documentParseInfo.create = Optional.of(token == JsonToken.VALUE_TRUE);
+                    continue;
+                }
+                if (token == JsonToken.VALUE_STRING && CONDITION.equals(parser.getCurrentName())) {
+                    documentParseInfo.condition = Optional.of(parser.getText());
+                    continue;
+                }
+                if (token == JsonToken.START_OBJECT) {
+                    try {
+                        if (!FIELDS.equals(parser.getCurrentName())) {
+                            throw new IllegalArgumentException("Unexpected object key: " + parser.getCurrentName());
+                        }
+                    } catch (IOException e) {
+                        // TODO more specific wrapping
+                        throw new RuntimeException(e);
+                    }
+                    bufferFields(token);
+                    continue;
+                }
+                if (token == JsonToken.END_OBJECT) {
+                    if (documentParseInfo.documentId == null) {
+                        throw new RuntimeException("Did not find document operation");
+                    }
+                    return Optional.of(documentParseInfo);
+                }
+                if (token == JsonToken.VALUE_STRING) {
+                    documentParseInfo.operationType = operationNameToOperationType(parser.getCurrentName());
+                    documentParseInfo.documentId = new DocumentId(parser.getText());
+                    continue;
+                }
+                throw new RuntimeException("Expected document start or document operation.");
+            } catch (IOException e) {
+                throw new IllegalStateException(e);
+            }
+
+        }
+    }
+
+    private static SupportedOperation operationNameToOperationType(String operationName) {
+        switch (operationName) {
+            case PUT:
+            case ID:
+                return SupportedOperation.PUT;
+            case REMOVE:
+                return SupportedOperation.REMOVE;
+            case UPDATE:
+                return SupportedOperation.UPDATE;
+            default:
+                throw new IllegalArgumentException(
+                        "Got " + operationName + " as document operation, only \"put\", " +
+                                "\"remove\" and \"update\" are supported.");
+        }
+    }
+
+    DocumentType readDocumentType(DocumentId docId) {
+        return getDocumentTypeFromString(docId.getDocType(), typeManager);
+    }
+
+    private static DocumentType getDocumentTypeFromString(String docTypeString, DocumentTypeManager typeManager) {
+        final DocumentType docType = typeManager.getDocumentType(docTypeString);
+        if (docType == null) {
+            throw new IllegalArgumentException(String.format("Document type %s does not exist", docTypeString));
+        }
+        return docType;
+    }
+
+    private JsonToken nextToken() {
+        try {
+            return parser.nextValue();
+        } catch (IOException e) {
+            // Jackson is not able to recover from structural parse errors
+            state = ReaderState.END_OF_FEED;
+            throw new RuntimeException(e);
+        }
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/json/JsonReaderException.java b/document/src/main/java/com/yahoo/document/json/JsonReaderException.java
new file mode 100644
index 00000000000..3346ecc3bd6
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/json/JsonReaderException.java
@@ -0,0 +1,45 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.json;
+
+import com.yahoo.document.DocumentId;
+import com.yahoo.document.Field;
+
+/**
+ * @author bjorncs
+ */
+public class JsonReaderException extends RuntimeException {
+    public final DocumentId docId;
+    public final Field field;
+    public final Throwable cause;
+
+    public JsonReaderException(DocumentId docId, Field field, Throwable cause) {
+        super(createErrorMessage(docId, field, cause), cause);
+        this.docId = docId;
+        this.field = field;
+        this.cause = cause;
+    }
+
+    public JsonReaderException(Field field, Throwable cause) {
+        super(createErrorMessage(null, field, cause), cause);
+        this.docId = null;
+        this.field = field;
+        this.cause = cause;
+    }
+
+    public static JsonReaderException addDocId(JsonReaderException oldException, DocumentId docId) {
+        return new JsonReaderException(docId, oldException.field, oldException.cause);
+    }
+
+    private static String createErrorMessage(DocumentId docId, Field field, Throwable cause) {
+        return String.format("Error in document '%s' - could not parse field '%s' of type '%s': %s",
+                docId, field.getName(), field.getDataType().getName(), cause.getMessage());
+    }
+
+    public DocumentId getDocId() {
+        return docId;
+    }
+
+    public Field getField() {
+        return field;
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/json/JsonWriter.java b/document/src/main/java/com/yahoo/document/json/JsonWriter.java
new file mode 100644
index 00000000000..79a1c040cbc
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/json/JsonWriter.java
@@ -0,0 +1,473 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.json;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+
+import com.yahoo.document.datatypes.*;
+import com.yahoo.tensor.Tensor;
+import com.yahoo.tensor.TensorAddress;
+import org.apache.commons.codec.binary.Base64;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.yahoo.document.Document;
+import com.yahoo.document.DocumentId;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.Field;
+import com.yahoo.document.PositionDataType;
+import com.yahoo.document.annotation.AnnotationReference;
+import com.yahoo.document.serialization.DocumentWriter;
+import com.yahoo.vespa.objects.FieldBase;
+import com.yahoo.vespa.objects.Serializer;
+
+import edu.umd.cs.findbugs.annotations.NonNull;
+
+/**
+ * Serialize Document and other FieldValue instances as JSON.
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+public class JsonWriter implements DocumentWriter {
+
+    private static final JsonFactory jsonFactory = new JsonFactory();
+    private final JsonGenerator generator;
+    private final Base64 base64Encoder = new Base64();
+
+    // I really hate exception unsafe constructors, but the alternative
+    // requires generator to not be a final
+    /**
+     *
+     * @param out
+     *            the target output stream
+     * @throws RuntimeException
+     *             if unable to create the internal JSON generator
+     */
+    public JsonWriter(OutputStream out) {
+        this(createPrivateGenerator(out));
+    }
+
+    private static JsonGenerator createPrivateGenerator(OutputStream out) {
+        try {
+            return jsonFactory.createGenerator(out);
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    /**
+     * Create a Document writer which will write to the input JSON generator.
+     * JsonWriter will not close the generator and only flush it explicitly
+     * after having written a full Document instance. In other words, JsonWriter
+     * will <i>not</i> take ownership of the generator.
+     *
+     * @param generator
+     *            the output JSON generator
+     */
+    public JsonWriter(JsonGenerator generator) {
+        this.generator = generator;
+    }
+
+    /**
+     * This method will only be called if there is some type which is not
+     * properly supported in the API, or if something has been changed without
+     * updating this class. This implementation throws an exception if it is
+     * reached.
+     *
+     * @throws UnsupportedOperationException
+     *             if invoked
+     */
+    @Override
+    public void write(FieldBase field, FieldValue value) {
+        throw new UnsupportedOperationException("Serializing "
+                + value.getClass().getName() + " is not supported.");
+    }
+
+    @Override
+    public void write(FieldBase field, Document value) {
+        try {
+            fieldNameIfNotNull(field);
+            generator.writeStartObject();
+            // this makes it impossible to refeed directly, not sure what's correct
+            // perhaps just change to "put"?
+            generator.writeStringField("id", value.getId().toString());
+            generator.writeObjectFieldStart(JsonReader.FIELDS);
+            for (Iterator<Entry<Field, FieldValue>> i = value.iterator(); i
+                    .hasNext();) {
+                Entry<Field, FieldValue> entry = i.next();
+                entry.getValue().serialize(entry.getKey(), this);
+            }
+            generator.writeEndObject();
+            generator.writeEndObject();
+            generator.flush();
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    @Override
+    public <T extends FieldValue> void write(FieldBase field, Array<T> value) {
+        try {
+            fieldNameIfNotNull(field);
+            generator.writeStartArray();
+            for (Iterator<T> i = value.iterator(); i.hasNext();) {
+                i.next().serialize(null, this);
+            }
+            generator.writeEndArray();
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+
+    }
+
+    private void fieldNameIfNotNull(FieldBase field) {
+        if (field != null) {
+            try {
+                generator.writeFieldName(field.getName());
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+        }
+    }
+
+    @Override
+    public <K extends FieldValue, V extends FieldValue> void write(
+            FieldBase field, MapFieldValue<K, V> map) {
+        fieldNameIfNotNull(field);
+        try {
+            generator.writeStartArray();
+            for (Map.Entry<K, V> entry : map.entrySet()) {
+                generator.writeStartObject();
+                generator.writeFieldName(JsonReader.MAP_KEY);
+                entry.getKey().serialize(null, this);
+                generator.writeFieldName(JsonReader.MAP_VALUE);
+                entry.getValue().serialize(null, this);
+                generator.writeEndObject();
+            }
+            generator.writeEndArray();
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    @Override
+    public void write(FieldBase field, ByteFieldValue value) {
+        putByte(field, value.getByte());
+    }
+
+    @Override
+    public <T extends FieldValue> void write(FieldBase field,
+            CollectionFieldValue<T> value) {
+        fieldNameIfNotNull(field);
+        try {
+            generator.writeStartArray();
+            for (Iterator<T> i = value.iterator(); i.hasNext();) {
+                i.next().serialize(null, this);
+            }
+            generator.writeEndArray();
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    @Override
+    public void write(FieldBase field, DoubleFieldValue value) {
+        putDouble(field, value.getDouble());
+    }
+
+    @Override
+    public void write(FieldBase field, FloatFieldValue value) {
+        putFloat(field, value.getFloat());
+    }
+
+    @Override
+    public void write(FieldBase field, IntegerFieldValue value) {
+        putInt(field, value.getInteger());
+    }
+
+    @Override
+    public void write(FieldBase field, LongFieldValue value) {
+        putLong(field, value.getLong());
+    }
+
+    @Override
+    public void write(FieldBase field, Raw value) {
+        put(field, value.getByteBuffer());
+    }
+
+    @Override
+    public void write(FieldBase field, PredicateFieldValue value) {
+        put(field, value.toString());
+    }
+
+    @Override
+    public void write(FieldBase field, StringFieldValue value) {
+        put(field, value.getString());
+    }
+
+    @Override
+    public void write(FieldBase field, TensorFieldValue value) {
+        try {
+            fieldNameIfNotNull(field);
+            generator.writeStartObject();
+            if (value.getTensor().isPresent()) {
+                Tensor tensor = value.getTensor().get();
+                writeTensorDimensions(tensor.dimensions());
+                writeTensorCells(tensor.cells());
+            }
+            generator.writeEndObject();
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    private void writeTensorDimensions(Set<String> dimensions) throws IOException {
+        generator.writeArrayFieldStart(JsonReader.TENSOR_DIMENSIONS);
+        for (String dimension : dimensions) {
+            generator.writeString(dimension);
+        }
+        generator.writeEndArray();
+    }
+
+    private void writeTensorCells(Map<TensorAddress, Double> cells) throws IOException {
+        generator.writeArrayFieldStart(JsonReader.TENSOR_CELLS);
+        for (Map.Entry<TensorAddress, Double> cell : cells.entrySet()) {
+            generator.writeStartObject();
+            writeTensorAddress(cell.getKey());
+            generator.writeNumberField(JsonReader.TENSOR_VALUE, cell.getValue());
+            generator.writeEndObject();
+        }
+        generator.writeEndArray();
+    }
+
+    private void writeTensorAddress(TensorAddress address) throws IOException {
+        generator.writeObjectFieldStart(JsonReader.TENSOR_ADDRESS);
+        for (TensorAddress.Element element : address.elements()) {
+            generator.writeStringField(element.dimension(), element.label());
+        }
+        generator.writeEndObject();
+    }
+
+    @Override
+    public void write(FieldBase field, Struct value) {
+        if (value.getDataType() == PositionDataType.INSTANCE) {
+            put(field, PositionDataType.renderAsString(value));
+            return;
+        }
+        fieldNameIfNotNull(field);
+        try {
+            generator.writeStartObject();
+            for (Iterator<Entry<Field, FieldValue>> i = value.iterator(); i
+                    .hasNext();) {
+                Entry<Field, FieldValue> entry = i.next();
+                entry.getValue().serialize(entry.getKey(), this);
+            }
+            generator.writeEndObject();
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    @Override
+    public void write(FieldBase field, StructuredFieldValue value) {
+        fieldNameIfNotNull(field);
+        try {
+            generator.writeStartObject();
+            for (Iterator<Entry<Field, FieldValue>> i = value.iterator(); i
+                    .hasNext();) {
+                Entry<Field, FieldValue> entry = i.next();
+                entry.getValue().serialize(entry.getKey(), this);
+            }
+            generator.writeEndObject();
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    @Override
+    public <T extends FieldValue> void write(FieldBase field,
+            WeightedSet<T> value) {
+        fieldNameIfNotNull(field);
+        try {
+            generator.writeStartObject();
+            // entrySet() is deprecated and there is no entry iterator
+            for (T key : value.keySet()) {
+                Integer weight = value.get(key);
+                // key.toString() is according to spec
+                generator.writeNumberField(key.toString(), weight);
+            }
+            generator.writeEndObject();
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    @Override
+    public void write(FieldBase field, AnnotationReference value) {
+        // not yet implemented, it's not available in XML either
+        // TODO implement
+    }
+
+    @Override
+    public Serializer putByte(FieldBase field, byte value) {
+        fieldNameIfNotNull(field);
+        try {
+            generator.writeNumber(value);
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+        return this;
+    }
+
+    @Override
+    public Serializer putShort(FieldBase field, short value) {
+        fieldNameIfNotNull(field);
+        try {
+            generator.writeNumber(value);
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+        return this;
+    }
+
+    @Override
+    public Serializer putInt(FieldBase field, int value) {
+        fieldNameIfNotNull(field);
+        try {
+            generator.writeNumber(value);
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+        return this;
+    }
+
+    @Override
+    public Serializer putLong(FieldBase field, long value) {
+        fieldNameIfNotNull(field);
+        try {
+            generator.writeNumber(value);
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+        return this;
+    }
+
+    @Override
+    public Serializer putFloat(FieldBase field, float value) {
+        fieldNameIfNotNull(field);
+        try {
+            generator.writeNumber(value);
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+        return this;
+    }
+
+    @Override
+    public Serializer putDouble(FieldBase field, double value) {
+        fieldNameIfNotNull(field);
+        try {
+            generator.writeNumber(value);
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+        return this;
+    }
+
+    @Override
+    public Serializer put(FieldBase field, byte[] value) {
+        return put(field, ByteBuffer.wrap(value));
+    }
+
+    @Override
+    public Serializer put(FieldBase field, ByteBuffer raw) {
+        final byte[] data = new byte[raw.remaining()];
+        final int origPosition = raw.position();
+
+        fieldNameIfNotNull(field);
+        // base64encoder has no encode methods with offset and
+        // limit, so no use trying to get at the backing array if
+        // available anyway
+        raw.get(data);
+        raw.position(origPosition);
+        try {
+            generator.writeString(base64Encoder.encodeToString(data));
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+        return this;
+    }
+
+    @Override
+    public Serializer put(FieldBase field, String value) {
+        if (value.length() == 0) {
+            return this;
+        }
+        fieldNameIfNotNull(field);
+        try {
+            generator.writeString(value);
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+        return this;
+    }
+
+    @Override
+    public void write(Document document) {
+        write(null, document);
+    }
+
+    @Override
+    public void write(DocumentId id) {
+        // NOP, fetched from Document
+    }
+
+    @Override
+    public void write(DocumentType type) {
+        // NOP, fetched from Document
+    }
+
+    /**
+     * Utility method to easily serialize a single document.
+     *
+     * @param document
+     *            the document to be serialized
+     * @return the input document serialised as UTF-8 encoded JSON
+     */
+    public static byte[] toByteArray(@NonNull Document document) {
+        ByteArrayOutputStream out = new ByteArrayOutputStream();
+        JsonWriter writer = new JsonWriter(out);
+        writer.write(document);
+        return out.toByteArray();
+    }
+
+    /**
+     * Utility method to easily serialize a single document ID as a remove
+     * operation.
+     *
+     * @param docId
+     *            the document to remove or which has been removed
+     * @return a document remove operation serialised as UTF-8 encoded JSON for
+     *         the input document ID
+     */
+    public static byte[] documentRemove(@NonNull DocumentId docId) {
+        ByteArrayOutputStream out = new ByteArrayOutputStream();
+        try {
+            JsonGenerator throwAway = jsonFactory.createGenerator(out);
+            throwAway.writeStartObject();
+            throwAway.writeStringField(JsonReader.REMOVE, docId.toString());
+            throwAway.writeEndObject();
+            throwAway.close();
+        } catch (IOException e) {
+            // Under normal circumstances, nothing here will be triggered
+            throw new RuntimeException(e);
+        }
+        return out.toByteArray();
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/json/SingleDocumentParser.java b/document/src/main/java/com/yahoo/document/json/SingleDocumentParser.java
new file mode 100644
index 00000000000..2b210cb2ee5
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/json/SingleDocumentParser.java
@@ -0,0 +1,55 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.json;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.yahoo.document.DocumentOperation;
+import com.yahoo.document.DocumentPut;
+import com.yahoo.document.DocumentTypeManager;
+import com.yahoo.document.DocumentUpdate;
+import com.yahoo.vespaxmlparser.VespaXMLFeedReader;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+/**
+ * Parser that supports parsing PUT operation and UPDATE operation.
+ *
+ * @author dybdahl
+ */
+public class SingleDocumentParser {
+    private static final JsonFactory jsonFactory = new JsonFactory();
+    private DocumentTypeManager docMan;
+
+    public SingleDocumentParser(DocumentTypeManager docMan) {
+        this.docMan = docMan;
+    }
+
+    public VespaXMLFeedReader.Operation parsePut(InputStream inputStream, String docId) {
+        return parse(inputStream, docId, JsonReader.SupportedOperation.PUT);
+    }
+
+    public VespaXMLFeedReader.Operation parseUpdate(InputStream inputStream, String docId)  {
+        return parse(inputStream, docId, JsonReader.SupportedOperation.UPDATE);
+    }
+
+    private VespaXMLFeedReader.Operation parse(InputStream inputStream, String docId, JsonReader.SupportedOperation supportedOperation)  {
+        final JsonReader reader = new JsonReader(docMan, inputStream, jsonFactory);
+        final DocumentOperation documentOperation = reader.readSingleDocument(supportedOperation, docId);
+        VespaXMLFeedReader.Operation operation = new VespaXMLFeedReader.Operation();
+        try {
+            inputStream.close();
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+        if (supportedOperation == JsonReader.SupportedOperation.PUT) {
+            operation.setDocument(((DocumentPut) documentOperation).getDocument());
+        } else {
+            operation.setDocumentUpdate((DocumentUpdate) documentOperation);
+        }
+
+        // (A potentially empty) test-and-set condition is always set by JsonReader
+        operation.setCondition(documentOperation.getCondition());
+
+        return operation;
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/json/TokenBuffer.java b/document/src/main/java/com/yahoo/document/json/TokenBuffer.java
new file mode 100644
index 00000000000..8f3395b989e
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/json/TokenBuffer.java
@@ -0,0 +1,195 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.json;
+
+import java.io.IOException;
+import java.util.ArrayDeque;
+import java.util.Deque;
+import java.util.Iterator;
+
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonToken;
+import com.google.common.base.Preconditions;
+
+/**
+ * Helper class to enable lookahead in the token stream.
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+class TokenBuffer {
+    static final class Token {
+        final JsonToken token;
+        final String name;
+        final String text;
+
+        Token(JsonToken token, String name, String text) {
+            this.token = token;
+            this.name = name;
+            this.text = text;
+        }
+    }
+
+    private Deque<Token> buffer;
+    private int nesting = 0;
+
+    TokenBuffer() {
+        this(new ArrayDeque<>());
+    }
+
+    private TokenBuffer(Deque<Token> buffer) {
+        this.buffer = buffer;
+        if (buffer.size() > 0) {
+            updateNesting(buffer.peekFirst().token);
+        }
+    }
+
+    JsonToken next() {
+        buffer.removeFirst();
+        Token t = buffer.peekFirst();
+        if (t == null) {
+            return null;
+        }
+        updateNesting(t.token);
+        return t.token;
+    }
+
+    JsonToken currentToken() {
+        return buffer.peekFirst().token;
+    }
+
+    String currentName() {
+        return buffer.peekFirst().name;
+    }
+
+    String currentText() {
+        return buffer.peekFirst().text;
+    }
+
+    int size() {
+        return buffer.size();
+    }
+
+    private void add(JsonToken token, String name, String text) {
+        buffer.addLast(new Token(token, name, text));
+    }
+
+    void bufferObject(JsonToken first, JsonParser tokens) {
+        int localNesting = 0;
+        JsonToken t = first;
+
+        Preconditions.checkArgument(first == JsonToken.START_OBJECT,
+                "Expected START_OBJECT, got %s.", t);
+        if (size() == 0) {
+            updateNesting(t);
+        }
+        localNesting = storeAndPeekNesting(t, localNesting, tokens);
+        while (localNesting > 0) {
+            t = nextValue(tokens);
+            localNesting = storeAndPeekNesting(t, localNesting, tokens);
+        }
+    }
+
+    private int storeAndPeekNesting(JsonToken t, int nesting, JsonParser tokens) {
+        addFromParser(t, tokens);
+        return nesting + nestingOffset(t);
+    }
+
+    private int nestingOffset(JsonToken t) {
+        if (t.isStructStart()) {
+            return 1;
+        } else if (t.isStructEnd()) {
+            return -1;
+        } else {
+            return 0;
+        }
+    }
+
+    private void addFromParser(JsonToken t, JsonParser tokens) {
+        try {
+            add(t, tokens.getCurrentName(), tokens.getText());
+        } catch (IOException e) {
+            // TODO something sane
+            throw new RuntimeException(e);
+        }
+    }
+
+    private JsonToken nextValue(JsonParser tokens) {
+        try {
+            return tokens.nextValue();
+        } catch (IOException e) {
+            // TODO something sane
+            throw new RuntimeException(e);
+        }
+    }
+
+    private void updateNesting(JsonToken t) {
+        nesting += nestingOffset(t);
+    }
+
+    public int nesting() {
+        return nesting;
+    }
+
+    public String dumpContents() {
+        StringBuilder b = new StringBuilder();
+        b.append("[nesting: ").append(nesting()).append("\n");
+        for (Token t : buffer) {
+            b.append("(").append(t.token).append(", \"").append(t.name).append("\", \"").append(t.text).append("\")\n");
+        }
+        b.append("]\n");
+        return b.toString();
+    }
+
+    public void fastForwardToEndObject() {
+        JsonToken t = currentToken();
+        while (t != JsonToken.END_OBJECT) {
+            t = next();
+        }
+    }
+
+    TokenBuffer prefetchCurrentElement() {
+        Deque<Token> copy = new ArrayDeque<>();
+
+        if (currentToken().isScalarValue()) {
+            copy.add(buffer.peekFirst());
+        } else {
+            int localNesting = nesting();
+            int nestingBarrier = localNesting;
+            for (Token t : buffer) {
+                copy.add(t);
+                localNesting += nestingOffset(t.token);
+                if (localNesting < nestingBarrier) {
+                    break;
+                }
+            }
+        }
+        return new TokenBuffer(copy);
+    }
+
+    Token prefetchScalar(String name) {
+        int localNesting = nesting();
+        int nestingBarrier = localNesting;
+        Token toReturn = null;
+        Iterator<Token> i;
+
+        if (name.equals(currentName()) && currentToken().isScalarValue()) {
+            toReturn = buffer.peekFirst();
+        } else {
+            i = buffer.iterator();
+            i.next(); // just ignore the first value, as we know it's not what
+                      // we're looking for, and it's nesting effect is already
+                      // included
+            while (i.hasNext()) {
+                Token t = i.next();
+                if (localNesting == nestingBarrier && name.equals(t.name) && t.token.isScalarValue()) {
+                    toReturn = t;
+                    break;
+                }
+                localNesting += nestingOffset(t.token);
+                if (localNesting < nestingBarrier) {
+                    break;
+                }
+            }
+        }
+        return toReturn;
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/json/package-info.java b/document/src/main/java/com/yahoo/document/json/package-info.java
new file mode 100644
index 00000000000..85d939f5b18
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/json/package-info.java
@@ -0,0 +1,8 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * Infrastructure for building Vespa documents and feed operations from JSON.
+ */
+@ExportPackage
+package com.yahoo.document.json;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/document/src/main/java/com/yahoo/document/package-info.java b/document/src/main/java/com/yahoo/document/package-info.java
new file mode 100644
index 00000000000..e27bbadacb7
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/package-info.java
@@ -0,0 +1,7 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+@PublicApi
+package com.yahoo.document;
+
+import com.yahoo.api.annotations.PublicApi;
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/document/src/main/java/com/yahoo/document/select/BucketSelector.java b/document/src/main/java/com/yahoo/document/select/BucketSelector.java
new file mode 100644
index 00000000000..96b8e4b617f
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/select/BucketSelector.java
@@ -0,0 +1,65 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.select;
+
+import com.yahoo.document.BucketIdFactory;
+import com.yahoo.document.select.parser.ParseException;
+import com.yahoo.document.select.parser.SelectInput;
+import com.yahoo.document.select.parser.SelectParser;
+import com.yahoo.document.select.parser.TokenMgrError;
+import com.yahoo.document.select.simple.SelectionParser;
+
+/**
+ * This class is used to find out in which locations a document might be in, if
+ * it matches a given document selection string.
+ *
+ * @author <a href="mailto:humbe@yahoo-inc.com">H&aring;kon Humberset</a>
+ */
+public class BucketSelector {
+
+    // A local reference to the factory used by the current application.
+    private BucketIdFactory factory;
+
+    /**
+     * The bucket selector needs to be instantiated to be used, as it will
+     * depend on config.
+     *
+     * @param factory The bucket factory is needed to get information of how
+     *                bucket ids are put together.
+     */
+    public BucketSelector(BucketIdFactory factory) {
+        this.factory = factory;
+    }
+
+    /**
+     * Get the set of buckets that may contain documents that match the given
+     * document selection, as long as the document selection does not result in
+     * an unknown set of buckets. If it does, <code>null</code> will be
+     * returned. This requires the caller to be aware of the meaning of these
+     * return values, but also removes the need for redundant space utilization
+     * when dealing with unknown bucket sets.
+     *
+     * @param selector The document selection string
+     * @return A list of buckets with arbitrary number of location bits set,
+     *         <i>or</i>, <code>null</code> if the document selection resulted
+     *         in an unknown set
+     * @throws ParseException if <code>selector</code> couldn't be parsed
+     */
+    public BucketSet getBucketList(String selector) throws ParseException {
+         try {
+             SelectionParser simple = new SelectionParser();
+             if (simple.parse(selector) && (simple.getRemaining().length() == 0)) {
+                 return simple.getNode().getBucketSet(factory);
+             } else {
+                SelectParser parser = new SelectParser(new SelectInput(selector));
+                return parser.expression().getBucketSet(factory);
+             }
+        } catch (TokenMgrError e) {
+            ParseException t = new ParseException();
+            throw (ParseException) t.initCause(e);
+        } catch (RuntimeException e) {
+            ParseException t = new ParseException(
+                    "Unexpected exception while parsing '" + selector + "'.");
+            throw (ParseException) t.initCause(e);
+        }
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/select/BucketSet.java b/document/src/main/java/com/yahoo/document/select/BucketSet.java
new file mode 100644
index 00000000000..e7bb4ac7807
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/select/BucketSet.java
@@ -0,0 +1,72 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.select;
+
+import com.yahoo.document.BucketId;
+
+import java.util.HashSet;
+
+/**
+ * A set of bucket ids covered by a document selector.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class BucketSet extends HashSet<BucketId> {
+
+    /**
+     * Constructs a new bucket set that contains no ids.
+     */
+    public BucketSet() {
+        // empty
+    }
+
+    /**
+     * Constructs a new bucket set that contains a single id.
+     *
+     * @param id The id to add to this as initial value.
+     */
+    public BucketSet(BucketId id) {
+        add(id);
+    }
+
+    /**
+     * Constructs a new bucket set that is a copy of another.
+     *
+     * @param set The set to copy.
+     */
+    public BucketSet(BucketSet set) {
+        this.addAll(set);
+    }
+
+    /**
+     * Returns the intersection between this bucket set and another.
+     *
+     * @param rhs The set to form an intersection with.
+     * @return The intersection.
+     */
+    public BucketSet intersection(BucketSet rhs) {
+        if (rhs == null) {
+            return new BucketSet(this); // The other has all buckets marked, this is the smaller.
+        } else {
+            BucketSet ret = new BucketSet(this);
+            ret.retainAll(rhs);
+            return ret;
+        }
+    }
+
+    /**
+     * Returns the union between this bucket set and another.
+     *
+     * @param rhs The set to form a union with.
+     * @return The union.
+     */
+    public BucketSet union(BucketSet rhs) {
+        if (rhs == null) {
+            return null;
+        } else {
+            BucketSet ret = new BucketSet(this);
+            ret.addAll(rhs);
+            return ret;
+        }
+    }
+
+}
diff --git a/document/src/main/java/com/yahoo/document/select/Context.java b/document/src/main/java/com/yahoo/document/select/Context.java
new file mode 100644
index 00000000000..6cf5ab12bbf
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/select/Context.java
@@ -0,0 +1,34 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.select;
+
+import com.yahoo.document.DocumentOperation;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class Context {
+    private DocumentOperation documentOperation = null;
+
+    public Context(DocumentOperation documentOperation) {
+        this.documentOperation = documentOperation;
+    }
+
+    public DocumentOperation getDocumentOperation() {
+        return documentOperation;
+    }
+
+    public void setDocumentOperation(DocumentOperation documentOperation) {
+        this.documentOperation = documentOperation;
+    }
+
+    public Map<String, Object> getVariables() {
+        return variables;
+    }
+
+    public void setVariables(Map<String, Object> variables) {
+        this.variables = variables;
+    }
+
+    private Map<String, Object> variables = new HashMap<String, Object>();
+
+}
diff --git a/document/src/main/java/com/yahoo/document/select/DocumentSelector.java b/document/src/main/java/com/yahoo/document/select/DocumentSelector.java
new file mode 100644
index 00000000000..aa26efb0c2d
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/select/DocumentSelector.java
@@ -0,0 +1,118 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.select;
+
+import com.yahoo.document.DocumentOperation;
+import com.yahoo.document.select.parser.ParseException;
+import com.yahoo.document.select.parser.SelectInput;
+import com.yahoo.document.select.parser.SelectParser;
+import com.yahoo.document.select.parser.TokenMgrError;
+import com.yahoo.document.select.rule.ExpressionNode;
+
+/**
+ * <p>A document selector is a filter which accepts or rejects documents
+ * based on their type and content. A document selector has a textual
+ * representation which is called the <i>Document Selection Language</i></p>
+ *
+ * <p>Document selectors are multithread safe.</p>
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public class DocumentSelector {
+
+    private ExpressionNode expression;
+
+    /**
+     * Creates a document selector from a Document Selection Language string
+     *
+     * @param selector The string to parse as a selector.
+     * @throws ParseException Thrown if the string could not be parsed.
+     */
+    public DocumentSelector(String selector) throws ParseException {
+        SelectInput input = new SelectInput(selector);
+        try {
+            SelectParser parser = new SelectParser(input);
+            expression = parser.expression();
+        } catch (TokenMgrError e) {
+            ParseException t = new ParseException("Tokenization error parsing document selector '" + selector + "'");
+            throw (ParseException)t.initCause(e);
+        } catch (RuntimeException | ParseException e) {
+            ParseException t = new ParseException("Exception parsing document selector '" + selector + "'");
+            throw (ParseException)t.initCause(e instanceof ParseException ?
+                                              new ParseException(input.formatException(e.getMessage())) : e);
+        }
+    }
+
+    /**
+     * Returns true if the document referenced by this document operation is accepted by this selector
+     *
+     * @param op A document operation
+     * @return True if the document is accepted.
+     * @throws RuntimeException if the evaluation enters an illegal state
+     */
+    public Result accepts(DocumentOperation op) {
+        return accepts(new Context(op));
+    }
+
+    /**
+     * Returns true if the document referenced by this context is accepted by this selector
+     *
+     * @param context The context to match in.
+     * @return True if the document is accepted.
+     * @throws RuntimeException if the evaluation enters an illegal state
+     */
+    public Result accepts(Context context) {
+        return Result.toResult(expression.evaluate(context));
+    }
+
+    /**
+     * Returns the list of different variables resulting in a true state for this
+     * expression.
+     *
+     * @param op The document to evaluate.
+     * @return True if the document is accepted.
+     * @throws RuntimeException if the evaluation enters an illegal state
+     */
+    public ResultList getMatchingResultList(DocumentOperation op) {
+        return getMatchingResultList(new Context(op));
+    }
+
+    /**
+     * Returns the list of different variables resulting in a true state for this
+     * expression.
+     *
+     * @param context The context to match in.
+     * @return True if the document is accepted.
+     * @throws RuntimeException if the evaluation enters an illegal state
+     */
+    public ResultList getMatchingResultList(Context context) {
+        return ResultList.toResultList(expression.evaluate(context));
+    }
+
+    /**
+     * Returns this selector as a Document Selection Language string.
+     *
+     * @return The selection string.
+     */
+    public String toString() {
+        return expression.toString();
+    }
+
+    /**
+     *  Returns the ordering specification, if any, implied by this document
+     *  selection expression.
+     *
+     *  @param order The order of the
+     */
+    public OrderingSpecification getOrdering(int order) {
+        return expression.getOrdering(order);
+    }
+
+    /**
+     * Visits the expression tree.
+     *
+     * @param visitor The visitor to use.
+     */
+    public void visit(Visitor visitor) {
+        expression.accept(visitor);
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/select/NowCheckVisitor.java b/document/src/main/java/com/yahoo/document/select/NowCheckVisitor.java
new file mode 100644
index 00000000000..50613f531b2
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/select/NowCheckVisitor.java
@@ -0,0 +1,67 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.select;
+
+import com.yahoo.document.select.Visitor;
+import com.yahoo.document.select.rule.*;
+
+/**
+ * Traverse and check if there exists any now() function in the expression tree.
+ *
+ * @author <a href="mailto:lulf@yahoo-inc.com">Ulf Lilleengen</a>
+ */
+
+public class NowCheckVisitor implements Visitor {
+    private int nowNodeCount = 0;
+
+    public boolean requiresConversion() {
+        return (nowNodeCount > 0);
+    }
+
+    public void visit(ArithmeticNode node) {
+        for (ArithmeticNode.NodeItem item : node.getItems()) {
+            item.getNode().accept(this);
+        }
+    }
+
+    public void visit(AttributeNode node) {
+        node.getValue().accept(this);
+    }
+
+    public void visit(ComparisonNode node) {
+        node.getLHS().accept(this);
+        node.getRHS().accept(this);
+    }
+
+    public void visit(DocumentNode node) {
+    }
+
+    public void visit(EmbracedNode node) {
+        node.getNode().accept(this);
+    }
+
+    public void visit(IdNode node) {
+    }
+
+    public void visit(LiteralNode node) {
+    }
+
+    public void visit(LogicNode node) {
+        for (LogicNode.NodeItem item : node.getItems()) {
+            item.getNode().accept(this);
+        }
+    }
+
+    public void visit(NegationNode node) {
+        node.getNode().accept(this);
+    }
+
+    public void visit(NowNode node) {
+        nowNodeCount++;
+    }
+
+    public void visit(SearchColumnNode node) {
+    }
+
+    public void visit(VariableNode node) {
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/select/OrderingSpecification.java b/document/src/main/java/com/yahoo/document/select/OrderingSpecification.java
new file mode 100644
index 00000000000..3f5a7a58733
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/select/OrderingSpecification.java
@@ -0,0 +1,43 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.select;
+
+public class OrderingSpecification {
+    public static int ASCENDING = 0;
+    public static int DESCENDING = 1;
+
+    public int order;
+    public long orderingStart;
+    public short widthBits;
+    public short divisionBits;
+
+    public OrderingSpecification() {
+        this(ASCENDING, (long)0, (short)0, (short)0);
+    }
+
+    public OrderingSpecification(int order) {
+        this(order, (long)0, (short)0, (short)0);
+    }
+
+    public OrderingSpecification(int order, long orderingStart, short widthBits, short divisionBits) {
+        this.order = order;
+        this.orderingStart = orderingStart;
+        this.widthBits = widthBits;
+        this.divisionBits = divisionBits;
+    }
+
+    public int getOrder() { return order; }
+    public long getOrderingStart() { return orderingStart; }
+    public short getWidthBits() { return widthBits; }
+    public short getDivisionBits() { return divisionBits; }
+
+    public boolean equals(Object other) {
+        OrderingSpecification o = (OrderingSpecification)other;
+        if (o == null) return false;
+
+        return (order == o.order && orderingStart == o.orderingStart && widthBits == o.widthBits && divisionBits == o.divisionBits);
+    }
+
+    public String toString() {
+        return "O: " + order + " S:" + orderingStart + " W:" + widthBits + " D:" + divisionBits;
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/select/Result.java b/document/src/main/java/com/yahoo/document/select/Result.java
new file mode 100644
index 00000000000..3f1fa75d4ef
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/select/Result.java
@@ -0,0 +1,53 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.select;
+
+import com.yahoo.document.select.rule.AttributeNode;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public enum Result {
+
+    /**
+     * Defines all enumeration constants.
+     */
+    TRUE,
+    FALSE,
+    INVALID;
+
+    // Inherit doc from Object.
+    public String toString() {
+        return name().toLowerCase();
+    }
+    
+    /**
+     * Inverts the result value to the appropriate value. True → False, False → True and Invalid → Invalid.
+     * @return inverted result
+     */
+    public static Result invert(Result result) {
+    	if (result == Result.TRUE) return Result.FALSE;
+    	if (result == Result.FALSE) return Result.TRUE;
+    	return Result.INVALID;
+    }
+
+    /**
+     * Converts the given object value into an instance of this Result enumeration.
+     *
+     * @param value The object to convert.
+     * @return The corresponding result value.
+     */
+    public static Result toResult(Object value) {
+        if (value == null || value == Result.FALSE || value == Boolean.FALSE ||
+            (Number.class.isInstance(value) && ((Number)value).doubleValue() == 0)) {
+            return Result.FALSE;
+        } else if (value == INVALID) {
+            return Result.INVALID;
+        } else if (value instanceof AttributeNode.VariableValueList) {
+            return ((AttributeNode.VariableValueList)value).isEmpty() ? Result.FALSE : Result.TRUE;
+        } else if (value instanceof ResultList) {
+            return ((ResultList)value).toResult();
+        } else {
+            return Result.TRUE;
+        }
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/select/ResultList.java b/document/src/main/java/com/yahoo/document/select/ResultList.java
new file mode 100644
index 00000000000..e3fc7cadce7
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/select/ResultList.java
@@ -0,0 +1,199 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.select;
+
+import com.yahoo.document.datatypes.FieldPathIteratorHandler;
+import com.yahoo.document.select.rule.AttributeNode;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public class ResultList {
+    public static class ResultPair {
+        ResultPair(FieldPathIteratorHandler.VariableMap var, Result res) {
+            variables = var;
+            result = res;
+        }
+
+        FieldPathIteratorHandler.VariableMap variables;
+        Result result;
+
+        public FieldPathIteratorHandler.VariableMap getVariables() { return variables; }
+        public Result getResult() { return result; }
+
+        public String toString() {
+            return variables.toString() + " => " + result;
+        }
+    }
+
+    public static class VariableValue {
+       public VariableValue(FieldPathIteratorHandler.VariableMap vars, Object value) {
+           variables = vars;
+           this.value = value;
+       }
+
+        FieldPathIteratorHandler.VariableMap variables;
+        Object value;
+
+        public FieldPathIteratorHandler.VariableMap getVariables() { return variables; }
+        public Object getValue() { return value; }
+
+        public String toString() {
+            return variables.toString() + " => " + value;
+        }
+    }
+
+    List<ResultPair> results = new ArrayList<ResultPair>();
+
+    public ResultList() {
+    }
+
+    public ResultList(Result result) {
+        add(new FieldPathIteratorHandler.VariableMap(), result);
+    }
+
+    public void add(FieldPathIteratorHandler.VariableMap variables, Result result) {
+        results.add(new ResultPair(variables, result));
+    }
+
+    public List<ResultPair> getResults() {
+        return results;
+    }
+
+    public Result toResult() {
+        if (results.isEmpty()) {
+            return Result.FALSE;
+        }
+
+        boolean foundFalse = false;
+
+        for (ResultPair rp : results) {
+            if (rp.result == Result.TRUE) {
+                return Result.TRUE;
+            } else if (rp.result == Result.FALSE) {
+                foundFalse = true;
+            }
+        }
+
+        if (foundFalse) {
+            return Result.FALSE;
+        } else {
+            return Result.INVALID;
+        }
+    }
+
+    boolean combineVariables(FieldPathIteratorHandler.VariableMap output, FieldPathIteratorHandler.VariableMap input) {
+        // First, verify that all variables are overlapping
+        for (Map.Entry<String, FieldPathIteratorHandler.IndexValue> entry : output.entrySet()) {
+            FieldPathIteratorHandler.IndexValue found = input.get(entry.getKey());
+
+            if (found != null) {
+                if (!(found.equals(entry.getValue()))) {
+                    return false;
+                }
+            }
+        }
+
+        for (Map.Entry<String, FieldPathIteratorHandler.IndexValue> entry : input.entrySet()) {
+            FieldPathIteratorHandler.IndexValue found = output.get(entry.getKey());
+
+            if (found != null) {
+                if (!(found.equals(entry.getValue()))) {
+                    return false;
+                }
+            }
+        }
+
+        // Ok, variables are overlapping. Add all variables from input to output.
+        for (Map.Entry<String, FieldPathIteratorHandler.IndexValue> entry : input.entrySet()) {
+            output.put(entry.getKey(), entry.getValue());
+        }
+
+        return true;
+    }
+
+    public ResultList combineAND(ResultList other)
+    {
+        ResultList result = new ResultList();
+
+        // TODO: optimize
+        for (ResultPair pair : results) {
+            for (ResultPair otherPair : other.results) {
+                FieldPathIteratorHandler.VariableMap varMap = (FieldPathIteratorHandler.VariableMap)pair.variables.clone();
+
+                if (combineVariables(varMap, otherPair.variables)) {
+                    result.add(varMap, combineAND(pair.result, otherPair.result));
+                }
+            }
+        }
+
+        return result;
+    }
+
+    private static Result combineAND(Result lhs, Result rhs) {
+        if (lhs == Result.TRUE && rhs == Result.TRUE) {
+            return Result.TRUE;
+        }
+        if (lhs == Result.FALSE || rhs == Result.FALSE) {
+            return Result.FALSE;
+        }
+        return Result.INVALID;
+    }
+
+    public ResultList combineOR(ResultList other)
+    {
+        ResultList result = new ResultList();
+
+        // TODO: optimize
+        for (ResultPair pair : results) {
+            for (ResultPair otherPair : other.results) {
+                FieldPathIteratorHandler.VariableMap varMap = (FieldPathIteratorHandler.VariableMap)pair.variables.clone();
+
+                if (combineVariables(varMap, otherPair.variables)) {
+                    result.add(varMap, combineOR(pair.result, otherPair.result));
+                }
+            }
+        }
+
+        return result;
+    }
+
+    private static Result combineOR(Result lhs, Result rhs) {
+        if (lhs == Result.TRUE || rhs == Result.TRUE) {
+            return Result.TRUE;
+        }
+        if (lhs == Result.FALSE && rhs == Result.FALSE) {
+            return Result.FALSE;
+        }
+        return Result.INVALID;
+    }
+
+    /**
+     * Converts the given object value into a result list, so it can be compared by logical operators.
+     *
+     * @param value The object to convert.
+     * @return The corresponding result value.
+     */
+    public static ResultList toResultList(Object value) {
+        if (value instanceof ResultList) {
+            return (ResultList)value;
+        } else if (value instanceof AttributeNode.VariableValueList) {
+            ResultList retVal = new ResultList();
+            for (VariableValue vv : (AttributeNode.VariableValueList)value) {
+                retVal.add(vv.getVariables(), Result.TRUE);
+            }
+            return retVal;
+        } else if (value == null || value == Result.FALSE || value == Boolean.FALSE ||
+            (Number.class.isInstance(value) && ((Number)value).doubleValue() == 0)) {
+            return new ResultList(Result.FALSE);
+        } else if (value == Result.INVALID) {
+            return new ResultList(Result.INVALID);
+        } else {
+            return new ResultList(Result.TRUE);
+        }
+    }
+
+    public String toString() {
+        return results.toString();
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/select/Visitor.java b/document/src/main/java/com/yahoo/document/select/Visitor.java
new file mode 100644
index 00000000000..a28be4eadaf
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/select/Visitor.java
@@ -0,0 +1,25 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.select;
+
+import com.yahoo.document.select.rule.*;
+
+/**
+ * This interface can be used to create custom visitors for the selection tree.
+ *
+ * @author <a href="mailto:lulf@yahoo-inc.com">Ulf Lilleengen</a>
+ */
+
+public interface Visitor {
+    public void visit(ArithmeticNode node);
+    public void visit(AttributeNode node);
+    public void visit(ComparisonNode node);
+    public void visit(DocumentNode node);
+    public void visit(EmbracedNode node);
+    public void visit(IdNode node);
+    public void visit(LiteralNode node);
+    public void visit(LogicNode node);
+    public void visit(NegationNode node);
+    public void visit(NowNode node);
+    public void visit(SearchColumnNode node);
+    public void visit(VariableNode node);
+}
diff --git a/document/src/main/java/com/yahoo/document/select/convert/NowQueryExpression.java b/document/src/main/java/com/yahoo/document/select/convert/NowQueryExpression.java
new file mode 100644
index 00000000000..7041f7b760b
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/select/convert/NowQueryExpression.java
@@ -0,0 +1,39 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.select.convert;
+
+import com.yahoo.document.select.rule.ArithmeticNode;
+import com.yahoo.document.select.rule.AttributeNode;
+import com.yahoo.document.select.rule.ComparisonNode;
+
+/**
+ * Represents a query containing a valid now() expression. The now expression
+ * is very strict right now, but can be expanded later.
+ *
+ * @author <a href="mailto:lulf@yahoo-inc.com">Ulf Lilleengen</a>
+ */
+public class NowQueryExpression {
+    private final AttributeNode attribute;
+    private final ComparisonNode comparison;
+    private final NowQueryNode now;
+
+    public NowQueryExpression(AttributeNode attribute, ComparisonNode comparison, ArithmeticNode arithmetic) {
+        this.attribute = attribute;
+        this.comparison = comparison;
+        this.now = (arithmetic != null ? new NowQueryNode(arithmetic) : new NowQueryNode(0));
+    }
+
+    public String getDocumentType() {
+        return attribute.getValue().toString();
+    }
+
+    @Override
+    public String toString() {
+        StringBuilder sb = new StringBuilder();
+        for(AttributeNode.Item item : attribute.getItems()) {
+            sb.append(item.getName()).append(".");
+        }
+        sb.deleteCharAt(sb.length() - 1);
+        return sb.toString() + ":" + comparison.getOperator() + now;
+    }
+
+}
diff --git a/document/src/main/java/com/yahoo/document/select/convert/NowQueryNode.java b/document/src/main/java/com/yahoo/document/select/convert/NowQueryNode.java
new file mode 100644
index 00000000000..58b57cc2983
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/select/convert/NowQueryNode.java
@@ -0,0 +1,24 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.select.convert;
+
+import com.yahoo.document.select.rule.ArithmeticNode;
+
+/**
+ * Represents the now node in a query expression.
+ *
+ * @author <a href="mailto:lulf@yahoo-inc.com">Ulf Lilleengen</a>
+ */
+public class NowQueryNode {
+    private final long value;
+    public NowQueryNode(long value) {
+        this.value = value;
+    }
+    public NowQueryNode(ArithmeticNode node) {
+    	// Assumes that the structure is checked and verified earlier
+        this.value = Long.parseLong(node.getItems().get(1).getNode().toString());
+    }
+    @Override
+    public String toString() {
+        return "now(" + this.value + ")";
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/select/convert/SelectionExpressionConverter.java b/document/src/main/java/com/yahoo/document/select/convert/SelectionExpressionConverter.java
new file mode 100644
index 00000000000..eec14ed7cee
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/select/convert/SelectionExpressionConverter.java
@@ -0,0 +1,152 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.select.convert;
+
+import com.yahoo.document.select.NowCheckVisitor;
+import com.yahoo.document.select.Visitor;
+import com.yahoo.document.select.rule.*;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Class which converts a selection tree into a set of queries per document type.
+ * If unsupported operations are or illegal arguments are encountered, an exception is thrown.
+ *
+ * @author <a href="mailto:lulf@yahoo-inc.com">Ulf Lilleengen</a>
+ */
+public class SelectionExpressionConverter implements Visitor {
+
+    private Map<String, NowQueryExpression> expressionMap = new HashMap<String, NowQueryExpression>();
+
+    private class BuildState {
+        public AttributeNode attribute;
+        public ComparisonNode comparison;
+        public ArithmeticNode arithmetic;
+        public NowNode now;
+        public boolean hasNow() { return now != null; }
+    }
+
+    private BuildState state;
+
+    private boolean hasNow(ExpressionNode node) {
+        NowCheckVisitor visitor = new NowCheckVisitor();
+        node.accept(visitor);
+        return visitor.requiresConversion();
+    }
+
+    public SelectionExpressionConverter() {
+        this.state = null;
+    }
+
+    public Map<String, String> getQueryMap() {
+        Map<String, String> ret = new HashMap<String, String>();
+        for (NowQueryExpression expression : expressionMap.values()) {
+            ret.put(expression.getDocumentType(), expression.toString());
+        }
+        return ret;
+    }
+
+
+    public void visit(ArithmeticNode node) {
+        if (state == null ) return;
+        if (node.getItems().size() > 2) {
+            throw new IllegalArgumentException("Too many arithmetic operations");
+        }
+        for (ArithmeticNode.NodeItem item : node.getItems()) {
+            if (item.getOperator() != ArithmeticNode.SUB && item.getOperator() != ArithmeticNode.NOP) {
+                throw new IllegalArgumentException("Arithmetic operator '" + node.operatorToString(item.getOperator()) + "' is not supported");
+            }
+        }
+        state.arithmetic = node;
+
+    }
+
+    public void visit(AttributeNode node) {
+        if (state == null ) return;
+        if (expressionMap.containsKey(node.getValue().toString())) {
+            throw new IllegalArgumentException("Specifying multiple document types is not allowed");
+        }
+        for (AttributeNode.Item item : node.getItems()) {
+            if (item.getType() != AttributeNode.Item.ATTRIBUTE) {
+                throw new IllegalArgumentException("Only attribute items are supported");
+            }
+        }
+        state.attribute = node;
+    }
+
+    public void visit(ComparisonNode node) {
+        if (state != null) {
+            throw new IllegalArgumentException("Comparison cannot be done within now expression");
+        }
+        if (!hasNow(node)) {
+            return;
+        }
+        state = new BuildState();
+        node.getLHS().accept(this);
+        node.getRHS().accept(this);
+
+        if (!">".equals(node.getOperator())) {
+            throw new IllegalArgumentException("Comparison operator '" + node.getOperator() + "' is not supported");
+        }
+        if (!(node.getLHS() instanceof AttributeNode)) {
+            throw new IllegalArgumentException("Left hand side of comparison must be a document field");
+        }
+        state.comparison = node;
+        if (state.attribute != null &&
+            state.comparison != null &&
+            (state.arithmetic != null || state.now != null)) {
+            NowQueryExpression expression = new NowQueryExpression(state.attribute, state.comparison, state.arithmetic);
+            expressionMap.put(expression.getDocumentType(), expression);
+            state = null;
+        }
+    }
+
+    public void visit(DocumentNode node) {
+        // Silently ignore
+    }
+
+    public void visit(EmbracedNode node) {
+        if (state == null ) return;
+        throw new UnsupportedOperationException("Grouping is not supported yet.");
+    }
+
+    public void visit(IdNode node) {
+        if (state == null ) return;
+        throw new UnsupportedOperationException("Document id not supported yet.");
+    }
+
+    public void visit(LiteralNode node) {
+        if (state == null ) return;
+        if (!(node.getValue() instanceof Long)) {
+            throw new IllegalArgumentException("Literal " + node + " is not supported");
+        }
+    }
+
+    public void visit(LogicNode node) {
+        if (state != null) {
+            throw new IllegalArgumentException("Logic expressions not supported in now expressions");
+        }
+        for (LogicNode.NodeItem item : node.getItems()) {
+            item.getNode().accept(this);
+        }
+    }
+
+    public void visit(NegationNode node) {
+        if (state == null ) return;
+        throw new UnsupportedOperationException("Negation not supported yet.");
+    }
+
+    public void visit(NowNode node) {
+        if (state == null ) return;
+        state.now = node;
+    }
+
+    public void visit(SearchColumnNode node) {
+        if (state == null ) return;
+        throw new UnsupportedOperationException("Searchcolumn not supported yet.");
+    }
+
+    public void visit(VariableNode node) {
+        if (state == null ) return;
+        throw new UnsupportedOperationException("Variables not supported yet.");
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/select/convert/package-info.java b/document/src/main/java/com/yahoo/document/select/convert/package-info.java
new file mode 100644
index 00000000000..08e965f317b
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/select/convert/package-info.java
@@ -0,0 +1,7 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+@PublicApi
+package com.yahoo.document.select.convert;
+
+import com.yahoo.api.annotations.PublicApi;
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/document/src/main/java/com/yahoo/document/select/package-info.java b/document/src/main/java/com/yahoo/document/select/package-info.java
new file mode 100644
index 00000000000..bbc33bcedf8
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/select/package-info.java
@@ -0,0 +1,7 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+@PublicApi
+package com.yahoo.document.select;
+
+import com.yahoo.api.annotations.PublicApi;
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/document/src/main/java/com/yahoo/document/select/parser/SelectInput.java b/document/src/main/java/com/yahoo/document/select/parser/SelectInput.java
new file mode 100644
index 00000000000..957a038d44a
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/select/parser/SelectInput.java
@@ -0,0 +1,14 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.select.parser;
+
+import com.yahoo.javacc.FastCharStream;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a>
+ */
+public class SelectInput extends FastCharStream implements CharStream {
+
+    public SelectInput(String input) {
+        super(input);
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/select/parser/SelectParserUtils.java b/document/src/main/java/com/yahoo/document/select/parser/SelectParserUtils.java
new file mode 100644
index 00000000000..c8458bcb5aa
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/select/parser/SelectParserUtils.java
@@ -0,0 +1,27 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.select.parser;
+
+import com.yahoo.javacc.UnicodeUtilities;
+
+import java.math.BigInteger;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a>
+ */
+public class SelectParserUtils {
+
+    public static long decodeLong(String str) {
+        if (str.startsWith("0x") || str.startsWith("0X")) {
+            str = Long.toString(new BigInteger(str.substring(2), 16).longValue());
+        }
+        return Long.decode(str);
+    }
+
+    public static String quote(String str, char quote) {
+        return UnicodeUtilities.quote(str, quote);
+    }
+
+    public static String unquote(String str) {
+        return UnicodeUtilities.unquote(str);
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/select/parser/package-info.java b/document/src/main/java/com/yahoo/document/select/parser/package-info.java
new file mode 100644
index 00000000000..7f058e86e28
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/select/parser/package-info.java
@@ -0,0 +1,7 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+@PublicApi
+package com.yahoo.document.select.parser;
+
+import com.yahoo.api.annotations.PublicApi;
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/document/src/main/java/com/yahoo/document/select/rule/ArithmeticNode.java b/document/src/main/java/com/yahoo/document/select/rule/ArithmeticNode.java
new file mode 100644
index 00000000000..2fe4609b4e6
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/select/rule/ArithmeticNode.java
@@ -0,0 +1,209 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.select.rule;
+
+import com.yahoo.document.BucketIdFactory;
+import com.yahoo.document.datatypes.NumericFieldValue;
+import com.yahoo.document.select.*;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Stack;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class ArithmeticNode implements ExpressionNode {
+
+    public static final int NOP = 0;
+    public static final int ADD = 1;
+    public static final int SUB = 2;
+    public static final int MOD = 3;
+    public static final int DIV = 4;
+    public static final int MUL = 5;
+
+    private final List<NodeItem> items = new ArrayList<NodeItem>();
+
+    public ArithmeticNode() {
+        // empty
+    }
+
+    public ArithmeticNode add(String operator, ExpressionNode node) {
+        items.add(new NodeItem(stringToOperator(operator), node));
+        return this;
+    }
+
+    public List<NodeItem> getItems() {
+        return items;
+    }
+
+    // Inherit doc from ExpressionNode.
+    public BucketSet getBucketSet(BucketIdFactory factory) {
+        return null;
+    }
+
+    // Inherit doc from ExpressionNode.
+    public Object evaluate(Context context) {
+        StringBuilder ret = null;        
+        Stack<ValueItem> buf = new Stack<ValueItem>();
+        for (int i = 0; i < items.size(); ++i) {
+            NodeItem item = items.get(i);
+            Object val = item.node.evaluate(context);
+
+            if (val == null) {
+                throw new IllegalStateException("Null value found!");
+            }
+
+            if (val instanceof AttributeNode.VariableValueList) {
+                AttributeNode.VariableValueList value = (AttributeNode.VariableValueList)val;
+                if (value.size() == 0) {
+                    throw new IllegalArgumentException("Can not perform arithmetic on missing field: "
+                            + item.node.toString());
+                } else if (value.size() != 1) {
+                    throw new IllegalStateException("Arithmetic is only valid for single values.");
+                } else {
+                    val = value.get(0).getValue();
+                }
+            }
+
+            if (val instanceof NumericFieldValue) {
+                val = ((NumericFieldValue)val).getNumber();  
+            }
+
+            if (val instanceof String) {
+                if (i == 0) {
+                    ret = new StringBuilder();
+                }
+                if (ret != null) {
+                    ret.append(val);
+                    continue;
+                }
+            } else if (Number.class.isInstance(val)) {
+                if (!buf.isEmpty()) {
+                    while (buf.peek().operator > item.operator) {
+                        popOffTheTop(buf);
+                    }
+                }
+                buf.push(new ValueItem(item.operator, (Number)val));
+                continue;
+            }
+            throw new IllegalStateException("Term '" + item.node + " with class " + val.getClass() + "' does not evaluate to a number.");
+        }
+        if (ret != null) {
+            return ret.toString();
+        }
+        while (buf.size() > 1) {
+            popOffTheTop(buf);
+        }
+        return buf.pop().value;
+    }
+
+    private void popOffTheTop(Stack<ValueItem> buf) {
+        ValueItem rhs = buf.pop();
+        ValueItem lhs = buf.pop();
+        switch (rhs.operator) {
+            case ADD:
+                lhs.value = lhs.value.doubleValue() + rhs.value.doubleValue();
+                break;
+            case SUB:
+                lhs.value = lhs.value.doubleValue() - rhs.value.doubleValue();
+                break;
+            case DIV:
+                lhs.value = lhs.value.doubleValue() / rhs.value.doubleValue();
+                break;
+            case MUL:
+                lhs.value = lhs.value.doubleValue() * rhs.value.doubleValue();
+                break;
+            case MOD:
+                lhs.value = lhs.value.longValue() % rhs.value.longValue();
+                break;
+            default:
+                throw new IllegalStateException("Arithmetic operator " + rhs.operator + " not supported.");
+        }
+        buf.push(lhs);
+    }
+
+    @Override
+    public String toString() {
+        StringBuilder ret = new StringBuilder();
+        for (NodeItem item : items) {
+            if (item.operator != NOP) {
+                ret.append(" ").append(operatorToString(item.operator)).append(" ");
+            }
+            ret.append(item.node);
+        }
+        return ret.toString();
+    }
+
+    public String operatorToString(int operator) {
+        switch (operator) {
+            case NOP:
+                return null;
+            case ADD:
+                return "+";
+            case SUB:
+                return "-";
+            case MOD:
+                return "%";
+            case DIV:
+                return "/";
+            case MUL:
+                return "*";
+            default:
+                throw new IllegalStateException("Arithmetic operator " + operator + " not supported.");
+        }
+    }
+
+    private int stringToOperator(String operator) {
+        if (operator == null) {
+            return NOP;
+        } else if (operator.equals("+")) {
+            return ADD;
+        } else if (operator.equals("-")) {
+            return SUB;
+        } else if (operator.equals("%")) {
+            return MOD;
+        } else if (operator.equals("/")) {
+            return DIV;
+        } else if (operator.equals("*")) {
+            return MUL;
+        } else {
+            throw new IllegalStateException("Arithmetic operator '" + operator + "' not supported.");
+        }
+    }
+
+    public OrderingSpecification getOrdering(int order) {
+        return null;
+    }
+
+    public void accept(Visitor visitor) {
+        visitor.visit(this);
+    }
+
+    private class ValueItem {
+        public int operator;
+        public Number value;
+
+        public ValueItem(int operator, Number value) {
+            this.operator = operator;
+            this.value = value;
+        }
+    }
+
+    public static class NodeItem {
+        private int operator;
+        private ExpressionNode node;
+
+        public NodeItem(int operator, ExpressionNode node) {
+            this.operator = operator;
+            this.node = node;
+        }
+        
+        public int getOperator() {
+            return operator;
+        }
+
+        public ExpressionNode getNode() {
+            return node;
+        }
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/select/rule/AttributeNode.java b/document/src/main/java/com/yahoo/document/select/rule/AttributeNode.java
new file mode 100644
index 00000000000..048eb70ac94
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/select/rule/AttributeNode.java
@@ -0,0 +1,205 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.select.rule;
+
+import com.yahoo.collections.BobHash;
+import com.yahoo.document.*;
+import com.yahoo.document.datatypes.FieldPathIteratorHandler;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.select.*;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class AttributeNode implements ExpressionNode {
+
+    private ExpressionNode value;
+    private final List<Item> items = new ArrayList<Item>();
+
+    public AttributeNode(ExpressionNode value, List items) {
+        this.value = value;
+        for (Object obj : items) {
+            if (obj instanceof Item) {
+                this.items.add((Item)obj);
+            } else {
+                throw new IllegalStateException("Can not add an instance of " + obj.getClass().getName() +
+                                                " as a function item.");
+            }
+        }
+    }
+
+    public ExpressionNode getValue() {
+        return value;
+    }
+
+    public AttributeNode setValue(ExpressionNode value) {
+        this.value = value;
+        return this;
+    }
+
+    public List<Item> getItems() {
+        return items;
+    }
+
+    // Inherit doc from ExpressionNode.
+    public BucketSet getBucketSet(BucketIdFactory factory) {
+        return null;
+    }
+
+    // Inherit doc from ExpressionNode.
+    public Object evaluate(Context context) {
+        String pos = value.toString();
+        Object obj = value.evaluate(context);
+
+        StringBuilder builder = new StringBuilder();
+        for (Item item : items) {
+            if (obj == null) {
+                throw new IllegalStateException("Can not invoke '" + item + "' on '" + pos + "' because that term " +
+                                                "evaluated to null.");
+            }
+            if (item.getType() != Item.FUNCTION) {
+                if (builder.length() > 0) {
+                    builder.append(".");
+                }
+
+                builder.append(item.getName());
+            } else {
+                if (builder.length() > 0) {
+                    obj = evaluateFieldPath(builder.toString(), obj);
+                    builder = new StringBuilder();
+                }
+
+                obj = evaluateFunction(item.getName(), obj);
+            }
+
+            pos = pos + "." + item;
+        }
+
+        if (builder.length() > 0) {
+            obj = evaluateFieldPath(builder.toString(), obj);
+        }
+        return obj;
+    }
+
+    public static class VariableValueList extends ArrayList<ResultList.VariableValue> {
+
+    }
+
+    static class IteratorHandler extends FieldPathIteratorHandler {
+        VariableValueList values = new VariableValueList();
+
+        @Override
+        public void onPrimitive(FieldValue fv) {
+            values.add(new ResultList.VariableValue((VariableMap)getVariables().clone(), fv));
+        }
+    }
+
+    private static Object applyFunction(String function, Object value) {
+        if (function.equalsIgnoreCase("abs")) {
+            if (Number.class.isInstance(value)) {
+                Number nValue = (Number)value;
+                if (value instanceof Double) {
+                    return nValue.doubleValue() * (nValue.doubleValue() < 0 ? -1 : 1);
+                } else if (value instanceof Float) {
+                    return nValue.floatValue() * (nValue.floatValue() < 0 ? -1 : 1);
+                } else if (value instanceof Long) {
+                    return nValue.longValue() * (nValue.longValue() < 0 ? -1 : 1);
+                } else if (value instanceof Integer) {
+                    return nValue.intValue() * (nValue.intValue() < 0 ? -1 : 1);
+                }
+            }
+            throw new IllegalStateException("Function 'abs' is only available for numerical values.");
+        } else if (function.equalsIgnoreCase("hash")) {
+            return BobHash.hash(value.toString());
+        } else if (function.equalsIgnoreCase("lowercase")) {
+            return value.toString().toLowerCase();
+        } else if (function.equalsIgnoreCase("uppercase")) {
+            return value.toString().toUpperCase();
+        }
+        throw new IllegalStateException("Function '" + function + "' is not supported.");
+    }
+
+    private static Object evaluateFieldPath(String fieldPth, Object value) {
+        if (value instanceof DocumentPut) {
+            final Document doc = ((DocumentPut) value).getDocument();
+            FieldPath fieldPath = doc.getDataType().buildFieldPath(fieldPth);
+            IteratorHandler handler = new IteratorHandler();
+            doc.iterateNested(fieldPath, 0, handler);
+            return handler.values;
+        } else if (value instanceof DocumentUpdate) {
+            return Result.INVALID;
+        }
+        return Result.FALSE;
+        //throw new IllegalStateException("Attributes are only available for document types for value '" + value + "'. Looking for " + fieldPth);
+    }
+
+    private static Object evaluateFunction(String function, Object value) {
+        if (value instanceof VariableValueList) {
+            VariableValueList retVal = new VariableValueList();
+
+            for (ResultList.VariableValue val : ((VariableValueList)value)) {
+                retVal.add(new ResultList.VariableValue(
+                        (FieldPathIteratorHandler.VariableMap)val.getVariables().clone(),
+                        applyFunction(function, val.getValue())));
+            }
+
+            return retVal;
+        }
+
+        return applyFunction(function, value);
+    }
+
+    public void accept(Visitor visitor) {
+        visitor.visit(this);
+    }
+
+    @Override
+    public String toString() {
+        StringBuilder ret = new StringBuilder();
+        ret.append(value);
+        for (Item item : items) {
+            ret.append(".").append(item);
+        }
+        return ret.toString();
+    }
+
+    public OrderingSpecification getOrdering(int order) {
+        return null;
+    }
+
+    public static class Item {
+        public static final int ATTRIBUTE = 0;
+        public static final int FUNCTION = 1;
+
+        private String name;
+        private int type = ATTRIBUTE;
+
+        public Item(String name) {
+            this.name = name;
+        }
+
+        public String getName() {
+            return name;
+        }
+
+        public Item setName(String name) {
+            this.name = name;
+            return this;
+        }
+
+        public int getType() {
+            return type;
+        }
+
+        public Item setType(int type) {
+            this.type = type;
+            return this;
+        }
+
+        @Override public String toString() {
+            return name + (type == FUNCTION ? "()" : "");
+        }
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/select/rule/ComparisonNode.java b/document/src/main/java/com/yahoo/document/select/rule/ComparisonNode.java
new file mode 100644
index 00000000000..b0d5030978e
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/select/rule/ComparisonNode.java
@@ -0,0 +1,435 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.select.rule;
+
+import com.yahoo.document.BucketId;
+import com.yahoo.document.BucketIdFactory;
+import com.yahoo.document.DocumentId;
+import com.yahoo.document.datatypes.FieldPathIteratorHandler;
+import com.yahoo.document.datatypes.NumericFieldValue;
+import com.yahoo.document.idstring.GroupDocIdString;
+import com.yahoo.document.select.*;
+
+import java.util.List;
+import java.util.regex.Pattern;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class ComparisonNode implements ExpressionNode {
+
+    // The left- and right-hand-side of this comparison.
+    private ExpressionNode lhs, rhs;
+
+    // The operator string for this.
+    private String operator;
+
+    /**
+     * Constructs a new comparison node.
+     *
+     * @param lhs The left-hand-side of the comparison.
+     * @param operator The comparison operator.
+     * @param rhs The right-hand-side of the comparison.
+     */
+    public ComparisonNode(ExpressionNode lhs, String operator, ExpressionNode rhs) {
+        this.lhs = lhs;
+        this.operator = operator;
+        this.rhs = rhs;
+    }
+
+    /**
+     * Returns the left hand side of this comparison.
+     *
+     * @return The left hand side expression.
+     */
+    public ExpressionNode getLHS() {
+        return lhs;
+    }
+
+    /**
+     * Sets the left hand side of this comparison.
+     *
+     * @param lhs The new left hand side.
+     * @return This, to allow chaining.
+     */
+    public ComparisonNode setLHS(ExpressionNode lhs) {
+        this.lhs = lhs;
+        return this;
+    }
+
+    /**
+     * Returns the comparison operator of this.
+     *
+     * @return The operator.
+     */
+    public String getOperator() {
+        return operator;
+    }
+
+    /**
+     * Sets the comparison operator of this.
+     *
+     * @param operator The operator string.
+     * @return This, to allow chaining.
+     */
+    public ComparisonNode setOperator(String operator) {
+        this.operator = operator;
+        return this;
+    }
+
+    /**
+     * Returns the right hand side of this comparison.
+     *
+     * @return The right hand side expression.
+     */
+    public ExpressionNode getRHS() {
+        return rhs;
+    }
+
+    /**
+     * Sets the right hand side of this comparison.
+     *
+     * @param rhs The new right hand side.
+     * @return This, to allow chaining.
+     */
+    public ComparisonNode setRHS(ExpressionNode rhs) {
+        this.rhs = rhs;
+        return this;
+    }
+
+    public OrderingSpecification getOrdering(IdNode lhs, LiteralNode rhs, String operator, int order) {
+        if (lhs.getWidthBits() == -1 || lhs.getDivisionBits() == -1 || !(rhs.getValue() instanceof Long)) {
+            return null;
+        }
+
+        if (operator.equals("==") || operator.equals("=")) {
+            return new OrderingSpecification(order, (Long)rhs.getValue(), lhs.getWidthBits(), lhs.getDivisionBits());
+        } 
+
+        if (order == OrderingSpecification.ASCENDING) {
+            if ((operator.equals("<") || operator.equals("<="))) {
+                return new OrderingSpecification(order, 0, lhs.getWidthBits(), lhs.getDivisionBits());
+            } 
+            if (operator.equals(">")) {
+                return new OrderingSpecification(order, (Long)rhs.getValue() + 1, lhs.getWidthBits(), lhs.getDivisionBits());
+            }
+            if (operator.equals(">=")) {
+                return new OrderingSpecification(order, (Long)rhs.getValue(), lhs.getWidthBits(), lhs.getDivisionBits());
+            }
+        } else {
+            if (operator.equals("<")) {
+                return new OrderingSpecification(order, (Long)rhs.getValue() - 1, lhs.getWidthBits(), lhs.getDivisionBits());
+            }
+            if (operator.equals("<=")) {
+                return new OrderingSpecification(order, (Long)rhs.getValue(), lhs.getWidthBits(), lhs.getDivisionBits());
+            }
+        }
+        return null;
+    }
+
+    public OrderingSpecification getOrdering(int order) {
+        if (lhs instanceof IdNode && rhs instanceof LiteralNode) {
+            return getOrdering((IdNode)lhs, (LiteralNode)rhs, operator, order);
+        } else if (rhs instanceof IdNode && lhs instanceof LiteralNode) {
+            return getOrdering((IdNode)rhs, (LiteralNode)rhs, operator, order);
+        }
+
+        return null;
+    }
+
+    // Inherit doc from ExpressionNode.
+    public BucketSet getBucketSet(BucketIdFactory factory) {
+        if (operator.equals("==") || operator.equals("=")) {
+            if (lhs instanceof IdNode && rhs instanceof LiteralNode) {
+                return compare(factory, (IdNode)lhs, (LiteralNode)rhs, operator);
+            } else if (rhs instanceof IdNode && lhs instanceof LiteralNode) {
+                return compare(factory, (IdNode)rhs, (LiteralNode)lhs, operator);
+            } else if (lhs instanceof SearchColumnNode && rhs instanceof LiteralNode) {
+                return compare(factory, (SearchColumnNode)lhs, (LiteralNode)rhs);
+            } else if (rhs instanceof SearchColumnNode && lhs instanceof LiteralNode) {
+                return compare(factory, (SearchColumnNode)rhs, (LiteralNode)lhs);
+            }
+        }
+        return null;
+    }
+
+    /**
+     * Compares a search column node with a literal node.
+     *
+     * @param factory The bucket id factory used.
+     * @param node The search column node.
+     * @param literal The literal node to compare to.
+     * @return The bucket set containing the buckets covered.
+     */
+    private BucketSet compare(BucketIdFactory factory, SearchColumnNode node, LiteralNode literal) {
+        Object value = literal.getValue();
+        int bucketCount = (int) Math.pow(2, 16);
+        if (value instanceof Long) {
+            BucketSet ret = new BucketSet();
+            for (int i = 0; i < bucketCount; i++) {
+                BucketId id = new BucketId(16, i);
+                if ((Long)value == node.getDistribution().getColumn(id)) {
+                    ret.add(new BucketId(16, i));
+                }
+            }
+            return ret;
+        }
+        return null;
+    }
+
+    private BucketSet compare(BucketIdFactory factory, IdNode id, LiteralNode literal, String operator) {
+        String field = id.getField();
+        Object value = literal.getValue();
+        if (field == null) {
+            if (value instanceof String) {
+                String name = (String)value;
+                if ((operator.equals("=") && name.contains("*")) ||
+                    (operator.equals("=~") && ((name.contains("*") || name.contains("?")))))
+                {
+                    return null; // no idea
+                }
+                return new BucketSet(factory.getBucketId(new DocumentId(name)));
+            }
+        } else if (field.equalsIgnoreCase("user")) {
+            if (value instanceof Long) {
+                return new BucketSet(new BucketId(factory.getLocationBitCount(), (Long)value));
+            }
+        } else if (field.equalsIgnoreCase("group")) {
+            if (value instanceof String) {
+                String name = (String)value;
+                if ((operator.equals("=") && name.contains("*")) ||
+                        (operator.equals("=~") && ((name.contains("*") || name.contains("?")))))
+                {
+                    return null; // no idea
+                }
+                return new BucketSet(new BucketId(factory.getLocationBitCount(), new GroupDocIdString("", name, "").getLocation()));
+            }
+        } else if (field.equalsIgnoreCase("bucket")) {
+            if (value instanceof Long) {
+                return new BucketSet(new BucketId((Long)value));
+            }
+        }
+        return null;
+    }
+
+    // Inherit doc from Node.
+    public Object evaluate(Context context) {
+        Object oLeft = lhs.evaluate(context);
+        Object oRight = rhs.evaluate(context);
+        if (oLeft == null && oRight == null) {
+            return new ResultList(Result.TRUE);
+        }
+        if (oLeft == Result.INVALID || oRight == Result.INVALID) {
+            return new ResultList(Result.INVALID);
+        }
+        if (oLeft instanceof AttributeNode.VariableValueList && oRight instanceof AttributeNode.VariableValueList) {
+            if (operator.equals("==")) {
+                return evaluateListsTrue((AttributeNode.VariableValueList)oLeft, (AttributeNode.VariableValueList)oRight);
+            } else if (operator.equals("!=")) {
+                return evaluateListsFalse((AttributeNode.VariableValueList)oLeft, (AttributeNode.VariableValueList)oRight);
+            } else {
+                return new ResultList(Result.INVALID);
+            }
+        } else if (oLeft instanceof AttributeNode.VariableValueList) {
+            return evaluateListAndSingle((AttributeNode.VariableValueList)oLeft, oRight);
+        } else if (oRight instanceof AttributeNode.VariableValueList) {
+            return evaluateListAndSingle((AttributeNode.VariableValueList)oRight, oLeft);
+        }
+        return new ResultList(evaluateBool(oLeft, oRight));
+    }
+
+    public ResultList evaluateListsTrue(AttributeNode.VariableValueList lhs, AttributeNode.VariableValueList rhs) {
+        if (lhs.size() != rhs.size()) {
+            return new ResultList(Result.FALSE);
+        }
+
+        for (int i = 0; i < lhs.size(); i++) {
+            if (!lhs.get(i).getVariables().equals(rhs.get(i).getVariables())) {
+                return new ResultList(Result.FALSE);
+            }
+
+            if (evaluateEquals(lhs.get(i).getValue(), rhs.get(i).getValue()) == Result.FALSE) {
+                return new ResultList(Result.FALSE);
+            }
+        }
+
+        return new ResultList(Result.TRUE);
+    }
+
+    public ResultList evaluateListsFalse(AttributeNode.VariableValueList lhs, AttributeNode.VariableValueList rhs) {
+        ResultList lst = evaluateListsTrue(lhs, rhs);
+        if (lst.toResult() == Result.TRUE) {
+            return new ResultList(Result.FALSE);
+        } else if (lst.toResult() == Result.FALSE) {
+            return new ResultList(Result.TRUE);
+        } else {
+            return lst;
+        }
+    }
+
+    public ResultList evaluateListAndSingle(AttributeNode.VariableValueList lhs, Object rhs) {
+        if (rhs == null && lhs == null) {
+            return new ResultList(Result.TRUE);
+        }
+
+        if (rhs == null || lhs == null) {
+            return new ResultList(Result.FALSE);
+        }
+
+        ResultList retVal = new ResultList();
+        for (int i = 0; i < lhs.size(); i++) {
+        	Result result = evaluateBool(lhs.get(i).getValue(), rhs);
+        	retVal.add((FieldPathIteratorHandler.VariableMap)lhs.get(i).getVariables().clone(), result);
+        }
+
+        return retVal;
+    }
+
+    /**
+     * Evaluate this expression on two operands, given that they are not invalid.
+     *
+     * @param lhs Left hand side of operation.
+     * @param rhs Right hand side of operation.
+     * @return The evaluation result.
+     */
+    private Result evaluateBool(Object lhs, Object rhs) {
+        if (operator.equals("==")) {
+            return evaluateEquals(lhs, rhs);
+        } else if (operator.equals("!=")) {
+            return Result.invert(evaluateEquals(lhs, rhs));
+        } else if (operator.equals("<") || operator.equals("<=") ||
+                   operator.equals(">") || operator.equals(">=")) {
+            return evaluateNumber(lhs, rhs);
+        } else if (operator.equals("=~") || operator.equals("=")) {
+            return evaluateString(lhs, rhs);
+        }
+        throw new IllegalStateException("Comparison operator '" + operator + "' is not supported.");
+    }
+
+    /**
+     * Compare two operands for equality.
+     *
+     * @param lhs Left hand side of operation.
+     * @param rhs Right hand side of operation.
+     * @return Wether or not the two operands are equal.
+     */
+    private Result evaluateEquals(Object lhs, Object rhs) {
+        if (lhs == null || rhs == null) {
+            return Result.toResult(lhs == rhs);
+        }
+
+        double a = getAsNumber(lhs);
+        double b = getAsNumber(rhs);
+        if (Double.isNaN(a) || Double.isNaN(b)) {
+        	return Result.toResult(lhs.toString().equals(rhs.toString()));
+        }
+        return Result.toResult(a == b); // Ugh, comparing doubles? Should be converted to long value perhaps...
+    }
+
+    private double getAsNumber(Object value) {
+        if (value instanceof Number) {
+            return ((Number)value).doubleValue();
+        } else if (value instanceof NumericFieldValue) {
+            return getAsNumber(((NumericFieldValue)value).getNumber());
+        } else {
+            return Double.NaN; //new IllegalStateException("Term '" + value + "' (" + value.getClass() + ") does not evaluate to a number.");
+        }
+    }
+
+    /**
+     * Evalutes the value of this term over a document, given that both operands must be numbers.
+     *
+     * @param lhs Left hand side of operation.
+     * @param rhs Right hand side of operation.
+     * @return The evaluation result.
+     */
+    private Result evaluateNumber(Object lhs, Object rhs) {
+    	double a = getAsNumber(lhs);
+    	double b = getAsNumber(rhs);
+    	if (Double.isNaN(a) || Double.isNaN(b)) {
+    		return Result.INVALID;
+    	}
+        if (operator.equals("<")) {
+            return Result.toResult(a < b);
+        } else if (operator.equals("<=")) {
+            return Result.toResult(a <= b);
+        } else if (operator.equals(">")) {
+            return Result.toResult(a > b);
+        } else {
+            return Result.toResult(a >= b);
+        }
+    }
+
+    /**
+     * Evalutes the value of this term over a document, given that both operands must be strings.
+     *
+     * @param lhs Left hand side of operation.
+     * @param rhs Right hand side of operation.
+     * @return The evaluation result.
+     */
+    private Result evaluateString(Object lhs, Object rhs) {
+        String left = "" + lhs; // Allows null objects to evaluate to string.
+        String right = "" + rhs;
+        if (operator.equals("=~")) {
+            return Result.toResult(Pattern.compile(right).matcher(left).find());
+        } else {
+            return Result.toResult(Pattern.compile(globToRegex(right)).matcher(left).find());
+        }
+    }
+
+    /**
+     * Converts a glob pattern to a corresponding regular expression string.
+     *
+     * @param glob The glob pattern.
+     * @return The regex string.
+     */
+    private String globToRegex(String glob) {
+        StringBuilder ret = new StringBuilder();
+        ret.append("^");
+        for (int i = 0; i < glob.length(); i++) {
+            ret.append(globToRegex(glob.charAt(i)));
+        }
+        ret.append("$");
+
+        return ret.toString();
+    }
+
+    /**
+     * Converts a single character in a glob expression to the corresponding regular expression string.
+     *
+     * @param glob The glob character.
+     * @return The regex string.
+     */
+    private String globToRegex(char glob) {
+        switch (glob) {
+            case'*':
+                return ".*";
+            case'?':
+                return ".";
+            case'^':
+            case'$':
+            case'|':
+            case'{':
+            case'}':
+            case'(':
+            case')':
+            case'[':
+            case']':
+            case'\\':
+            case'+':
+            case'.':
+                return "\\" + glob;
+            default:
+                return "" + glob;
+        }
+    }
+
+    public void accept(Visitor visitor) {
+        visitor.visit(this);
+    }
+
+    // Inherit doc from Object.
+    @Override
+    public String toString() {
+        return lhs + " " + operator + " " + rhs;
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/select/rule/DocumentNode.java b/document/src/main/java/com/yahoo/document/select/rule/DocumentNode.java
new file mode 100644
index 00000000000..c071e6674aa
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/select/rule/DocumentNode.java
@@ -0,0 +1,65 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.select.rule;
+
+import com.yahoo.document.*;
+import com.yahoo.document.select.BucketSet;
+import com.yahoo.document.select.Context;
+import com.yahoo.document.select.OrderingSpecification;
+import com.yahoo.document.select.Visitor;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class DocumentNode implements ExpressionNode {
+
+    private String type;
+
+    public DocumentNode(String type) {
+        this.type = type;
+    }
+
+    public String getType() {
+        return type;
+    }
+
+    public DocumentNode setType(String type) {
+        this.type = type;
+        return this;
+    }
+
+    @Override
+    public BucketSet getBucketSet(BucketIdFactory factory) {
+        return null;
+    }
+
+    @Override
+    public Object evaluate(Context context) {
+        return evaluate(context.getDocumentOperation());
+    }
+
+    public Object evaluate(DocumentOperation op) {
+        DocumentType doct;
+        if (op instanceof DocumentPut) {
+            doct = ((DocumentPut)op).getDocument().getDataType();
+        } else if (op instanceof DocumentUpdate) {
+            doct = ((DocumentUpdate)op).getDocumentType();
+        } else {
+            throw new IllegalStateException("Document class '" + op.getClass().getName() + "' is not supported.");
+        }
+        return doct.isA(this.type) ? op : Boolean.FALSE;
+    }
+
+    public void accept(Visitor visitor) {
+        visitor.visit(this);
+    }
+
+    @Override
+    public String toString() {
+        return type;
+    }
+
+    @Override
+    public OrderingSpecification getOrdering(int order) {
+        return null;
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/select/rule/EmbracedNode.java b/document/src/main/java/com/yahoo/document/select/rule/EmbracedNode.java
new file mode 100644
index 00000000000..2a74a5af619
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/select/rule/EmbracedNode.java
@@ -0,0 +1,51 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.select.rule;
+
+import com.yahoo.document.BucketIdFactory;
+import com.yahoo.document.select.BucketSet;
+import com.yahoo.document.select.Context;
+import com.yahoo.document.select.OrderingSpecification;
+import com.yahoo.document.select.Visitor;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class EmbracedNode implements ExpressionNode {
+
+    private ExpressionNode node;
+
+    public EmbracedNode(ExpressionNode node) {
+        this.node = node;
+    }
+
+    public ExpressionNode getNode() {
+        return node;
+    }
+
+    public EmbracedNode setNode(ExpressionNode node) {
+        this.node = node;
+        return this;
+    }
+
+    // Inherit doc from ExpressionNode.
+    public BucketSet getBucketSet(BucketIdFactory factory) {
+        return null;
+    }
+
+    public Object evaluate(Context context) {
+        return node.evaluate(context);
+    }
+
+    @Override
+    public String toString() {
+        return "(" + node + ")";
+    }
+
+    public void accept(Visitor visitor) {
+        visitor.visit(this);
+    }
+
+    public OrderingSpecification getOrdering(int order) {
+        return null;
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/select/rule/ExpressionNode.java b/document/src/main/java/com/yahoo/document/select/rule/ExpressionNode.java
new file mode 100644
index 00000000000..ba51ec840d4
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/select/rule/ExpressionNode.java
@@ -0,0 +1,48 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.select.rule;
+
+import com.yahoo.document.BucketIdFactory;
+import com.yahoo.document.select.BucketSet;
+import com.yahoo.document.select.Context;
+import com.yahoo.document.select.OrderingSpecification;
+import com.yahoo.document.select.Visitor;
+
+/**
+ * This is the interface of all expression nodes. It declares the methods requires by all expression nodes to maintain
+ * a working document selector language.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public interface ExpressionNode {
+
+    /**
+     * Evaluate the content of this node based on document object, and return that value.
+     *
+     * @param doc The document to evaluate over.
+     * @return The value of this.
+     */
+    public Object evaluate(Context doc);
+
+    /**
+     * Returns the set of bucket ids covered by this node.
+     *
+     * @param factory The factory used by the current application.
+     */
+    public BucketSet getBucketSet(BucketIdFactory factory);
+
+    /**
+     * If this document selection implies a specific ordering (using the orderdoc scheme),
+     * return that specification.  
+     *
+     * @param order The order in which we are looking to traverse the ordering (ASCENDING or DESCENDING)
+     */
+    public OrderingSpecification getOrdering(int order);
+
+    /**
+     * Perform visitation of this node.
+     *
+     * @param visitor The visitor that wishes to visit the node.
+     */
+    public void accept(Visitor visitor);
+
+}
diff --git a/document/src/main/java/com/yahoo/document/select/rule/IdNode.java b/document/src/main/java/com/yahoo/document/select/rule/IdNode.java
new file mode 100644
index 00000000000..f12ecbb752b
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/select/rule/IdNode.java
@@ -0,0 +1,108 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.select.rule;
+
+import com.yahoo.document.DocumentId;
+import com.yahoo.document.BucketIdFactory;
+import com.yahoo.document.select.*;
+import com.yahoo.document.idstring.*;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class IdNode implements ExpressionNode {
+
+    private String field;
+    private short widthBits = -1;
+    private short divisionBits = -1;
+
+    public IdNode() {
+        // empty
+    }
+
+    public String getField() {
+        return field;
+    }
+
+    public IdNode setField(String field) {
+        this.field = field;
+        return this;
+    }
+
+    public IdNode setWidthBits(short widthBits) {
+        this.widthBits = widthBits;
+        return this;
+    }
+
+    public IdNode setDivisionBits(short divisionBits) {
+        this.divisionBits = divisionBits;
+        return this;
+    }
+
+    public short getWidthBits() {
+        return widthBits;
+    }
+
+    public short getDivisionBits() {
+        return divisionBits;
+    }
+
+    // Inherit doc from ExpressionNode.
+    public BucketSet getBucketSet(BucketIdFactory factory) {
+        return null;
+    }
+
+    public OrderingSpecification getOrdering(int ordering) {
+        return null;
+    }
+
+    // Inherit doc from ExpressionNode.
+    public Object evaluate(Context context) {
+        DocumentId id = context.getDocumentOperation().getId();
+        if (id == null) {
+            throw new IllegalStateException("Document has no identifier.");
+        }
+        if (field == null) {
+            return id.toString();
+        } else if (field.equalsIgnoreCase("scheme")) {
+            return id.getScheme().getType().toString();
+        } else if (field.equalsIgnoreCase("namespace")) {
+            return id.getScheme().getNamespace();
+        } else if (field.equalsIgnoreCase("specific")) {
+            return id.getScheme().getNamespaceSpecific();
+        } else if (field.equalsIgnoreCase("group")) {
+            if (id.getScheme().hasGroup()) {
+                return id.getScheme().getGroup();
+            }
+            throw new IllegalStateException("Group identifier is null."); 
+        } else if (field.equalsIgnoreCase("user")) {
+            if (id.getScheme().hasNumber()) {
+                return id.getScheme().getNumber();
+            }
+            throw new IllegalStateException("User identifier is null.");
+        } else if (field.equalsIgnoreCase("type")) {
+            if (id.getScheme().hasDocType()) {
+                return id.getScheme().getDocType();
+            }
+            throw new IllegalStateException("Document id doesn't have doc type.");
+        } else if (field.equalsIgnoreCase("order")) {
+            if (id.getScheme() instanceof OrderDocIdString) {
+                OrderDocIdString ods = (OrderDocIdString)id.getScheme();
+                if (ods.getWidthBits() == widthBits && ods.getDivisionBits() == divisionBits) {
+                    return ods.getOrdering();
+                }
+            }
+        } else{
+            throw new IllegalStateException("Identifier field '" + field + "' is not supported.");
+        }
+        return null;
+    }
+
+    public void accept(Visitor visitor) {
+        visitor.visit(this);
+    }
+
+    @Override
+    public String toString() {
+        return "id" + (field != null ? "." + field : "") + (widthBits != -1 ? "(" + widthBits + "," + divisionBits + ")" : "");
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/select/rule/LiteralNode.java b/document/src/main/java/com/yahoo/document/select/rule/LiteralNode.java
new file mode 100644
index 00000000000..ae0d640d471
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/select/rule/LiteralNode.java
@@ -0,0 +1,61 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.select.rule;
+
+import com.yahoo.document.BucketIdFactory;
+import com.yahoo.document.select.BucketSet;
+import com.yahoo.document.select.Context;
+import com.yahoo.document.select.OrderingSpecification;
+import com.yahoo.document.select.Visitor;
+import com.yahoo.document.select.parser.SelectParserUtils;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class LiteralNode implements ExpressionNode {
+
+    private Object value;
+
+    public LiteralNode(Object value) {
+        this.value = value;
+    }
+
+    public Object getValue() {
+        return value;
+    }
+
+    public LiteralNode setValue(Object value) {
+        this.value = value;
+        return this;
+    }
+
+    @Override
+    public BucketSet getBucketSet(BucketIdFactory factory) {
+        return null;
+    }
+
+    @Override
+    public Object evaluate(Context context) {
+        return value;
+    }
+
+    @Override
+    public void accept(Visitor visitor) {
+        visitor.visit(this);
+    }
+
+    @Override
+    public String toString() {
+        if (value == null) {
+            return "null";
+        } else if (value instanceof String) {
+            return SelectParserUtils.quote((String)value, '"');
+        } else {
+            return value.toString();
+        }
+    }
+
+    @Override
+    public OrderingSpecification getOrdering(int order) {
+        return null;
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/select/rule/LogicNode.java b/document/src/main/java/com/yahoo/document/select/rule/LogicNode.java
new file mode 100644
index 00000000000..145c655fae2
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/select/rule/LogicNode.java
@@ -0,0 +1,316 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.select.rule;
+
+import com.yahoo.document.BucketIdFactory;
+import com.yahoo.document.select.BucketSet;
+import com.yahoo.document.select.Context;
+import com.yahoo.document.select.OrderingSpecification;
+import com.yahoo.document.select.ResultList;
+import com.yahoo.document.select.Visitor;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Stack;
+
+/**
+ * This class defines a logical expression of nodes. This implementation uses a stack to evaluate its content as to
+ * avoid deep recursions when building the parse tree.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class LogicNode implements ExpressionNode {
+
+    // A no-op value is defined for completeness.
+    public static final int NOP = 0;
+
+    // The OR operator has lower precedence than AND.
+    public static final int OR = 1;
+
+    // The AND operator has the highest precedence.
+    public static final int AND = 2;
+
+    // The items contained in this.
+    private final List<NodeItem> items = new ArrayList<NodeItem>();
+
+    /**
+     * Construct an empty logic expression.
+     */
+    public LogicNode() {
+        // empty
+    }
+
+    public List<NodeItem> getItems() {
+        return items;
+    }
+    
+    /**
+     * Adds an (operator, node) pair to this expression.
+     *
+     * @param operator The operator that combines the previous with the node given.
+     * @param node The node to add to this.
+     * @return This, to allow chaining.
+     */
+    public LogicNode add(String operator, ExpressionNode node) {
+        items.add(new LogicNode.NodeItem(stringToOperator(operator), node));
+        return this;
+    }
+
+    // Inherit doc from ExpressionNode.
+    public BucketSet getBucketSet(BucketIdFactory factory) {
+        Stack<BucketItem> buf = new Stack<>();
+        for (NodeItem item : items) {
+            if (!buf.isEmpty()) {
+                while (buf.peek().operator > item.operator) {
+                    combineBuckets(buf);
+                }
+            }
+            buf.push(new BucketItem(item.operator, item.node.getBucketSet(factory)));
+        }
+        while (buf.size() > 1) {
+            combineBuckets(buf);
+        }
+        return buf.pop().buckets;
+    }
+
+    public OrderingSpecification getOrdering(int order) {
+        Stack<OrderingItem> buf = new Stack<>();
+        for (NodeItem item : items) {
+            if (!buf.isEmpty()) {
+                while (buf.peek().operator > item.operator) {
+                    pickOrdering(buf);
+                }
+            }
+            buf.push(new OrderingItem(item.operator, item.node.getOrdering(order)));
+        }
+        while (buf.size() > 1) {
+            pickOrdering(buf);
+        }
+        return buf.pop().ordering;
+    }
+
+    private OrderingSpecification pickOrdering(OrderingSpecification a, OrderingSpecification b, boolean isAnd) {
+        if (a.getWidthBits() == b.getWidthBits() && a.getDivisionBits() == b.getDivisionBits() && a.getOrder() == b.getOrder()) {
+            if ((a.getOrder() == OrderingSpecification.ASCENDING && isAnd) ||
+                (a.getOrder() == OrderingSpecification.DESCENDING && !isAnd)) {
+                return new OrderingSpecification(a.getOrder(), Math.max(a.getOrderingStart(), b.getOrderingStart()), b.getWidthBits(), a.getDivisionBits());
+            } else {
+                return new OrderingSpecification(a.getOrder(), Math.min(a.getOrderingStart(), b.getOrderingStart()), b.getWidthBits(), a.getDivisionBits());
+            }
+        }
+        return null;
+    }
+
+    private void pickOrdering(Stack<OrderingItem> buf) {
+        OrderingItem rhs = buf.pop();
+        OrderingItem lhs = buf.pop();
+        switch (rhs.operator) {
+        case AND:
+            if (lhs.ordering == null) {
+                lhs.ordering = rhs.ordering;
+            } else if (rhs.ordering == null) {
+                // empty
+            } else {
+                lhs.ordering = pickOrdering(lhs.ordering, rhs.ordering, true);
+            }
+            break;
+        case OR:
+            if (lhs.ordering != null && rhs.ordering != null) {
+                lhs.ordering = pickOrdering(lhs.ordering, rhs.ordering, false);
+            } else {
+                lhs.ordering = null;
+            }
+            break;
+        default:
+            lhs.ordering = null;
+        }
+        buf.push(lhs);
+    }
+
+    /**
+     * Combines the top two items of the given stack using the operator of the second.
+     *
+     * @param buf The stack of bucket items.
+     */
+    private void combineBuckets(Stack<BucketItem> buf) {
+        BucketItem rhs = buf.pop();
+        BucketItem lhs = buf.pop();
+        switch (rhs.operator) {
+            case AND:
+                if (lhs.buckets == null) {
+                    lhs.buckets = rhs.buckets;
+                } else if (rhs.buckets == null) {
+                    // empty
+                } else {
+                    lhs.buckets = lhs.buckets.intersection(rhs.buckets);
+                }
+                break;
+            case OR:
+                if (lhs.buckets == null) {
+                    // empty
+                } else if (rhs.buckets == null) {
+                    lhs.buckets = null;
+                } else {
+                    lhs.buckets = lhs.buckets.union(rhs.buckets);
+                }
+                break;
+            default:
+                throw new IllegalStateException("Arithmetic operator " + rhs.operator + " not supported.");
+        }
+        buf.push(lhs);
+    }
+
+    // Inherit doc from ExpressionNode.
+    @Override
+    public Object evaluate(Context context) {
+        Stack<ValueItem> buf = new Stack<>();
+        for (NodeItem item : items) {
+            if ( ! buf.isEmpty()) {
+                while (buf.peek().operator > item.operator) {
+                    combineValues(buf);
+                }
+            }
+            
+            buf.push(new ValueItem(item.operator, ResultList.toResultList(item.node.evaluate(context))));
+        }
+        while (buf.size() > 1) {
+            combineValues(buf);
+        }
+        return buf.pop().value;
+    }
+
+    /**
+     * Combines the top two items of the given stack using the operator of the second.
+     *
+     * @param buf The stack of values.
+     */
+    private void combineValues(Stack<ValueItem> buf) {
+        ValueItem rhs = buf.pop();
+        ValueItem lhs = buf.pop();
+
+        switch (rhs.operator) {
+            case AND:
+                buf.push(new ValueItem(lhs.operator, lhs.value.combineAND(rhs.value)));
+                break;
+            case OR:
+                buf.push(new ValueItem(lhs.operator, lhs.value.combineOR(rhs.value)));
+                break;
+            default:
+                throw new IllegalStateException("Arithmetic operator " + rhs.operator + " not supported.");
+        }
+    }
+
+    public void accept(Visitor visitor) {
+        visitor.visit(this);
+    }
+
+    // Inherit doc from Object.
+    @Override
+    public String toString() {
+        StringBuilder ret = new StringBuilder();
+        for (LogicNode.NodeItem item : items) {
+            if (item.operator != NOP) {
+                ret.append(" ").append(operatorToString(item.operator)).append(" ");
+            }
+            ret.append(item.node);
+        }
+        return ret.toString();
+    }
+
+    /**
+     * Converts the given operator index to a string representation.
+     *
+     * @param operator The operator index to convert.
+     * @return The string representation.
+     */
+    public String operatorToString(int operator) {
+        switch (operator) {
+            case NOP:
+                return null;
+            case OR:
+                return "or";
+            case AND:
+                return "and";
+            default:
+                throw new IllegalStateException("Logical operator " + operator + " not supported.");
+        }
+    }
+
+    /**
+     * Converts the given operator string to a corresponding operator index. This is necessary to perform a stack
+     * traversal of logic expression.
+     *
+     * @param operator The operator to convert.
+     * @return The corresponding index.
+     */
+    private int stringToOperator(String operator) {
+        if (operator == null) {
+            return NOP;
+        } else if (operator.equalsIgnoreCase("or")) {
+            return OR;
+        } else if (operator.equalsIgnoreCase("and")) {
+            return AND;
+        } else {
+            throw new IllegalStateException("Logical operator '" + operator + "' not supported.");
+        }
+    }
+
+    /**
+     * Private class to store results in a stack.
+     */
+    private final class ValueItem {
+        private int operator;
+        private ResultList value;
+
+        public ValueItem(int operator, ResultList value) {
+            this.operator = operator;
+            this.value = value;
+        }
+    }
+
+    /**
+     * Private class to store bucket sets in a stack.
+     */
+    private final class BucketItem {
+        private int operator;
+        private BucketSet buckets;
+
+        public BucketItem(int operator, BucketSet buckets) {
+            this.operator = operator;
+            this.buckets = buckets;
+        }
+    }
+
+    /**
+     * Private class to store ordering expressions in a stack.
+     */
+    private final class OrderingItem {
+        private int operator;
+        private OrderingSpecification ordering;
+
+        public OrderingItem(int operator, OrderingSpecification orderSpec) {
+            this.operator = operator;
+            this.ordering = orderSpec;
+        }
+    }
+
+    /**
+     * Private class to store expression nodes in a stack.
+     */
+    public final class NodeItem {
+        private int operator;
+        private ExpressionNode node;
+
+        public NodeItem(int operator, ExpressionNode node) {
+            this.operator = operator;
+            this.node = node;
+        }
+
+        public int getOperator() {
+            return operator;
+        }
+
+        public ExpressionNode getNode() {
+            return node;
+        }
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/select/rule/NegationNode.java b/document/src/main/java/com/yahoo/document/select/rule/NegationNode.java
new file mode 100644
index 00000000000..2b85bc3eee6
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/select/rule/NegationNode.java
@@ -0,0 +1,53 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.select.rule;
+
+import com.yahoo.document.BucketIdFactory;
+import com.yahoo.document.select.BucketSet;
+import com.yahoo.document.select.Context;
+import com.yahoo.document.select.OrderingSpecification;
+import com.yahoo.document.select.Result;
+import com.yahoo.document.select.Visitor;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class NegationNode implements ExpressionNode {
+
+    private ExpressionNode node;
+
+    public NegationNode(ExpressionNode node) {
+        this.node = node;
+    }
+
+    public ExpressionNode getNode() {
+        return node;
+    }
+
+    public NegationNode setNode(ExpressionNode node) {
+        this.node = node;
+        return this;
+    }
+
+    // Inherit doc from ExpressionNode.
+    public BucketSet getBucketSet(BucketIdFactory factory) {
+        return null;
+    }
+
+    // Inherit doc from ExpressionNode.
+    public Object evaluate(Context context) {
+        return Result.invert(Result.toResult(node.evaluate(context)));
+    }
+
+    public void accept(Visitor visitor) {
+        visitor.visit(this);
+    }
+
+    @Override
+    public String toString() {
+        return "not " + node;
+    }
+
+    public OrderingSpecification getOrdering(int order) {
+        return null;
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/select/rule/NowNode.java b/document/src/main/java/com/yahoo/document/select/rule/NowNode.java
new file mode 100644
index 00000000000..600dbe536e4
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/select/rule/NowNode.java
@@ -0,0 +1,34 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.select.rule;
+import com.yahoo.document.BucketIdFactory;
+import com.yahoo.document.select.*;
+
+/**
+ * @author <a href="mailto:lulf@yahoo-inc.com">Ulf Lilleengen</a>
+ */
+public class NowNode implements ExpressionNode {
+
+    // Inherit doc from ExpressionNode.
+    public BucketSet getBucketSet(BucketIdFactory factory) {
+        return null;
+    }
+
+    // Inherit doc from ExpressionNode.
+    public Object evaluate(Context context) {
+        Object ret = System.currentTimeMillis() / 1000;  
+        return ret;
+    }
+
+    @Override
+    public String toString() {
+        return "now()";
+    }
+
+    public OrderingSpecification getOrdering(int order) {
+        return null;
+    }
+
+    public void accept(Visitor visitor) {
+        visitor.visit(this);
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/select/rule/SearchColumnNode.java b/document/src/main/java/com/yahoo/document/select/rule/SearchColumnNode.java
new file mode 100644
index 00000000000..c63197ae3a4
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/select/rule/SearchColumnNode.java
@@ -0,0 +1,56 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.select.rule;
+
+import com.yahoo.document.BucketDistribution;
+import com.yahoo.document.BucketIdFactory;
+import com.yahoo.document.select.*;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class SearchColumnNode implements ExpressionNode {
+
+    private int field;
+    private BucketIdFactory factory = new BucketIdFactory(); // why is this not an abstract class?
+    private BucketDistribution distribution;
+
+    public SearchColumnNode() {
+        setField(0);
+    }
+
+    public int getField() {
+        return field;
+    }
+
+    public SearchColumnNode setField(int field) {
+        distribution = new BucketDistribution(this.field = field, 16);
+        return this;
+    }
+
+    public BucketDistribution getDistribution() {
+        return distribution;
+    }
+
+    // Inherit doc from ExpressionNode.
+    public BucketSet getBucketSet(BucketIdFactory factory) {
+        return null;
+    }
+
+    // Inherit doc from ExpressionNode.
+    public Object evaluate(Context context) {
+        return distribution.getColumn(factory.getBucketId(context.getDocumentOperation().getId()));
+    }
+
+    public void accept(Visitor visitor) {
+        visitor.visit(this);
+    }
+
+    @Override
+    public String toString() {
+        return "searchcolumn." + field;
+    }
+
+    public OrderingSpecification getOrdering(int order) {
+        return null;
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/select/rule/VariableNode.java b/document/src/main/java/com/yahoo/document/select/rule/VariableNode.java
new file mode 100644
index 00000000000..77c462674db
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/select/rule/VariableNode.java
@@ -0,0 +1,58 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.select.rule;
+
+import com.yahoo.document.BucketIdFactory;
+import com.yahoo.document.select.BucketSet;
+import com.yahoo.document.select.Context;
+import com.yahoo.document.select.OrderingSpecification;
+import com.yahoo.document.select.Visitor;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class VariableNode implements ExpressionNode {
+
+    private String value;
+
+    public VariableNode(String value) {
+        this.value = value;
+    }
+
+    public Object getValue() {
+        return value;
+    }
+
+    public VariableNode setValue(String value) {
+        this.value = value;
+        return this;
+    }
+
+    @Override
+    public BucketSet getBucketSet(BucketIdFactory factory) {
+        return null;
+    }
+
+    @Override
+    public Object evaluate(Context context) {
+        Object o = context.getVariables().get(value);
+        if (o == null) {
+            throw new IllegalArgumentException("Variable " + value + " was not set in the variable list");
+        }
+        return o;
+    }
+
+    @Override
+    public void accept(Visitor visitor) {
+        visitor.visit(this);
+    }
+
+    @Override
+    public String toString() {
+        return "$" + value;
+    }
+
+    @Override
+    public OrderingSpecification getOrdering(int order) {
+        return null;
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/select/rule/package-info.java b/document/src/main/java/com/yahoo/document/select/rule/package-info.java
new file mode 100644
index 00000000000..b5268393037
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/select/rule/package-info.java
@@ -0,0 +1,7 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+@PublicApi
+package com.yahoo.document.select.rule;
+
+import com.yahoo.api.annotations.PublicApi;
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/document/src/main/java/com/yahoo/document/select/simple/IdSpecParser.java b/document/src/main/java/com/yahoo/document/select/simple/IdSpecParser.java
new file mode 100644
index 00000000000..b444c2d5ac2
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/select/simple/IdSpecParser.java
@@ -0,0 +1,63 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.select.simple;
+
+import com.yahoo.document.select.rule.IdNode;
+
+/**
+ * @author balder
+ */
+public class IdSpecParser extends Parser {
+    private IdNode id;
+    public IdNode getId() { return id; }
+    public boolean isUserSpec() { return "user".equals(id.getField()); }
+    public boolean parse(CharSequence s) {
+        boolean retval = false;
+        int pos = eatWhite(s);
+        if (pos+1 < s.length()) {
+            if (icmp(s.charAt(pos), 'i') && icmp(s.charAt(pos+1),'d')) {
+                pos += 2;
+                if (pos < s.length()) {
+                    switch (s.charAt(pos)) {
+                    case '.':
+                        {
+                            int startPos = ++pos;
+                            for (;pos < s.length() && (Character.toLowerCase(s.charAt(pos)) >= 'a') && (Character.toLowerCase(s.charAt(pos)) <= 'z'); pos++);
+                            int len = pos - startPos;
+                            if (((len == 4) && "user".equalsIgnoreCase(s.subSequence(startPos, startPos + 4).toString())) ||
+                                ((len == 5) && "group".equalsIgnoreCase(s.subSequence(startPos, startPos + 5).toString())) ||
+                                ((len == 6) && "scheme".equalsIgnoreCase(s.subSequence(startPos, startPos + 6).toString())) ||
+                                ((len == 8) && "specific".equalsIgnoreCase(s.subSequence(startPos, startPos + 8).toString())) ||
+                                ((len == 9) && "namespace".equalsIgnoreCase(s.subSequence(startPos, startPos + 9).toString())))
+                            {
+                                retval = true;
+                                id = new IdNode().setField(s.subSequence(startPos, startPos + len).toString());
+                            } else {
+                                pos = startPos;
+                            }
+                        }
+                        break;
+                    case '!':
+                    case '<':
+                    case '>':
+                    case '=':
+                    case '\t':
+                    case '\n':
+                    case '\r':
+                    case ' ':
+                        {
+                            retval = true;
+                            id = new IdNode();
+                        }
+                        break;
+                    default:
+                        break;
+                    }
+                }
+            }
+        }
+        setRemaining(s.subSequence(pos, s.length()));
+
+        return retval;
+    }
+
+}
diff --git a/document/src/main/java/com/yahoo/document/select/simple/IntegerParser.java b/document/src/main/java/com/yahoo/document/select/simple/IntegerParser.java
new file mode 100644
index 00000000000..52a0b0a2c4f
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/select/simple/IntegerParser.java
@@ -0,0 +1,45 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.select.simple;
+
+import com.yahoo.document.select.rule.LiteralNode;
+
+/**
+ * @author balder
+ */
+public class IntegerParser extends Parser {
+    private LiteralNode value;
+    public LiteralNode getValue() { return value; }
+
+    public boolean parse(CharSequence s) {
+        boolean retval = false;
+        int pos = eatWhite(s);
+        if (pos < s.length()) {
+            boolean isHex = ((s.length() - pos) > 2) && (s.charAt(pos) == '0') && (s.charAt(pos+1) == 'x');
+            Long v = null;
+            int startPos = pos;
+            if (isHex) {
+                for(startPos = pos+2; (pos < s.length()) && (((s.charAt(pos) >= '0') && (s.charAt(pos) <= '9')) ||
+                                                             ((s.charAt(pos) >= 'a') && (s.charAt(pos) <= 'f')) ||
+                                                             ((s.charAt(pos) >= 'A') && (s.charAt(pos) <= 'F'))); pos++);
+                if (pos > startPos) {
+                    v = Long.valueOf(s.subSequence(startPos, pos).toString(), 16);
+                    retval = true;
+                }
+
+            } else {
+                if ((s.charAt(pos) == '-') || (s.charAt(pos) == '+')) {
+                    pos++;
+                }
+                for(;(pos < s.length()) && (s.charAt(pos) >= '0') && (s.charAt(pos) <= '9') ; pos++);
+                if (pos > startPos) {
+                    v = Long.valueOf(s.subSequence(startPos, pos).toString(), 10);
+                    retval = true;
+                }
+            }
+            value = new LiteralNode(v);
+        }
+        setRemaining(s.subSequence(pos, s.length()));
+
+        return retval;
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/select/simple/OperatorParser.java b/document/src/main/java/com/yahoo/document/select/simple/OperatorParser.java
new file mode 100644
index 00000000000..58da1e0c179
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/select/simple/OperatorParser.java
@@ -0,0 +1,45 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.select.simple;
+
+/**
+ * @author balder
+ */
+public class OperatorParser extends Parser {
+    private String operator;
+    public String getOperator() { return operator; }
+    public boolean parse(CharSequence s) {
+        boolean retval = false;
+        int pos = eatWhite(s);
+
+        if (pos+1 < s.length()) {
+            retval = true;
+            int startPos = pos;
+            if (s.charAt(pos) == '=') {
+                pos++;
+                if ((s.charAt(pos) == '=') || (s.charAt(pos) == '~')) {
+                    pos++;
+                }
+            } else if (s.charAt(pos) == '>') {
+                pos++;
+                if (s.charAt(pos) == '=') {
+                    pos++;
+                }
+            } else if (s.charAt(pos) == '<') {
+                pos++;
+                if (s.charAt(pos) == '=') {
+                    pos++;
+                }
+            } else if ((s.charAt(pos) == '!') && (s.charAt(pos) == '=')) {
+                pos += 2;
+            } else {
+                retval = false;
+            }
+            if (retval) {
+                operator = s.subSequence(startPos, pos).toString();
+            }
+        }
+        setRemaining(s.subSequence(pos, s.length()));
+
+        return retval;
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/select/simple/Parser.java b/document/src/main/java/com/yahoo/document/select/simple/Parser.java
new file mode 100644
index 00000000000..169f100de83
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/select/simple/Parser.java
@@ -0,0 +1,20 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.select.simple;
+
+/**
+ * @author balder
+ */
+public abstract class Parser {
+    public abstract boolean parse(CharSequence s);
+    public CharSequence getRemaining() { return remaining; }
+    protected void setRemaining(CharSequence r) { remaining = r; }
+    private CharSequence remaining;
+    protected int eatWhite(CharSequence s) {
+        int pos = 0;
+        for (;pos < s.length() && Character.isSpaceChar(s.charAt(pos)); pos++);
+        return pos;
+    }
+    protected boolean icmp(char c, char l) {
+        return Character.toLowerCase(c) == l;
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/select/simple/SelectionParser.java b/document/src/main/java/com/yahoo/document/select/simple/SelectionParser.java
new file mode 100644
index 00000000000..df3507e67c8
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/select/simple/SelectionParser.java
@@ -0,0 +1,43 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.select.simple;
+
+import com.yahoo.document.select.rule.ComparisonNode;
+import com.yahoo.document.select.rule.ExpressionNode;
+
+/**
+ * @author balder
+ */
+public class SelectionParser extends Parser {
+    private ExpressionNode node;
+    public ExpressionNode getNode() { return node; }
+    public boolean parse(CharSequence s) {
+        boolean retval = false;
+        IdSpecParser id = new IdSpecParser();
+        if (id.parse(s)) {
+            OperatorParser op = new OperatorParser();
+            if (op.parse(id.getRemaining())) {
+                if (id.isUserSpec()) {
+                    IntegerParser v = new IntegerParser();
+                    if (v.parse(op.getRemaining())) {
+                        node = new ComparisonNode(id.getId(), op.getOperator(), v.getValue());
+                        retval = true;
+                    }
+                    setRemaining(v.getRemaining());
+                } else {
+                    StringParser v = new StringParser();
+                    if (v.parse(op.getRemaining())) {
+                        node = new ComparisonNode(id.getId(), op.getOperator(), v.getValue());
+                        retval = true;
+                    }
+                    setRemaining(v.getRemaining());
+                }
+            } else {
+                setRemaining(op.getRemaining());
+            }
+        } else {
+            setRemaining(id.getRemaining());
+        }
+
+        return retval;
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/select/simple/StringParser.java b/document/src/main/java/com/yahoo/document/select/simple/StringParser.java
new file mode 100644
index 00000000000..e18c2dad8e2
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/select/simple/StringParser.java
@@ -0,0 +1,35 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.select.simple;
+
+import com.yahoo.document.select.rule.LiteralNode;
+
+/**
+ * @author balder
+ */
+public class StringParser extends Parser {
+    private LiteralNode value;
+    public LiteralNode getValue() { return value; }
+    public boolean parse(CharSequence s) {
+        boolean retval = false;
+        int pos = eatWhite(s);
+        if (pos + 1 < s.length()) {
+        if (s.charAt(pos++) == '"') {
+            StringBuffer str = new StringBuffer("");
+            for(; (pos < s.length()) && (s.charAt(pos) != '"');pos++) {
+                if ((pos < s.length()) && (s.charAt(pos) == '\\')) {
+                    pos++;
+                }
+                str.append(s.charAt(pos));
+            }
+            if (s.charAt(pos) == '"') {
+                pos++;
+                retval = true;
+                value = new LiteralNode(str.toString());
+            }
+        }
+
+        setRemaining(s.subSequence(pos, s.length()));
+    }
+        return retval;
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/select/simple/package-info.java b/document/src/main/java/com/yahoo/document/select/simple/package-info.java
new file mode 100644
index 00000000000..58d4cc3428b
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/select/simple/package-info.java
@@ -0,0 +1,7 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+@PublicApi
+package com.yahoo.document.select.simple;
+
+import com.yahoo.api.annotations.PublicApi;
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/document/src/main/java/com/yahoo/document/serialization/AnnotationReader.java b/document/src/main/java/com/yahoo/document/serialization/AnnotationReader.java
new file mode 100644
index 00000000000..92b042249d2
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/serialization/AnnotationReader.java
@@ -0,0 +1,12 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.serialization;
+
+import com.yahoo.document.annotation.Annotation;
+import com.yahoo.document.annotation.AnnotationType;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public interface AnnotationReader {
+    public void read(Annotation annotation);
+}
diff --git a/document/src/main/java/com/yahoo/document/serialization/AnnotationWriter.java b/document/src/main/java/com/yahoo/document/serialization/AnnotationWriter.java
new file mode 100644
index 00000000000..061798e91b8
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/serialization/AnnotationWriter.java
@@ -0,0 +1,12 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.serialization;
+
+import com.yahoo.document.annotation.Annotation;
+import com.yahoo.document.annotation.AnnotationType;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public interface AnnotationWriter {
+    public void write(Annotation annotation);
+}
diff --git a/document/src/main/java/com/yahoo/document/serialization/DeserializationException.java b/document/src/main/java/com/yahoo/document/serialization/DeserializationException.java
new file mode 100644
index 00000000000..a189290fb13
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/serialization/DeserializationException.java
@@ -0,0 +1,21 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.serialization;
+
+/**
+ * Exception which is thrown when deserialization fails.
+ *
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class DeserializationException extends RuntimeException {
+    public DeserializationException(String msg) {
+        super(msg);
+    }
+
+    public DeserializationException(String msg, Throwable cause) {
+        super(msg, cause);
+    }
+
+    public DeserializationException(Throwable cause) {
+        super(cause);
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/serialization/DocumentDeserializer.java b/document/src/main/java/com/yahoo/document/serialization/DocumentDeserializer.java
new file mode 100644
index 00000000000..def358e0624
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/serialization/DocumentDeserializer.java
@@ -0,0 +1,21 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.serialization;
+
+import com.yahoo.io.GrowableByteBuffer;
+
+/**
+ * Interface for de-serializing documents.
+ *
+ * A particular instance of this class is tied to a version of the document format.
+ *
+ * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a>
+ */
+public interface DocumentDeserializer extends DocumentReader, DocumentUpdateReader, FieldReader, AnnotationReader, SpanNodeReader, SpanTreeReader {
+
+    /**
+     * Returns the underlying buffer used for de-serialization.
+     */
+    public GrowableByteBuffer getBuf();
+
+}
+
diff --git a/document/src/main/java/com/yahoo/document/serialization/DocumentDeserializerFactory.java b/document/src/main/java/com/yahoo/document/serialization/DocumentDeserializerFactory.java
new file mode 100644
index 00000000000..cef5b024837
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/serialization/DocumentDeserializerFactory.java
@@ -0,0 +1,35 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.serialization;
+
+import com.yahoo.document.DocumentTypeManager;
+import com.yahoo.io.GrowableByteBuffer;
+
+/**
+ * Factory for creating document de-serializers tied to a document format.
+ *
+ * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a>
+ */
+public class DocumentDeserializerFactory {
+
+    /**
+     * Creates a de-serializer for the current head document format.
+     * This format is an extension of the 4.2 format.
+     */
+    public static DocumentDeserializer createHead(DocumentTypeManager manager, GrowableByteBuffer buf) {
+        return new VespaDocumentDeserializerHead(manager, buf);
+    }
+
+    /**
+     * Creates a de-serializer for the document format that was created on Vespa 4.2.
+     */
+    public static DocumentDeserializer create42(DocumentTypeManager manager, GrowableByteBuffer buf) {
+        return new VespaDocumentDeserializer42(manager, buf);
+    }
+
+    /**
+     * Creates a de-serializer for the document format that was created on Vespa 4.2.
+     */
+    public static DocumentDeserializer create42(DocumentTypeManager manager, GrowableByteBuffer buf, GrowableByteBuffer body) {
+        return new VespaDocumentDeserializer42(manager, buf, body);
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/serialization/DocumentReader.java b/document/src/main/java/com/yahoo/document/serialization/DocumentReader.java
new file mode 100644
index 00000000000..8a5e889eb8c
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/serialization/DocumentReader.java
@@ -0,0 +1,27 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.serialization;
+
+import com.yahoo.document.Document;
+import com.yahoo.document.DocumentId;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.DocumentTypeManager;
+
+/**
+ * This interface is used to implement custom deserialization of document updates.
+ *
+ * @author <a href="mailto:ravishar@yahoo-inc.com">Ravi Sharma</a>
+ * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a>
+ */
+public interface DocumentReader {
+
+	/**
+	 * Read a document
+	 *
+	 * @param document - document to be read
+	 */
+	void read(Document document);
+
+    DocumentId readDocumentId();
+    DocumentType readDocumentType();
+
+}
diff --git a/document/src/main/java/com/yahoo/document/serialization/DocumentSerializer.java b/document/src/main/java/com/yahoo/document/serialization/DocumentSerializer.java
new file mode 100644
index 00000000000..6eb0fc60c3f
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/serialization/DocumentSerializer.java
@@ -0,0 +1,20 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.serialization;
+
+import com.yahoo.io.GrowableByteBuffer;
+
+/**
+ * Interface for serializing documents.
+ *
+ * A particular instance of this class is tied to a version of the document format.
+ *
+ * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a>
+ */
+public interface DocumentSerializer extends DocumentWriter, SpanNodeWriter, AnnotationWriter, SpanTreeWriter, DocumentUpdateWriter {
+
+    /**
+     * Returns the underlying buffer used for serialization.
+     */
+    public GrowableByteBuffer getBuf();
+}
+
diff --git a/document/src/main/java/com/yahoo/document/serialization/DocumentSerializerFactory.java b/document/src/main/java/com/yahoo/document/serialization/DocumentSerializerFactory.java
new file mode 100644
index 00000000000..50268161aad
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/serialization/DocumentSerializerFactory.java
@@ -0,0 +1,42 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.serialization;
+
+import com.yahoo.io.GrowableByteBuffer;
+
+/**
+ * Factory for creating document serializers tied to a document format.
+ *
+ * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a>
+ */
+public class DocumentSerializerFactory {
+
+    /**
+     * Creates a serializer for the current head document format.
+     * This format is an extension of the 4.2 format.
+     */
+    public static DocumentSerializer createHead(GrowableByteBuffer buf) {
+        return new VespaDocumentSerializerHead(buf);
+    }
+
+    /**
+     * Creates a serializer for the document format that was created on Vespa 4.2.
+     */
+    public static DocumentSerializer create42(GrowableByteBuffer buf) {
+        return new VespaDocumentSerializer42(buf);
+    }
+
+    /**
+     * Creates a serializer for the document format that was created on Vespa 4.2.
+     */
+    public static DocumentSerializer create42(GrowableByteBuffer buf, boolean headerOnly) {
+        return new VespaDocumentSerializer42(buf, headerOnly);
+    }
+
+    /**
+     * Creates a serializer for the document format that was created on Vespa 4.2.
+     */
+    public static DocumentSerializer create42() {
+        return new VespaDocumentSerializer42();
+    }
+
+}
diff --git a/document/src/main/java/com/yahoo/document/serialization/DocumentUpdateFlags.java b/document/src/main/java/com/yahoo/document/serialization/DocumentUpdateFlags.java
new file mode 100644
index 00000000000..808b5d90517
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/serialization/DocumentUpdateFlags.java
@@ -0,0 +1,38 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.serialization;
+
+/**
+ * Class used to represent up to 4 flags used in a DocumentUpdate.
+ * These flags are stored as the 4 most significant bits in a 32 bit integer.
+ *
+ * Flags currently used:
+ *   0) create-if-non-existent.
+ *
+ * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a>
+ */
+public class DocumentUpdateFlags {
+    private byte flags;
+    private DocumentUpdateFlags(byte flags) {
+        this.flags = flags;
+    }
+    public DocumentUpdateFlags() {
+        this.flags = 0;
+    }
+    public boolean getCreateIfNonExistent() {
+        return (flags & 1) != 0;
+    }
+    public void setCreateIfNonExistent(boolean value) {
+        flags &= ~1; // clear flag
+        flags |= value ? 1 : 0; // set flag
+    }
+    public int injectInto(int value) {
+        return extractValue(value) | (flags << 28);
+    }
+    public static DocumentUpdateFlags extractFlags(int combined) {
+        return new DocumentUpdateFlags((byte)(combined >> 28));
+    }
+    public static int extractValue(int combined) {
+        int mask = ~(~0 << 28);
+        return combined & mask;
+    }
+}
+\ No newline at end of file
diff --git a/document/src/main/java/com/yahoo/document/serialization/DocumentUpdateReader.java b/document/src/main/java/com/yahoo/document/serialization/DocumentUpdateReader.java
new file mode 100644
index 00000000000..4e31af08d00
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/serialization/DocumentUpdateReader.java
@@ -0,0 +1,30 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.serialization;
+
+import com.yahoo.document.*;
+import com.yahoo.document.fieldpathupdate.*;
+import com.yahoo.document.update.FieldUpdate;
+
+/**
+ * This interface is used to implement custom deserialization of document updates.
+ *
+ * @author <a href="mailto:thomasg@yahoo-inc.com">Thomas Gundersen</a>
+ */
+public interface DocumentUpdateReader {
+
+    void read(DocumentUpdate update);
+
+    void read(FieldUpdate update);
+
+    void read(FieldPathUpdate update);
+
+    void read(AssignFieldPathUpdate update);
+
+    void read(AddFieldPathUpdate update);
+
+    void read(RemoveFieldPathUpdate update);
+
+    DocumentId readDocumentId();
+    DocumentType readDocumentType();
+
+}
diff --git a/document/src/main/java/com/yahoo/document/serialization/DocumentUpdateWriter.java b/document/src/main/java/com/yahoo/document/serialization/DocumentUpdateWriter.java
new file mode 100644
index 00000000000..725de8935f5
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/serialization/DocumentUpdateWriter.java
@@ -0,0 +1,29 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.serialization;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.DocumentUpdate;
+import com.yahoo.document.update.AddValueUpdate;
+import com.yahoo.document.update.ArithmeticValueUpdate;
+import com.yahoo.document.update.AssignValueUpdate;
+import com.yahoo.document.update.ClearValueUpdate;
+import com.yahoo.document.update.FieldUpdate;
+import com.yahoo.document.update.MapValueUpdate;
+import com.yahoo.document.update.RemoveValueUpdate;
+
+/**
+ * Interface for writing document updates in custom serializers.
+ *
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ * @since 5.1.27
+ */
+public interface DocumentUpdateWriter {
+    public void write(DocumentUpdate update);
+    public void write(FieldUpdate update);
+    public void write(AddValueUpdate update, DataType superType);
+    public void write(MapValueUpdate update, DataType superType);
+    public void write(ArithmeticValueUpdate update);
+    public void write(AssignValueUpdate update, DataType superType);
+    public void write(RemoveValueUpdate update, DataType superType);
+    public void write(ClearValueUpdate clearValueUpdate, DataType superType);
+}
diff --git a/document/src/main/java/com/yahoo/document/serialization/DocumentWriter.java b/document/src/main/java/com/yahoo/document/serialization/DocumentWriter.java
new file mode 100644
index 00000000000..aba9c97cf67
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/serialization/DocumentWriter.java
@@ -0,0 +1,23 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.serialization;
+
+import com.yahoo.document.Document;
+import com.yahoo.document.DocumentId;
+import com.yahoo.document.DocumentType;
+
+/**
+ * @author <a href="mailto:ravishar@yahoo-inc.com">ravishar</a>
+ */
+public interface DocumentWriter extends FieldWriter {
+    /**
+     * write out a document
+     *
+     * @param document
+     *            document to be written
+     */
+    void write(Document document);
+
+    void write(DocumentId id);
+
+    void write(DocumentType type);
+}
diff --git a/document/src/main/java/com/yahoo/document/serialization/FieldReader.java b/document/src/main/java/com/yahoo/document/serialization/FieldReader.java
new file mode 100644
index 00000000000..d66cf8bb8e1
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/serialization/FieldReader.java
@@ -0,0 +1,161 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ *
+ */
+package com.yahoo.document.serialization;
+
+import com.yahoo.document.Document;
+import com.yahoo.document.annotation.AnnotationReference;
+import com.yahoo.document.datatypes.*;
+import com.yahoo.vespa.objects.Deserializer;
+import com.yahoo.vespa.objects.FieldBase;
+
+
+/**
+ * @author <a href="mailto:ravishar@yahoo-inc.com">ravishar</a>
+ *
+ */
+public interface FieldReader extends Deserializer {
+
+    /**
+     * Read in the value of field
+     *
+     * @param field - field description (name and data type)
+     * @param value - field value
+     */
+    void read(FieldBase field, Document value);
+    /**
+     * Read in the value of field
+     *
+     * @param field - field description (name and data type)
+     * @param value - field value
+     */
+    void read(FieldBase field, FieldValue value);
+
+    /**
+     * Read in the value of array field
+     *
+     * @param field - field description (name and data type)
+     * @param value - field value
+     */
+    <T extends FieldValue> void read(FieldBase field, Array<T> value);
+
+    /**
+     * Read the value of a map field
+     */
+    <K extends FieldValue, V extends FieldValue> void read(FieldBase field, MapFieldValue<K, V> map);
+
+    /**
+     * Read in the value of byte field
+     *
+     * @param field - field description (name and data type)
+     * @param value - field value
+     */
+    void read(FieldBase field, ByteFieldValue value);
+
+    /**
+     * Read in the value of collection field
+     *
+     * @param field - field description (name and data type)
+     * @param value - field value
+     */
+    <T extends FieldValue> void read(FieldBase field, CollectionFieldValue<T> value);
+
+    /**
+     * Read in the value of double field
+     *
+     * @param field - field description (name and data type)
+     * @param value - field value
+     */
+    void read(FieldBase field, DoubleFieldValue value);
+
+    /**
+     * Read in the value of float field
+     *
+     * @param field - field description (name and data type)
+     * @param value - field value
+     */
+    void read(FieldBase field, FloatFieldValue value);
+
+    /**
+     * Read in the value of integer field
+     *
+     * @param field - field description (name and data type)
+     * @param value - field value
+     */
+    void read(FieldBase field, IntegerFieldValue value);
+
+    /**
+     * Read in the value of long field
+     *
+     * @param field - field description (name and data type)
+     * @param value - field value
+     */
+    void read(FieldBase field, LongFieldValue value);
+
+    /**
+     * Read in the value of raw field
+     *
+     * @param field - field description (name and data type)
+     * @param value - field value
+     */
+    void read(FieldBase field, Raw value);
+
+    /**
+     * Read in the value of predicate field
+     *
+     * @param field - field description (name and data type)
+     * @param value - field value
+     */
+    void read(FieldBase field, PredicateFieldValue value);
+
+    /**
+     * Read in the value of string field
+     *
+     * @param field - field description (name and data type)
+     * @param value - field value
+     */
+    void read(FieldBase field, StringFieldValue value);
+
+    /**
+     * Read in the value of the given tensor field.
+     *
+     * @param field field description (name and data type)
+     * @param value tensor field value
+     */
+    void read(FieldBase field, TensorFieldValue value);
+
+    /**
+     * Read in the value of struct field
+     *
+     * @param field - field description (name and data type)
+     * @param value - field value
+     */
+    void read(FieldBase field, Struct value);
+
+    /**
+     * Read in the value of structured field
+     *
+     * @param field - field description (name and data type)
+     * @param value - field value
+     */
+    void read(FieldBase field, StructuredFieldValue value);
+
+
+    /**
+     * Read in the value of weighted set field
+     *
+     * @param field - field description (name and data type)
+     * @param value - field value
+     */
+    <T extends FieldValue> void read(FieldBase field, WeightedSet<T> value);
+
+    /**
+     * Read in the value of annotation reference.
+     *
+     * @param field - field description (name and data type)
+     * @param value - field value
+     */
+    void read(FieldBase field, AnnotationReference value);
+
+}
diff --git a/document/src/main/java/com/yahoo/document/serialization/FieldWriter.java b/document/src/main/java/com/yahoo/document/serialization/FieldWriter.java
new file mode 100644
index 00000000000..4a269a704d2
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/serialization/FieldWriter.java
@@ -0,0 +1,193 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.serialization;
+
+import com.yahoo.document.Document;
+import com.yahoo.document.annotation.AnnotationReference;
+import com.yahoo.document.datatypes.*;
+import com.yahoo.vespa.objects.FieldBase;
+import com.yahoo.vespa.objects.Serializer;
+
+/**
+ * Interface for writing out com.yahoo.document.datatypes.FieldValue.
+ *
+ * @author <a href="mailto:ravishar@yahoo-inc.com">ravishar</a>
+ *
+ */
+public interface FieldWriter extends Serializer {
+
+    /**
+     * Write out the value of field
+     *
+     * @param field
+     *            field description (name and data type)
+     * @param value
+     *            field value
+     */
+    void write(FieldBase field, FieldValue value);
+
+    /**
+     * Write out the value of field
+     *
+     * @param field
+     *            field description (name and data type)
+     * @param value
+     *            field value
+     */
+    public void write(FieldBase field, Document value);
+
+    /**
+     * Write out the value of array field
+     *
+     * @param field
+     *            field description (name and data type)
+     * @param value
+     *            field value
+     */
+    <T extends FieldValue> void write(FieldBase field, Array<T> value);
+
+    /**
+     * Write the value of a map field
+     */
+    <K extends FieldValue, V extends FieldValue> void write(FieldBase field,
+            MapFieldValue<K, V> map);
+
+    /**
+     * Write out the value of byte field
+     *
+     * @param field
+     *            field description (name and data type)
+     * @param value
+     *            field value
+     */
+    void write(FieldBase field, ByteFieldValue value);
+
+    /**
+     * Write out the value of collection field
+     *
+     * @param field
+     *            field description (name and data type)
+     * @param value
+     *            field value
+     */
+    <T extends FieldValue> void write(FieldBase field,
+            CollectionFieldValue<T> value);
+
+    /**
+     * Write out the value of double field
+     *
+     * @param field
+     *            field description (name and data type)
+     * @param value
+     *            field value
+     */
+    void write(FieldBase field, DoubleFieldValue value);
+
+    /**
+     * Write out the value of float field
+     *
+     * @param field
+     *            field description (name and data type)
+     * @param value
+     *            field value
+     */
+    void write(FieldBase field, FloatFieldValue value);
+
+    /**
+     * Write out the value of integer field
+     *
+     * @param field
+     *            field description (name and data type)
+     * @param value
+     *            field value
+     */
+    void write(FieldBase field, IntegerFieldValue value);
+
+    /**
+     * Write out the value of long field
+     *
+     * @param field
+     *            field description (name and data type)
+     * @param value
+     *            field value
+     */
+    void write(FieldBase field, LongFieldValue value);
+
+    /**
+     * Write out the value of raw field
+     *
+     * @param field
+     *            field description (name and data type)
+     * @param value
+     *            field value
+     */
+    void write(FieldBase field, Raw value);
+
+    /**
+     * Write out the value of predicate field
+     *
+     * @param field
+     *            field description (name and data type)
+     * @param value
+     *            field value
+     */
+    void write(FieldBase field, PredicateFieldValue value);
+
+    /**
+     * Write out the value of string field
+     *
+     * @param field
+     *            field description (name and data type)
+     * @param value
+     *            field value
+     */
+    void write(FieldBase field, StringFieldValue value);
+
+    /**
+     * Write out the value of the given tensor field value.
+     *
+     * @param field field description (name and data type)
+     * @param value tensor field value
+     */
+    void write(FieldBase field, TensorFieldValue value);
+
+    /**
+     * Write out the value of struct field
+     *
+     * @param field
+     *            field description (name and data type)
+     * @param value
+     *            field value
+     */
+    void write(FieldBase field, Struct value);
+
+    /**
+     * Write out the value of structured field
+     *
+     * @param field
+     *            field description (name and data type)
+     * @param value
+     *            field value
+     */
+    void write(FieldBase field, StructuredFieldValue value);
+
+    /**
+     * Write out the value of weighted set field
+     *
+     * @param field
+     *            field description (name and data type)
+     * @param value
+     *            field value
+     */
+    <T extends FieldValue> void write(FieldBase field, WeightedSet<T> value);
+
+    /**
+     * Write out the value of annotation data.
+     *
+     * @param field
+     *            field description (name and data type)
+     * @param value
+     *            field value
+     */
+    void write(FieldBase field, AnnotationReference value);
+
+}
diff --git a/document/src/main/java/com/yahoo/document/serialization/SerializationException.java b/document/src/main/java/com/yahoo/document/serialization/SerializationException.java
new file mode 100644
index 00000000000..44dd9d2ccf8
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/serialization/SerializationException.java
@@ -0,0 +1,21 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.serialization;
+
+/**
+ * Exception which is thrown when serialization fails.
+ *
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class SerializationException extends RuntimeException {
+    public SerializationException(String msg) {
+        super(msg);
+    }
+
+    public SerializationException(String msg, Throwable cause) {
+        super(msg, cause);
+    }
+
+    public SerializationException(Throwable cause) {
+        super(cause);
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/serialization/SpanNodeReader.java b/document/src/main/java/com/yahoo/document/serialization/SpanNodeReader.java
new file mode 100644
index 00000000000..652758e8e38
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/serialization/SpanNodeReader.java
@@ -0,0 +1,16 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.serialization;
+
+import com.yahoo.document.annotation.AlternateSpanList;
+import com.yahoo.document.annotation.Span;
+import com.yahoo.document.annotation.SpanList;
+import com.yahoo.document.annotation.SpanNode;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public interface SpanNodeReader {
+    public void read(Span span);
+    public void read(SpanList spanList);
+    public void read(AlternateSpanList altSpanList);
+}
diff --git a/document/src/main/java/com/yahoo/document/serialization/SpanNodeWriter.java b/document/src/main/java/com/yahoo/document/serialization/SpanNodeWriter.java
new file mode 100644
index 00000000000..8712c792f99
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/serialization/SpanNodeWriter.java
@@ -0,0 +1,18 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.serialization;
+
+import com.yahoo.document.annotation.AlternateSpanList;
+import com.yahoo.document.annotation.Span;
+import com.yahoo.document.annotation.SpanList;
+import com.yahoo.document.annotation.SpanNode;
+import com.yahoo.vespa.objects.Serializer;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public interface SpanNodeWriter extends Serializer {
+    public void write(SpanNode spanNode);
+    public void write(Span span);
+    public void write(SpanList spanList);
+    public void write(AlternateSpanList altSpanList);
+}
diff --git a/document/src/main/java/com/yahoo/document/serialization/SpanTreeReader.java b/document/src/main/java/com/yahoo/document/serialization/SpanTreeReader.java
new file mode 100644
index 00000000000..ca670f527b5
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/serialization/SpanTreeReader.java
@@ -0,0 +1,11 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.serialization;
+
+import com.yahoo.document.annotation.SpanTree;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public interface SpanTreeReader {
+    public void read(SpanTree tree);
+}
diff --git a/document/src/main/java/com/yahoo/document/serialization/SpanTreeWriter.java b/document/src/main/java/com/yahoo/document/serialization/SpanTreeWriter.java
new file mode 100644
index 00000000000..629a21f149e
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/serialization/SpanTreeWriter.java
@@ -0,0 +1,11 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.serialization;
+
+import com.yahoo.document.annotation.SpanTree;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public interface SpanTreeWriter {
+    public void write(SpanTree tree);
+}
diff --git a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer42.java b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer42.java
new file mode 100644
index 00000000000..c3e51b2602a
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer42.java
@@ -0,0 +1,786 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.serialization;
+
+import com.yahoo.collections.Tuple2;
+import com.yahoo.compress.CompressionType;
+import com.yahoo.compress.Compressor;
+import com.yahoo.document.ArrayDataType;
+import com.yahoo.document.CollectionDataType;
+import com.yahoo.document.DataType;
+import com.yahoo.document.DataTypeName;
+import com.yahoo.document.Document;
+import com.yahoo.document.DocumentId;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.DocumentTypeManager;
+import com.yahoo.document.DocumentUpdate;
+import com.yahoo.document.Field;
+import com.yahoo.document.MapDataType;
+import com.yahoo.document.StructDataType;
+import com.yahoo.document.WeightedSetDataType;
+import com.yahoo.document.annotation.AlternateSpanList;
+import com.yahoo.document.annotation.Annotation;
+import com.yahoo.document.annotation.AnnotationReference;
+import com.yahoo.document.annotation.AnnotationType;
+import com.yahoo.document.annotation.Span;
+import com.yahoo.document.annotation.SpanList;
+import com.yahoo.document.annotation.SpanNode;
+import com.yahoo.document.annotation.SpanNodeParent;
+import com.yahoo.document.annotation.SpanTree;
+import com.yahoo.document.datatypes.Array;
+import com.yahoo.document.datatypes.ByteFieldValue;
+import com.yahoo.document.datatypes.CollectionFieldValue;
+import com.yahoo.document.datatypes.DoubleFieldValue;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.FloatFieldValue;
+import com.yahoo.document.datatypes.IntegerFieldValue;
+import com.yahoo.document.datatypes.LongFieldValue;
+import com.yahoo.document.datatypes.MapFieldValue;
+import com.yahoo.document.datatypes.PredicateFieldValue;
+import com.yahoo.document.datatypes.Raw;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.document.datatypes.Struct;
+import com.yahoo.document.datatypes.StructuredFieldValue;
+import com.yahoo.document.datatypes.TensorFieldValue;
+import com.yahoo.document.datatypes.WeightedSet;
+import com.yahoo.document.fieldpathupdate.AddFieldPathUpdate;
+import com.yahoo.document.fieldpathupdate.AssignFieldPathUpdate;
+import com.yahoo.document.fieldpathupdate.FieldPathUpdate;
+import com.yahoo.document.fieldpathupdate.RemoveFieldPathUpdate;
+import com.yahoo.document.predicate.BinaryFormat;
+import com.yahoo.document.select.parser.ParseException;
+import com.yahoo.document.update.AddValueUpdate;
+import com.yahoo.document.update.ArithmeticValueUpdate;
+import com.yahoo.document.update.AssignValueUpdate;
+import com.yahoo.document.update.ClearValueUpdate;
+import com.yahoo.document.update.FieldUpdate;
+import com.yahoo.document.update.MapValueUpdate;
+import com.yahoo.document.update.RemoveValueUpdate;
+import com.yahoo.document.update.ValueUpdate;
+import com.yahoo.io.GrowableByteBuffer;
+import com.yahoo.tensor.serialization.TypedBinaryFormat;
+import com.yahoo.text.Utf8;
+import com.yahoo.text.Utf8Array;
+import com.yahoo.text.Utf8String;
+import com.yahoo.vespa.objects.FieldBase;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+import static com.yahoo.text.Utf8.calculateStringPositions;
+
+/**
+ * Class used for de-serializing documents on the Vespa 4.2 document format.
+ *
+ * @deprecated Please use {@link com.yahoo.document.serialization.VespaDocumentDeserializerHead} instead for new code.
+ * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a>
+ */
+@Deprecated // OK: Don't remove on Vespa 6: Mail may have documents on this format still
+// When removing: Move content of this class into VespaDocumentDeserializerHead (and subclass VespaDocumentSerializerHead in that)
+public class VespaDocumentDeserializer42 extends VespaDocumentSerializer42 implements DocumentDeserializer {
+
+    private final Compressor compressor = new Compressor();
+    private DocumentTypeManager manager;
+    GrowableByteBuffer body;
+    private short version;
+    private List<SpanNode> spanNodes;
+    private List<Annotation> annotations;
+    private int[] stringPositions;
+
+    VespaDocumentDeserializer42(DocumentTypeManager manager, GrowableByteBuffer header, GrowableByteBuffer body, short version) {
+        super(header);
+        this.manager = manager;
+        this.body = body;
+        this.version = version;
+    }
+
+    VespaDocumentDeserializer42(DocumentTypeManager manager, GrowableByteBuffer buf) {
+        this(manager, buf, null, Document.SERIALIZED_VERSION);
+    }
+
+    VespaDocumentDeserializer42(DocumentTypeManager manager, GrowableByteBuffer buf, GrowableByteBuffer body) {
+        this(manager, buf, body, Document.SERIALIZED_VERSION);
+    }
+
+    final public DocumentTypeManager getDocumentTypeManager() { return manager; }
+
+    public void read(Document document) {
+         read(null, document);
+    }
+    public void read(FieldBase field, Document doc) {
+
+        // Verify that we have correct version
+        version = getShort(null);
+        if (version < 6 || version > Document.SERIALIZED_VERSION) {
+            throw new DeserializationException(
+                    "Unknown version " + version + ", expected " + Document.SERIALIZED_VERSION + ".");
+        }
+
+        int dataLength = 0;
+        int dataPos = 0;
+
+        if (version < 7) {
+            getInt2_4_8Bytes(null); // Total document size.. Ignore
+        } else {
+            dataLength = getInt(null);
+            dataPos = position();
+        }
+
+        doc.setId(readDocumentId());
+
+        Byte content = getByte(null);
+
+        doc.setDataType(readDocumentType());
+
+        if ((content & 0x2) != 0) {
+            doc.getHeader().deserialize(new Field("header"),this);
+        }
+        if ((content & 0x4) != 0) {
+            doc.getBody().deserialize(new Field("body"),this);
+        } else if (body != null) {
+            GrowableByteBuffer header = getBuf();
+            setBuf(body);
+            body = null;
+            doc.getBody().deserialize(new Field("body"), this);
+            body = getBuf();
+            setBuf(header);
+        }
+
+        if (version < 8) {
+            int crcVal = getInt(null);
+        }
+
+        if (version > 6) {
+            if (dataLength != (position() - dataPos)) {
+                throw new DeserializationException("Length mismatch");
+            }
+        }
+    }
+    public void read(FieldBase field, FieldValue value) {
+        throw new IllegalArgumentException("read not implemented yet.");
+    }
+
+    public <T extends FieldValue> void read(FieldBase field, Array<T> array) {
+        int numElements = getNumCollectionElems();
+        ArrayList<T> list = new ArrayList<T>(numElements);
+        ArrayDataType type = array.getDataType();
+        for (int i = 0; i < numElements; i++) {
+            if (version < 7) {
+                getInt(null); // We don't need size for anything
+            }
+            FieldValue fv = type.getNestedType().createFieldValue();
+            fv.deserialize(null, this);
+            list.add((T) fv);
+        }
+        array.clear();
+        array.addAll(list);
+    }
+
+    public <K extends FieldValue, V extends FieldValue> void read(FieldBase field, MapFieldValue<K, V> map) {
+        int numElements = getNumCollectionElems();
+        Map<K,V> hash = new HashMap<>();
+        MapDataType type = map.getDataType();
+        for (int i = 0; i < numElements; i++) {
+            if (version < 7) {
+                getInt(null); // We don't need size for anything
+            }
+            K key = (K) type.getKeyType().createFieldValue();
+            V val = (V) type.getValueType().createFieldValue();
+            key.deserialize(null, this);
+            val.deserialize(null, this);
+            hash.put(key, val);
+        }
+        map.clear();
+        map.putAll(hash);
+    }
+
+    private int getNumCollectionElems() {
+        int numElements;
+        if (version < 7) {
+            getInt(null); // We already know the nested type, so ignore that..
+            numElements = getInt(null);
+        } else {
+            numElements = getInt1_2_4Bytes(null);
+        }
+        if (numElements < 0) {
+            throw new DeserializationException("Bad number of array/map elements, " + numElements);
+        }
+        return numElements;
+    }
+
+    public <T extends FieldValue> void read(FieldBase field, CollectionFieldValue<T> value) {
+        throw new IllegalArgumentException("read not implemented yet.");
+    }
+    public void read(FieldBase field, ByteFieldValue value)    { value.assign(getByte(null)); }
+    public void read(FieldBase field, DoubleFieldValue value)  { value.assign(getDouble(null)); }
+    public void read(FieldBase field, FloatFieldValue value)   { value.assign(getFloat(null)); }
+    public void read(FieldBase field, IntegerFieldValue value) { value.assign(getInt(null)); }
+    public void read(FieldBase field, LongFieldValue value)    { value.assign(getLong(null)); }
+
+    public void read(FieldBase field, Raw value) {
+        int rawsize = getInt(null);
+        byte[] rawBytes = getBytes(null, rawsize);
+        value.assign(rawBytes);
+    }
+
+    @Override
+    public void read(FieldBase field, PredicateFieldValue value) {
+        int len = getInt(null);
+        byte[] buf = getBytes(null, len);
+        value.assign(BinaryFormat.decode(buf));
+    }
+
+    public void read(FieldBase field, StringFieldValue value) {
+        byte coding = getByte(null);
+
+        int length = getInt1_4Bytes(null);
+
+        //OK, it seems that this length includes null termination.
+        //NOTE: the following four lines are basically parseNullTerminatedString() inlined,
+        //but we need to use the UTF-8 buffer below, so not using that method...
+        byte[] stringArray = new byte[length - 1];
+        buf.get(stringArray);
+        buf.get();    //move past 0-termination
+        value.setUnChecked(Utf8.toString(stringArray));
+
+        if ((coding & 64) == 64) {
+            //we have a span tree!
+            try {
+                //we don't support serialization of nested span trees, so this is safe:
+                stringPositions = calculateStringPositions(stringArray);
+                //total length:
+                int size = buf.getInt();
+                int startPos = buf.position();
+
+                int numSpanTrees = buf.getInt1_2_4Bytes();
+
+                for (int i = 0; i < numSpanTrees; i++) {
+                    SpanTree tree = new SpanTree();
+                    StringFieldValue treeName = new StringFieldValue();
+                    treeName.deserialize(this);
+                    tree.setName(treeName.getString());
+                    value.setSpanTree(tree);
+                    readSpanTree(tree, false);
+                }
+
+                buf.position(startPos + size);
+            } finally {
+                stringPositions = null;
+            }
+        }
+    }
+
+    @Override
+    public void read(FieldBase field, TensorFieldValue value) {
+        int encodedTensorLength = buf.getInt1_4Bytes();
+        if (encodedTensorLength > 0) {
+            byte[] encodedTensor = getBytes(null, encodedTensorLength);
+            value.assign(TypedBinaryFormat.decode(encodedTensor));
+        } else {
+            value.clear();
+        }
+    }
+
+    public void read(FieldBase fieldDef, Struct s) {
+        s.setVersion(version);
+        int startPos = position();
+
+        if (version < 6) {
+            throw new DeserializationException("Illegal document serialization version " + version);
+        }
+
+        int dataSize;
+        if (version < 7) {
+            long rSize = getInt2_4_8Bytes(null);
+            //TODO: Look into how to support data segments larger than INT_MAX bytes
+            if (rSize > Integer.MAX_VALUE) {
+                throw new DeserializationException("Raw size of data block is too large.");
+            }
+            dataSize = (int)rSize;
+        } else {
+            dataSize = getInt(null);
+        }
+
+        byte comprCode = getByte(null);
+        CompressionType compression = CompressionType.valueOf(comprCode);
+
+        int uncompressedSize = 0;
+        if (compression != CompressionType.NONE &&
+            compression != CompressionType.INCOMPRESSIBLE)
+        {
+            // uncompressedsize (full size of FIELDS only, after decompression)
+            long pSize = getInt2_4_8Bytes(null);
+            //TODO: Look into how to support data segments larger than INT_MAX bytes
+            if (pSize > Integer.MAX_VALUE) {
+                throw new DeserializationException("Uncompressed size of data block is too large.");
+            }
+            uncompressedSize = (int) pSize;
+        }
+
+        int numberOfFields = getInt1_4Bytes(null);
+
+        List<Tuple2<Integer, Long>> fieldIdsAndLengths = new ArrayList<>(numberOfFields);
+        for (int i=0; i<numberOfFields; ++i) {
+            // id, length (length only used for unknown fields
+            fieldIdsAndLengths.add(new Tuple2<>(getInt1_4Bytes(null), getInt2_4_8Bytes(null)));
+        }
+
+        //save a reference to the big buffer we're reading from:
+        GrowableByteBuffer bigBuf = buf;
+
+        if (version < 7) {
+            // In V6 and earlier, the length included the header.
+            int headerSize = position() - startPos;
+            dataSize -= headerSize;
+        }
+        byte[] destination = compressor.decompress(compression, getBuf().array(), position(), uncompressedSize, Optional.of(dataSize));
+
+        // set position in original buffer to after data
+        position(position() + dataSize);
+
+        // for a while: deserialize from this buffer instead:
+        buf = GrowableByteBuffer.wrap(destination);
+
+        s.clear();
+        StructDataType type = s.getDataType();
+        for (int i=0; i<numberOfFields; ++i) {
+            Field structField = type.getField(fieldIdsAndLengths.get(i).first, version);
+            if (structField == null) {
+                //ignoring unknown field:
+                position(position() + fieldIdsAndLengths.get(i).second.intValue());
+            } else {
+                int posBefore = position();
+                FieldValue value = structField.getDataType().createFieldValue();
+                value.deserialize(structField, this);
+                s.setFieldValue(structField, value);
+                //jump to beginning of next field:
+                position(posBefore + fieldIdsAndLengths.get(i).second.intValue());
+            }
+        }
+
+        // restore the original buffer
+        buf = bigBuf;
+    }
+
+    public void read(FieldBase field, StructuredFieldValue value) {
+        throw new IllegalArgumentException("read not implemented yet.");
+    }
+    public <T extends FieldValue> void read(FieldBase field, WeightedSet<T> ws) {
+        WeightedSetDataType type = ws.getDataType();
+        getInt(null); // Have no need for type
+
+        int numElements = getInt(null);
+        if (numElements < 0) {
+            throw new DeserializationException("Bad number of weighted set elements, " + numElements);
+        }
+
+        ws.clearAndReserve(numElements * 2); // Avoid resizing
+        for (int i = 0; i < numElements; i++) {
+            int size = getInt(null);
+            FieldValue value = type.getNestedType().createFieldValue();
+            value.deserialize(null, this);
+            IntegerFieldValue weight = new IntegerFieldValue(getInt(null));
+            ws.putUnChecked((T) value, weight);
+        }
+
+    }
+
+    public void read(FieldBase field, AnnotationReference value) {
+        int seqId = buf.getInt1_2_4Bytes();
+        try {
+            Annotation a = annotations.get(seqId);
+            value.setReferenceNoCompatibilityCheck(a);
+        } catch (IndexOutOfBoundsException iiobe) {
+            throw new SerializationException("Could not serialize AnnotationReference value, reference not found.", iiobe);
+        }
+    }
+
+    private Utf8String deserializeAttributeString() throws DeserializationException {
+        int length = getByte(null);
+        return new Utf8String(parseNullTerminatedString(length));
+    }
+
+    private Utf8Array parseNullTerminatedString() { return parseNullTerminatedString(getBuf().getByteBuffer()); }
+    private Utf8Array parseNullTerminatedString(int lengthExcludingNull) { return parseNullTerminatedString(getBuf().getByteBuffer(), lengthExcludingNull); }
+
+    static Utf8Array parseNullTerminatedString(ByteBuffer buf, int lengthExcludingNull) throws DeserializationException {
+        Utf8Array utf8 = new Utf8Array(buf, lengthExcludingNull);
+        buf.get();    //move past 0-termination
+        return utf8;
+    }
+
+    static  Utf8Array parseNullTerminatedString(ByteBuffer buf) throws DeserializationException {
+        //search for 0-byte
+        int end = getFirstNullByte(buf);
+
+        if (end == -1) {
+            throw new DeserializationException("Could not locate terminating 0-byte for string");
+        }
+
+        return parseNullTerminatedString(buf, end - buf.position());
+    }
+
+    private static int getFirstNullByte(ByteBuffer buf) {
+        int end = -1;
+        int start = buf.position();
+
+        while (true) {
+            try {
+                byte dataByte = buf.get();
+                if (dataByte == (byte) 0) {
+                    end = buf.position() - 1;
+                    break;
+                }
+            } catch (Exception e) {
+                break;
+            }
+        }
+
+        buf.position(start);
+        return end;
+    }
+
+    public void read(DocumentUpdate update) {
+        short serializationVersion = getShort(null);
+
+        update.setId(new DocumentId(this));
+
+        byte contents = getByte(null);
+
+        if ((contents & 0x1) == 0) {
+            throw new DeserializationException("Cannot deserialize DocumentUpdate without doctype");
+        }
+
+        update.setDocumentType(readDocumentType());
+
+        int size = getInt(null);
+
+        for (int i = 0; i < size; i++) {
+            // TODO: Should use checked method, but doesn't work according to test now.
+            update.addFieldUpdateNoCheck(new FieldUpdate(this, update.getDocumentType(), serializationVersion));
+        }
+    }
+
+    public void read(FieldPathUpdate update) {
+        String fieldPath = getString(null);
+        String whereClause = getString(null);
+        update.setFieldPath(fieldPath);
+
+        try {
+            update.setWhereClause(whereClause);
+        } catch (ParseException e) {
+            throw new DeserializationException(e);
+        }
+    }
+
+    public void read(AssignFieldPathUpdate update) {
+        byte flags = getByte(null);
+        update.setRemoveIfZero((flags & AssignFieldPathUpdate.REMOVE_IF_ZERO) != 0);
+        update.setCreateMissingPath((flags & AssignFieldPathUpdate.CREATE_MISSING_PATH) != 0);
+        if ((flags & AssignFieldPathUpdate.ARITHMETIC_EXPRESSION) != 0) {
+            update.setExpression(getString(null));
+        } else {
+            DataType dt = update.getFieldPath().getResultingDataType();
+            FieldValue fv = dt.createFieldValue();
+            fv.deserialize(this);
+            update.setNewValue(fv);
+        }
+    }
+
+    public void read(RemoveFieldPathUpdate update) {
+
+    }
+
+    public void read(AddFieldPathUpdate update) {
+        DataType dt = update.getFieldPath().getResultingDataType();
+        FieldValue fv = dt.createFieldValue();
+        dt.createFieldValue();
+        fv.deserialize(this);
+
+        if (!(fv instanceof Array)) {
+            throw new DeserializationException("Add only applicable to array types");
+        }
+        update.setNewValues((Array)fv);
+    }
+
+    public ValueUpdate getValueUpdate(DataType superType, DataType subType) {
+        int vuTypeId = getInt(null);
+
+        ValueUpdate.ValueUpdateClassID op = ValueUpdate.ValueUpdateClassID.getID(vuTypeId);
+        if (op == null) {
+            throw new IllegalArgumentException("Read type "+vuTypeId+" of bytebuffer, but this is not a legal value update type.");
+        }
+
+        switch (op) {
+            case ADD:
+            {
+                FieldValue fval = subType.createFieldValue();
+                fval.deserialize(this);
+                int weight = getInt(null);
+                return new AddValueUpdate(fval, weight);
+            }
+            case ARITHMETIC:
+                int opId = getInt(null);
+                ArithmeticValueUpdate.Operator operator = ArithmeticValueUpdate.Operator.getID(opId);
+                double operand = getDouble(null);
+                return new ArithmeticValueUpdate(operator, operand);
+            case ASSIGN:
+            {
+                byte contents = getByte(null);
+                FieldValue fval = null;
+                if (contents == (byte) 1) {
+                    fval = superType.createFieldValue();
+                    fval.deserialize(this);
+                }
+                return new AssignValueUpdate(fval);
+            }
+            case CLEAR:
+                return new ClearValueUpdate();
+            case MAP:
+                if (superType instanceof ArrayDataType) {
+                    CollectionDataType type = (CollectionDataType) superType;
+                    IntegerFieldValue index = new IntegerFieldValue();
+                    index.deserialize(this);
+                    ValueUpdate update = getValueUpdate(type.getNestedType(), null);
+                    return new MapValueUpdate(index, update);
+                } else if (superType instanceof WeightedSetDataType) {
+                    CollectionDataType type = (CollectionDataType) superType;
+                    FieldValue fval = type.getNestedType().createFieldValue();
+                    fval.deserialize(this);
+                    ValueUpdate update = getValueUpdate(DataType.INT, null);
+                    return new MapValueUpdate(fval, update);
+                } else {
+                    throw new DeserializationException("MapValueUpdate only works for arrays and weighted sets");
+                }
+            case REMOVE:
+                FieldValue fval = ((CollectionDataType) superType).getNestedType().createFieldValue();
+                fval.deserialize(this);
+                return new RemoveValueUpdate(fval);
+            default:
+                throw new DeserializationException(
+                        "Could not deserialize ValueUpdate, unknown valueUpdateClassID type " + vuTypeId);
+        }
+    }
+
+    public void read(FieldUpdate fieldUpdate) {
+        int fieldId = getInt(null);
+        Field field = fieldUpdate.getDocumentType().getField(fieldId, fieldUpdate.getSerializationVersion());
+        if (field == null) {
+            throw new DeserializationException(
+                    "Cannot deserialize FieldUpdate, field fieldId " + fieldId + " not found in " + fieldUpdate.getDocumentType());
+        }
+
+        fieldUpdate.setField(field);
+        int size = getInt(null);
+
+        for (int i = 0; i < size; i++) {
+            if (field.getDataType() instanceof CollectionDataType) {
+                CollectionDataType collType = (CollectionDataType) field.getDataType();
+                fieldUpdate.addValueUpdate(getValueUpdate(collType, collType.getNestedType()));
+            } else {
+                fieldUpdate.addValueUpdate(getValueUpdate(field.getDataType(), null));
+            }
+        }
+    }
+
+    public DocumentId readDocumentId() {
+        Utf8String uri = new Utf8String(parseNullTerminatedString(getBuf().getByteBuffer()));
+        return new DocumentId(uri.toString());
+    }
+
+    public DocumentType readDocumentType() {
+        Utf8Array docTypeName = parseNullTerminatedString();
+        int ignored = getShort(null); // used to hold the version
+
+        DocumentType docType = manager.getDocumentType(new DataTypeName(docTypeName));
+        if (docType == null) {
+            throw new DeserializationException(
+                    "No known document type with name " + new Utf8String(docTypeName).toString());
+        }
+        return docType;
+    }
+
+    private SpanNode readSpanNode() {
+        byte type = buf.get();
+        buf.position(buf.position() - 1);
+
+        SpanNode retval;
+        if ((type & Span.ID) == Span.ID) {
+            retval = new Span();
+            if (spanNodes != null) {
+                spanNodes.add(retval);
+            }
+            read((Span) retval);
+        } else if ((type & SpanList.ID) == SpanList.ID) {
+            retval = new SpanList();
+            if (spanNodes != null) {
+                spanNodes.add(retval);
+            }
+            read((SpanList) retval);
+        } else if ((type & AlternateSpanList.ID) == AlternateSpanList.ID) {
+            retval = new AlternateSpanList();
+            if (spanNodes != null) {
+                spanNodes.add(retval);
+            }
+            read((AlternateSpanList) retval);
+        } else {
+            throw new DeserializationException("Cannot read SpanNode of type " + type);
+        }
+        return retval;
+    }
+
+    private void readSpanTree(SpanTree tree, boolean readName) {
+        //we don't support serialization of nested span trees:
+        if (spanNodes != null || annotations != null) {
+            throw new SerializationException("Deserialization of nested SpanTrees is not supported.");
+        }
+
+        //we're going to write a new SpanTree, create a new Map for nodes:
+        spanNodes = new ArrayList<SpanNode>();
+        annotations = new ArrayList<Annotation>();
+
+        try {
+            if (readName) {
+                StringFieldValue treeName = new StringFieldValue();
+                treeName.deserialize(this);
+                tree.setName(treeName.getString());
+            }
+
+            SpanNode root = readSpanNode();
+            tree.setRoot(root);
+
+            int numAnnotations = buf.getInt1_2_4Bytes();
+
+            for (int i = 0; i < numAnnotations; i++) {
+                Annotation a = new Annotation();
+                annotations.add(a);
+            }
+            for (int i = 0; i < numAnnotations; i++) {
+                read(annotations.get(i));
+            }
+            for (Annotation a : annotations) {
+                tree.annotate(a);
+            }
+
+            for (SpanNode node: spanNodes) {
+                if (node instanceof Span) {
+                    correctIndexes((Span) node);
+                }
+            }
+        } finally {
+            //we're done, let's set this to null to save memory and prevent madness:
+            spanNodes = null;
+            annotations = null;
+        }
+    }
+
+    public void read(SpanTree tree) {
+        readSpanTree(tree, true);
+    }
+
+    public void read(Annotation annotation) {
+        int annotationTypeId = buf.getInt();
+        AnnotationType type = manager.getAnnotationTypeRegistry().getType(annotationTypeId);
+
+        if (type == null) {
+            throw new DeserializationException("Cannot deserialize annotation of type " + annotationTypeId + " (unknown type)");
+        }
+
+        annotation.setType(type);
+
+        byte features = buf.get();
+        int length = buf.getInt1_2_4Bytes();
+
+        if ((features & (byte) 1) == (byte) 1) {
+            //we have a span node
+            int spanNodeId = buf.getInt1_2_4Bytes();
+            try {
+                SpanNode node = spanNodes.get(spanNodeId);
+                annotation.setSpanNode(node);
+            } catch (IndexOutOfBoundsException ioobe) {
+                throw new DeserializationException("Could not deserialize annotation, associated span node not found ", ioobe);
+            }
+        }
+        if ((features & (byte) 2) == (byte) 2) {
+            //we have a value:
+            int dataTypeId = buf.getInt();
+
+            //if this data type ID the same as the one in our config?
+            if (dataTypeId != type.getDataType().getId()) {
+                //not the same, but we will handle it gracefully, and just skip past the data:
+                buf.position(buf.position() + length - 4);
+            } else {
+                FieldValue value = type.getDataType().createFieldValue();
+                value.deserialize(this);
+                annotation.setFieldValue(value);
+            }
+        }
+    }
+
+    public void read(Span span) {
+        byte type = buf.get();
+        if ((type & Span.ID) != Span.ID) {
+            throw new DeserializationException("Cannot deserialize Span with type " + type);
+        }
+        span.setFrom(buf.getInt1_2_4Bytes());
+        span.setLength(buf.getInt1_2_4Bytes());
+    }
+
+    private void correctIndexes(Span span) {
+        if (stringPositions == null) {
+            throw new DeserializationException("Cannot deserialize Span, no access to parent StringFieldValue.");
+        }
+        int fromIndex = stringPositions[span.getFrom()];
+        int toIndex = stringPositions[span.getTo()];
+        int length = toIndex - fromIndex;
+
+        span.setFrom(fromIndex);
+        span.setLength(length);
+    }
+
+    public void read(SpanList spanList) {
+        byte type = buf.get();
+        if ((type & SpanList.ID) != SpanList.ID) {
+            throw new DeserializationException("Cannot deserialize SpanList with type " + type);
+        }
+        List<SpanNode> nodes = readSpanList(spanList);
+        for (SpanNode node : nodes) {
+            spanList.add(node);
+        }
+    }
+
+    public void read(AlternateSpanList altSpanList) {
+        byte type = buf.get();
+        if ((type & AlternateSpanList.ID) != AlternateSpanList.ID) {
+            throw new DeserializationException("Cannot deserialize AlternateSpanList with type " + type);
+        }
+        int numSubTrees = buf.getInt1_2_4Bytes();
+
+        for (int i = 0; i < numSubTrees; i++) {
+            double prob = buf.getDouble();
+            List<SpanNode> list = readSpanList(altSpanList);
+
+            if (i == 0) {
+                for (SpanNode node : list) {
+                    altSpanList.add(node);
+                }
+                altSpanList.setProbability(0, prob);
+            } else {
+                altSpanList.addChildren(i, list, prob);
+            }
+        }
+    }
+
+    private List<SpanNode> readSpanList(SpanNodeParent parent) {
+        int size = buf.getInt1_2_4Bytes();
+        List<SpanNode> spanList = new ArrayList<SpanNode>();
+        for (int i = 0; i < size; i++) {
+            spanList.add(readSpanNode());
+        }
+        return spanList;
+    }
+
+}
diff --git a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializerHead.java b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializerHead.java
new file mode 100644
index 00000000000..927075a25b5
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializerHead.java
@@ -0,0 +1,50 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.serialization;
+
+import com.yahoo.document.DocumentId;
+import com.yahoo.document.DocumentTypeManager;
+import com.yahoo.document.DocumentUpdate;
+import com.yahoo.document.fieldpathupdate.FieldPathUpdate;
+import com.yahoo.document.select.parser.ParseException;
+import com.yahoo.document.update.FieldUpdate;
+import com.yahoo.io.GrowableByteBuffer;
+
+/**
+ * Class used for de-serializing documents on the current head document format.
+ *
+ * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a>
+ */
+public class VespaDocumentDeserializerHead extends VespaDocumentDeserializer42 {
+
+    public VespaDocumentDeserializerHead(DocumentTypeManager manager, GrowableByteBuffer buffer) {
+        super(manager, buffer);
+    }
+
+    @Override
+    public void read(DocumentUpdate update) {
+        update.setId(new DocumentId(this));
+        update.setDocumentType(readDocumentType());
+
+        int size = getInt(null);
+
+        for (int i = 0; i < size; i++) {
+            // TODO: Should use checked method, but doesn't work according to test now.
+            update.addFieldUpdateNoCheck(new FieldUpdate(this, update.getDocumentType(), 8));
+        }
+
+        try {
+            int sizeAndFlags = getInt(null);
+            update.setCreateIfNonExistent(DocumentUpdateFlags.extractFlags(sizeAndFlags).getCreateIfNonExistent());
+            size = DocumentUpdateFlags.extractValue(sizeAndFlags);
+
+            for (int i = 0; i < size; i++) {
+                int type = getByte(null);
+                update.addFieldPathUpdate(FieldPathUpdate.create(FieldPathUpdate.Type.valueOf(type),
+                        update.getDocumentType(), this));
+            }
+        } catch (ParseException e) {
+            throw new DeserializationException(e);
+        }
+    }
+
+}
diff --git a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer42.java b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer42.java
new file mode 100644
index 00000000000..64aea73d000
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer42.java
@@ -0,0 +1,644 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.serialization;
+
+import com.yahoo.compress.Compressor;
+import com.yahoo.document.*;
+import com.yahoo.document.annotation.*;
+import com.yahoo.document.datatypes.*;
+import com.yahoo.document.predicate.BinaryFormat;
+import com.yahoo.document.update.*;
+import com.yahoo.io.GrowableByteBuffer;
+import com.yahoo.tensor.serialization.TypedBinaryFormat;
+import com.yahoo.vespa.objects.BufferSerializer;
+import com.yahoo.vespa.objects.FieldBase;
+
+import java.nio.ByteBuffer;
+import java.util.*;
+import java.util.logging.Logger;
+
+import static com.yahoo.text.Utf8.calculateBytePositions;
+
+/**
+ * Class used for serializing documents on the Vespa 4.2 document format.
+ *
+ * @deprecated Please use {@link com.yahoo.document.serialization.VespaDocumentSerializerHead} instead for new code.
+ * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a>
+ */
+@Deprecated // OK: Don't remove on Vespa 6: Mail may have documents on this format still
+// When removing: Move content into VespaDocumentSerializerHead
+public class VespaDocumentSerializer42 extends BufferSerializer implements DocumentSerializer {
+
+    private final Compressor compressor = new Compressor();
+    private final static Logger log = Logger.getLogger(VespaDocumentSerializer42.class.getName());
+    private boolean headerOnly;
+    private int spanNodeCounter = -1;
+    private int[] bytePositions;
+
+    VespaDocumentSerializer42(GrowableByteBuffer buf) {
+        super(buf);
+    }
+
+    VespaDocumentSerializer42(ByteBuffer buf) {
+        super(buf);
+    }
+
+    VespaDocumentSerializer42(byte[] buf) {
+        super(buf);
+    }
+
+    VespaDocumentSerializer42() {
+        super();
+    }
+
+    VespaDocumentSerializer42(GrowableByteBuffer buf, boolean headerOnly) {
+        this(buf);
+        this.headerOnly = headerOnly;
+    }
+
+    public void setHeaderOnly(boolean headerOnly) {
+        this.headerOnly = headerOnly;
+    }
+
+    public void write(Document doc) {
+        write(new Field(doc.getDataType().getName(), 0, doc.getDataType(), true), doc);
+    }
+
+    public void write(FieldBase field, Document doc) {
+        //save the starting position in the buffer
+        int startPos = buf.position();
+
+        buf.putShort(Document.SERIALIZED_VERSION);
+
+        // Temporary length, fill in after serialization is done.
+        buf.putInt(0);
+
+        doc.getId().serialize(this);
+
+        byte contents = 0x01; // Indicating we have document type which we always have
+        if (doc.getHeader().getFieldCount() > 0) {
+            contents |= 0x2; // Indicate we have header
+        }
+        if (!headerOnly && doc.getBody().getFieldCount() > 0) {
+            contents |= 0x4; // Indicate we have a body
+        }
+        buf.put(contents);
+
+        doc.getDataType().serialize(this);
+
+        if (doc.getHeader().getFieldCount() > 0) {
+            doc.getHeader().serialize(doc.getDataType().getField("header"), this);
+        }
+
+        if (!headerOnly && doc.getBody().getFieldCount() > 0) {
+            doc.getBody().serialize(doc.getDataType().getField("body"), this);
+        }
+
+        int finalPos = buf.position();
+
+        buf.position(startPos + 2);
+        buf.putInt(finalPos - startPos - 2 - 4); // Don't include the length itself or the version
+        buf.position(finalPos);
+
+    }
+
+    /**
+     * Write out the value of field
+     *
+     * @param field - field description (name and data type)
+     * @param value - field value
+     */
+    public void write(FieldBase field, FieldValue value) {
+        throw new IllegalArgumentException("Not Implemented");
+    }
+
+    /**
+     * Write out the value of array field
+     *
+     * @param field - field description (name and data type)
+     * @param array - field value
+     */
+    public <T extends FieldValue> void write(FieldBase field, Array<T> array) {
+        buf.putInt1_2_4Bytes(array.size());
+
+        List<T> lst = array.getValues();
+        for (FieldValue value : lst) {
+            value.serialize(this);
+        }
+
+    }
+
+    public <K extends FieldValue, V extends FieldValue> void write(FieldBase field, MapFieldValue<K, V> map) {
+        buf.putInt1_2_4Bytes(map.size());
+        for (Map.Entry<K, V> e : map.entrySet()) {
+            e.getKey().serialize(this);
+            e.getValue().serialize(this);
+        }
+    }
+
+    /**
+     * Write out the value of byte field
+     *
+     * @param field - field description (name and data type)
+     * @param value - field value
+     */
+    public void write(FieldBase field, ByteFieldValue value) {
+        buf.put(value.getByte());
+    }
+
+    /**
+     * Write out the value of collection field
+     *
+     * @param field - field description (name and data type)
+     * @param value - field value
+     */
+    public <T extends FieldValue> void write(FieldBase field, CollectionFieldValue<T> value) {
+        throw new IllegalArgumentException("Not Implemented");
+    }
+
+    /**
+     * Write out the value of double field
+     *
+     * @param field - field description (name and data type)
+     * @param value - field value
+     */
+    public void write(FieldBase field, DoubleFieldValue value) {
+        buf.putDouble(value.getDouble());
+    }
+
+    /**
+     * Write out the value of float field
+     *
+     * @param field - field description (name and data type)
+     * @param value - field value
+     */
+    public void write(FieldBase field, FloatFieldValue value) {
+        buf.putFloat(value.getFloat());
+    }
+
+    /**
+     * Write out the value of integer field
+     *
+     * @param field - field description (name and data type)
+     * @param value - field value
+     */
+    public void write(FieldBase field, IntegerFieldValue value) {
+        buf.putInt(value.getInteger());
+    }
+
+    /**
+     * Write out the value of long field
+     *
+     * @param field - field description (name and data type)
+     * @param value - field value
+     */
+    public void write(FieldBase field, LongFieldValue value) {
+        buf.putLong(value.getLong());
+    }
+
+    /**
+     * Write out the value of raw field
+     *
+     * @param field - field description (name and data type)
+     * @param value - field value
+     */
+    public void write(FieldBase field, Raw value) {
+        ByteBuffer rawBuf = value.getByteBuffer();
+        int origPos = rawBuf.position();
+        buf.putInt(rawBuf.remaining());
+        buf.put(rawBuf);
+        rawBuf.position(origPos);
+
+    }
+
+    @Override
+    public void write(FieldBase field, PredicateFieldValue value) {
+        byte[] buf = BinaryFormat.encode(value.getPredicate());
+        this.buf.putInt(buf.length);
+        this.buf.put(buf);
+    }
+
+    /**
+     * Write out the value of string field
+     *
+     * @param field - field description (name and data type)
+     * @param value - field value
+     */
+    public void write(FieldBase field, StringFieldValue value) {
+        byte[] stringBytes = createUTF8CharArray(value.getString());
+
+        byte coding = 0;
+        //Use bit 6 of "coding" to say whether span tree is available or not
+        if (!value.getSpanTrees().isEmpty()) {
+            coding |= 64;
+        }
+        buf.put(coding);
+        buf.putInt1_4Bytes(stringBytes.length + 1);
+
+        buf.put(stringBytes);
+        buf.put(((byte) 0));
+
+        Map<String, SpanTree> trees = value.getSpanTreeMap();
+        if ((trees != null) && !trees.isEmpty()) {
+            try {
+                //we don't support serialization of nested span trees, so this is safe:
+                bytePositions = calculateBytePositions(value.getString());
+                //total length. record position and go back here if necessary:
+                int posBeforeSize = buf.position();
+                buf.putInt(0);
+                buf.putInt1_2_4Bytes(trees.size());
+
+                for (SpanTree tree : trees.values()) {
+                    try {
+                        write(tree);
+                    } catch (SerializationException e) {
+                        throw e;
+                    } catch (RuntimeException e) {
+                        throw new SerializationException("Exception thrown while serializing span tree '" +
+                                                         tree.getName() + "'; string='" + value.getString() + "'", e);
+                    }
+                }
+                int endPos = buf.position();
+                buf.position(posBeforeSize);
+                buf.putInt(endPos - posBeforeSize - 4); //length shall exclude itself
+                buf.position(endPos);
+            } finally {
+                bytePositions = null;
+            }
+        }
+    }
+
+    @Override
+    public void write(FieldBase field, TensorFieldValue value) {
+        if (value.getTensor().isPresent()) {
+            byte[] encodedTensor = TypedBinaryFormat.encode(value.getTensor().get());
+            buf.putInt1_4Bytes(encodedTensor.length);
+            buf.put(encodedTensor);
+        } else {
+            buf.putInt1_4Bytes(0);
+        }
+    }
+
+    /**
+     * Write out the value of struct field
+     *
+     * @param field - field description (name and data type)
+     * @param s     - field value
+     */
+    public void write(FieldBase field, Struct s) {
+        // Serialize all parts first.. As we need to know length before starting
+        // Serialize all the fields.
+
+        //keep the buffer we're serializing everything into:
+        GrowableByteBuffer bigBuffer = buf;
+
+        //create a new buffer and serialize into that for a while:
+        GrowableByteBuffer buffer = new GrowableByteBuffer(4096, 2.0f);
+        buf = buffer;
+
+        List<Integer> fieldIds = new LinkedList<>();
+        List<java.lang.Integer> fieldLengths = new LinkedList<>();
+
+        for (Map.Entry<Field, FieldValue> value : s.getFields()) {
+
+            int startPos = buffer.position();
+            value.getValue().serialize(value.getKey(), this);
+
+            fieldLengths.add(buffer.position() - startPos);
+            fieldIds.add(value.getKey().getId(s.getVersion()));
+        }
+
+        // Switch buffers again:
+        buffer.flip();
+        buf = bigBuffer;
+
+        int uncompressedSize = buffer.remaining();
+        Compressor.Compression compression =
+            s.getDataType().getCompressor().compress(buffer.getByteBuffer().array(), buffer.remaining());
+
+        // Actual serialization starts here.
+        int lenPos = buf.position();
+        putInt(null, 0); // Move back to this after compression is done.
+        buf.put(compression.type().getCode());
+
+        if (compression.data() != null && compression.type().isCompressed()) {
+            buf.putInt2_4_8Bytes(uncompressedSize);
+        }
+
+        buf.putInt1_4Bytes(s.getFieldCount());
+
+        for (int i = 0; i < s.getFieldCount(); ++i) {
+            putInt1_4Bytes(null, fieldIds.get(i));
+            putInt2_4_8Bytes(null, fieldLengths.get(i));
+        }
+
+        int pos = buf.position();
+        if (compression.data() != null) {
+            put(null, compression.data());
+        } else {
+            put(null, buffer.getByteBuffer());
+        }
+        int dataLength = buf.position() - pos;
+
+        int posNow = buf.position();
+        buf.position(lenPos);
+        putInt(null, dataLength);
+        buf.position(posNow);
+    }
+
+    /**
+     * Write out the value of structured field
+     *
+     * @param field - field description (name and data type)
+     * @param value - field value
+     */
+    public void write(FieldBase field, StructuredFieldValue value) {
+        throw new IllegalArgumentException("Not Implemented");
+    }
+
+    /**
+     * Write out the value of weighted set field
+     *
+     * @param field - field description (name and data type)
+     * @param ws    - field value
+     */
+    public <T extends FieldValue> void write(FieldBase field, WeightedSet<T> ws) {
+        WeightedSetDataType type = ws.getDataType();
+        putInt(null, type.getNestedType().getId());
+        putInt(null, ws.size());
+
+        Iterator<T> it = ws.fieldValueIterator();
+        while (it.hasNext()) {
+            FieldValue key = it.next();
+            java.lang.Integer value = ws.get(key);
+            int sizePos = buf.position();
+            putInt(null, 0);
+            int startPos = buf.position();
+            key.serialize(this);
+            putInt(null, value);
+            int finalPos = buf.position();
+            int size = finalPos - startPos;
+            buf.position(sizePos);
+            putInt(null, size);
+            buf.position(finalPos);
+        }
+
+    }
+
+    public void write(FieldBase field, AnnotationReference value) {
+        int annotationId = value.getReference().getScratchId();
+        if (annotationId >= 0) {
+            buf.putInt1_2_4Bytes(annotationId);
+        } else {
+            throw new SerializationException("Could not serialize AnnotationReference value, reference not found (" + value + ")");
+        }
+    }
+
+    public void write(DocumentId id) {
+        put(null, id.getScheme().toUtf8().getBytes());
+        putByte(null, (byte) 0);
+    }
+
+    public void write(DocumentType type) {
+        byte[] docType = createUTF8CharArray(type.getName());
+        put(null, docType);
+        putByte(null, ((byte) 0));
+        putShort(null, (short) 0); // Used to hold the version. Is now always 0.
+    }
+
+
+    private static void serializeAttributeString(GrowableByteBuffer data, String input) {
+        byte[] inputBytes = createUTF8CharArray(input);
+        data.put((byte) (inputBytes.length));
+        data.put(inputBytes);
+        data.put((byte) 0);
+    }
+
+    public void write(Annotation annotation) {
+        buf.putInt(annotation.getType().getId());  //name hash
+
+        byte features = 0;
+        if (annotation.isSpanNodeValid()) {
+            features |= ((byte) 1);
+        }
+        if (annotation.hasFieldValue()) {
+            features |= ((byte) 2);
+        }
+        buf.put(features);
+
+        int posBeforeSize = buf.position();
+        buf.putInt1_2_4BytesAs4(0);
+
+        //write ID of span node:
+        if (annotation.isSpanNodeValid()) {
+            int spanNodeId = annotation.getSpanNode().getScratchId();
+            if (spanNodeId >= 0) {
+                buf.putInt1_2_4Bytes(spanNodeId);
+            } else {
+                throw new SerializationException("Could not serialize annotation, associated SpanNode not found (" + annotation + ")");
+            }
+        }
+
+        //write annotation value:
+        if (annotation.hasFieldValue()) {
+            buf.putInt(annotation.getType().getDataType().getId());
+            annotation.getFieldValue().serialize(this);
+        }
+
+        int end = buf.position();
+        buf.position(posBeforeSize);
+        buf.putInt1_2_4BytesAs4(end - posBeforeSize - 4);
+        buf.position(end);
+    }
+
+    public void write(SpanTree tree) {
+        //we don't support serialization of nested span trees:
+        if (spanNodeCounter >= 0) {
+            throw new SerializationException("Serialization of nested SpanTrees is not supported.");
+        }
+
+        //we're going to write a new SpanTree, create a new Map for nodes:
+        spanNodeCounter = 0;
+
+        //make sure tree is consistent before continuing:
+        tree.cleanup();
+
+        try {
+            new StringFieldValue(tree.getName()).serialize(this);
+
+            write(tree.getRoot());
+            {
+                //add all annotations to temporary list and sort it, to get predictable serialization
+                List<Annotation> tmpAnnotationList = new ArrayList<Annotation>(tree.numAnnotations());
+                for (Annotation annotation : tree) {
+                    tmpAnnotationList.add(annotation);
+                }
+                Collections.sort(tmpAnnotationList);
+
+                int annotationCounter = 0;
+                //add all annotations to map here, in case of back-references:
+                for (Annotation annotation : tmpAnnotationList) {
+                    annotation.setScratchId(annotationCounter++);
+                }
+
+                buf.putInt1_2_4Bytes(tmpAnnotationList.size());
+                for (Annotation annotation : tmpAnnotationList) {
+                    write(annotation);
+                }
+            }
+        } finally {
+            //we're done, let's set these to null to save memory and prevent madness:
+            spanNodeCounter = -1;
+        }
+    }
+
+    public void write(SpanNode spanNode) {
+        if (spanNodeCounter >= 0) {
+            spanNode.setScratchId(spanNodeCounter++);
+        }
+        if (spanNode instanceof Span) {
+            write((Span) spanNode);
+        } else if (spanNode instanceof AlternateSpanList) {
+            write((AlternateSpanList) spanNode);
+        } else if (spanNode instanceof SpanList) {
+            write((SpanList) spanNode);
+        } else {
+            throw new IllegalStateException("BUG!! Unable to serialize " + spanNode);
+        }
+    }
+
+    public void write(Span span) {
+        buf.put(Span.ID);
+
+        if (bytePositions != null) {
+            int byteFrom = bytePositions[span.getFrom()];
+            int byteLength = bytePositions[span.getFrom() + span.getLength()] - byteFrom;
+
+            buf.putInt1_2_4Bytes(byteFrom);
+            buf.putInt1_2_4Bytes(byteLength);
+        } else {
+            throw new SerializationException("Cannot serialize Span " + span + ", no access to parent StringFieldValue.");
+        }
+    }
+
+    public void write(SpanList spanList) {
+        buf.put(SpanList.ID);
+        buf.putInt1_2_4Bytes(spanList.numChildren());
+        Iterator<SpanNode> children = spanList.childIterator();
+        while (children.hasNext()) {
+            write(children.next());
+        }
+    }
+
+    public void write(AlternateSpanList altSpanList) {
+        buf.put(AlternateSpanList.ID);
+        buf.putInt1_2_4Bytes(altSpanList.getNumSubTrees());
+        for (int i = 0; i < altSpanList.getNumSubTrees(); i++) {
+            buf.putDouble(altSpanList.getProbability(i));
+            buf.putInt1_2_4Bytes(altSpanList.numChildren(i));
+            Iterator<SpanNode> children = altSpanList.childIterator(i);
+            while (children.hasNext()) {
+                write(children.next());
+            }
+        }
+    }
+
+    @Override
+    public void write(DocumentUpdate update) {
+        putShort(null, Document.SERIALIZED_VERSION);
+        update.getId().serialize(this);
+
+        byte contents = 0x1; // Legacy to say we have document type
+        putByte(null, contents);
+        update.getDocumentType().serialize(this);
+
+        putInt(null, update.getFieldUpdates().size());
+
+        for (FieldUpdate up : update.getFieldUpdates()) {
+            up.serialize(this);
+        }
+    }
+
+    @Override
+    public void write(FieldUpdate update) {
+        putInt(null, update.getField().getId(Document.SERIALIZED_VERSION));
+        putInt(null, update.getValueUpdates().size());
+        for (ValueUpdate vupd : update.getValueUpdates()) {
+            putInt(null, vupd.getValueUpdateClassID().id);
+            vupd.serialize(this, update.getField().getDataType());
+        }
+    }
+
+    @Override
+    public void write(AddValueUpdate update, DataType superType) {
+        writeValue(this, ((CollectionDataType)superType).getNestedType(), update.getValue());
+        putInt(null, update.getWeight());
+    }
+
+    @Override
+    public void write(MapValueUpdate update, DataType superType) {
+        if (superType instanceof ArrayDataType) {
+            CollectionDataType type = (CollectionDataType) superType;
+            IntegerFieldValue index = (IntegerFieldValue) update.getValue();
+            index.serialize(this);
+            putInt(null, update.getUpdate().getValueUpdateClassID().id);
+            update.getUpdate().serialize(this, type.getNestedType());
+        } else if (superType instanceof WeightedSetDataType) {
+            writeValue(this, ((CollectionDataType)superType).getNestedType(), update.getValue());
+            putInt(null, update.getUpdate().getValueUpdateClassID().id);
+            update.getUpdate().serialize(this, DataType.INT);
+        } else {
+            throw new SerializationException("MapValueUpdate only works for arrays and weighted sets");
+        }
+    }
+
+    @Override
+    public void write(ArithmeticValueUpdate update) {
+        putInt(null, update.getOperator().id);
+        putDouble(null, update.getOperand().doubleValue());
+    }
+
+    @Override
+    public void write(AssignValueUpdate update, DataType superType) {
+        if (update.getValue() == null) {
+            putByte(null, (byte) 0);
+        } else {
+            putByte(null, (byte) 1);
+            writeValue(this, superType, update.getValue());
+        }
+    }
+
+    @Override
+    public void write(RemoveValueUpdate update, DataType superType) {
+        writeValue(this, ((CollectionDataType)superType).getNestedType(), update.getValue());
+    }
+
+    @Override
+    public void write(ClearValueUpdate clearValueUpdate, DataType superType) {
+        //TODO: This has never ever been implemented. Has this ever worked?
+    }
+
+    /**
+     * Returns the serialized size of the given {@link Document}. Please note that this method performs actual
+     * serialization of the document, but simply return the size of the final {@link GrowableByteBuffer}. If you need
+     * the buffer itself, do NOT use this method.
+     *
+     * @param doc The Document whose size to calculate.
+     * @return The size in bytes.
+     */
+    public static long getSerializedSize(Document doc) {
+        DocumentSerializer serializer = new VespaDocumentSerializerHead(new GrowableByteBuffer());
+        serializer.write(doc);
+        return serializer.getBuf().position();
+    }
+
+    private static void writeValue(VespaDocumentSerializer42 serializer, DataType dataType, Object value) {
+        FieldValue fieldValue;
+        if (value instanceof FieldValue) {
+            fieldValue = (FieldValue)value;
+        } else {
+            fieldValue = dataType.createFieldValue(value);
+        }
+        fieldValue.serialize(serializer);
+    }
+
+}
diff --git a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializerHead.java b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializerHead.java
new file mode 100644
index 00000000000..7f4c1960122
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializerHead.java
@@ -0,0 +1,72 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.serialization;
+
+import com.yahoo.document.DocumentUpdate;
+import com.yahoo.document.fieldpathupdate.AddFieldPathUpdate;
+import com.yahoo.document.fieldpathupdate.AssignFieldPathUpdate;
+import com.yahoo.document.fieldpathupdate.FieldPathUpdate;
+import com.yahoo.document.update.FieldUpdate;
+import com.yahoo.io.GrowableByteBuffer;
+
+/**
+ * Class used for serializing documents on the current head document format.
+ *
+ * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a>
+ */
+public class VespaDocumentSerializerHead extends VespaDocumentSerializer42 {
+
+    public VespaDocumentSerializerHead(GrowableByteBuffer buf) {
+        super(buf);
+    }
+
+    @Override
+    public void write(DocumentUpdate update) {
+        update.getId().serialize(this);
+
+        update.getDocumentType().serialize(this);
+
+        putInt(null, update.getFieldUpdates().size());
+
+        for (FieldUpdate up : update.getFieldUpdates()) {
+            up.serialize(this);
+        }
+
+        DocumentUpdateFlags flags = new DocumentUpdateFlags();
+        flags.setCreateIfNonExistent(update.getCreateIfNonExistent());
+        putInt(null, flags.injectInto(update.getFieldPathUpdates().size()));
+
+        for (FieldPathUpdate up : update.getFieldPathUpdates()) {
+            up.serialize(this);
+        }
+    }
+
+    public void write(FieldPathUpdate update) {
+        putByte(null, (byte)update.getUpdateType().getCode());
+        put(null, update.getOriginalFieldPath());
+        put(null, update.getOriginalWhereClause());
+    }
+
+    public void write(AssignFieldPathUpdate update) {
+        write((FieldPathUpdate)update);
+        byte flags = 0;
+        if (update.getRemoveIfZero()) {
+            flags |= AssignFieldPathUpdate.REMOVE_IF_ZERO;
+        }
+        if (update.getCreateMissingPath()) {
+            flags |= AssignFieldPathUpdate.CREATE_MISSING_PATH;
+        }
+        if (update.isArithmetic()) {
+            flags |= AssignFieldPathUpdate.ARITHMETIC_EXPRESSION;
+            putByte(null, flags);
+            put(null, update.getExpression());
+        } else {
+            putByte(null, flags);
+            update.getFieldValue().serialize(this);
+        }
+    }
+
+    public void write(AddFieldPathUpdate update) {
+        write((FieldPathUpdate)update);
+        update.getNewValues().serialize(this);
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/serialization/XmlDocumentWriter.java b/document/src/main/java/com/yahoo/document/serialization/XmlDocumentWriter.java
new file mode 100644
index 00000000000..ffe6073cd6c
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/serialization/XmlDocumentWriter.java
@@ -0,0 +1,314 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.serialization;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayDeque;
+import java.util.Deque;
+import java.util.Iterator;
+import java.util.Map;
+
+import com.yahoo.document.Document;
+import com.yahoo.document.DocumentId;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.Field;
+import com.yahoo.document.annotation.AnnotationReference;
+import com.yahoo.document.datatypes.*;
+import com.yahoo.vespa.objects.FieldBase;
+import com.yahoo.vespa.objects.Serializer;
+
+// TODO: Just inline all use of XmlSerializationHelper when the toXml methods in FieldValue subclasses are to be removed
+// TODO: More cleanup, the put() methods generate a lot of superfluous objects (write should call put, not the other way around)
+// TODO: remove pingpong between XmlSerializationHelper and FieldValue, this will go away when the toXml methods go away
+/**
+ * Render a Document instance as XML.
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+@SuppressWarnings("deprecation")
+public final class XmlDocumentWriter implements DocumentWriter {
+
+    private final String indent;
+    private XmlStream buffer;
+    private Deque<FieldBase> optionalWrapperMarker = new ArrayDeque<FieldBase>();
+
+    public static XmlDocumentWriter createWriter(String indent) {
+        return new XmlDocumentWriter(indent);
+    }
+
+    public XmlDocumentWriter() {
+        this("  ");
+    }
+
+    private XmlDocumentWriter(String indent) {
+        this.indent = indent;
+    }
+
+    // this method is silly, what is the intended way of doing this?
+    @Override
+    public void write(FieldBase field, FieldValue value) {
+        Class<?> valueType = value.getClass();
+        if (valueType == AnnotationReference.class) {
+            write(field, (AnnotationReference) value);
+        } else if (valueType == Array.class) {
+            write(field, (Array<?>) value);
+        } else if (valueType == WeightedSet.class) {
+            write(field, (WeightedSet<?>) value);
+        } else if (valueType == Document.class) {
+            write(field, (Document) value);
+        } else if (valueType == Struct.class) {
+            write(field, (Struct) value);
+        } else if (valueType == ByteFieldValue.class) {
+            write(field, (ByteFieldValue) value);
+        } else if (valueType == DoubleFieldValue.class) {
+            write(field, (DoubleFieldValue) value);
+        } else if (valueType == FloatFieldValue.class) {
+            write(field, (FloatFieldValue) value);
+        } else if (valueType == IntegerFieldValue.class) {
+            write(field, (IntegerFieldValue) value);
+        } else if (valueType == LongFieldValue.class) {
+            write(field, (LongFieldValue) value);
+        } else if (valueType == Raw.class) {
+            write(field, (Raw) value);
+        } else if (valueType == PredicateFieldValue.class) {
+            write(field, (PredicateFieldValue) value);
+        } else if (valueType == StringFieldValue.class) {
+            write(field, (StringFieldValue) value);
+        } else {
+            throw new UnsupportedOperationException("Cannot serialize a "
+                    + valueType.getName());
+        }
+    }
+
+    @Override
+    public void write(FieldBase field, Document value) {
+        buffer.beginTag("document");
+        buffer.addAttribute("documenttype", value.getDataType().getName());
+        buffer.addAttribute("documentid", value.getId());
+        final java.lang.Long lastModified = value.getLastModified();
+        if (lastModified != null) {
+            buffer.addAttribute("lastmodifiedtime", lastModified);
+        }
+        write(null, value.getHeader());
+        write(null, value.getBody());
+
+        buffer.endTag();
+    }
+
+    @Override
+    public <T extends FieldValue> void write(FieldBase field, Array<T> value) {
+        buffer.beginTag(field.getName());
+        XmlSerializationHelper.printArrayXml(value, buffer);
+        buffer.endTag();
+    }
+
+    private void singleValueTag(FieldBase field, FieldValue value) {
+        buffer.beginTag(field.getName());
+        value.printXml(buffer);
+        buffer.endTag();
+    }
+
+    @Override
+    public <K extends FieldValue, V extends FieldValue> void write(FieldBase field, MapFieldValue<K, V> map) {
+        // TODO Auto-generated method stub
+        buffer.beginTag(field.getName());
+        XmlSerializationHelper.printMapXml(map, buffer);
+        buffer.endTag();
+    }
+
+    @Override
+    public void write(FieldBase field, ByteFieldValue value) {
+        singleValueTag(field, value);
+    }
+
+    @Override
+    public <T extends FieldValue> void write(FieldBase field,
+            CollectionFieldValue<T> value) {
+        buffer.beginTag(field.getName());
+        for (@SuppressWarnings("unchecked")
+        Iterator<FieldValue> i = (Iterator<FieldValue>) value.iterator();
+                i.hasNext();) {
+            buffer.beginTag("item");
+            i.next().printXml(buffer);
+            buffer.endTag();
+        }
+        buffer.endTag();
+    }
+
+    @Override
+    public void write(FieldBase field, DoubleFieldValue value) {
+        singleValueTag(field, value);
+    }
+
+    @Override
+    public void write(FieldBase field, FloatFieldValue value) {
+        singleValueTag(field, value);
+    }
+
+    @Override
+    public void write(FieldBase field, IntegerFieldValue value) {
+        singleValueTag(field, value);
+    }
+
+    @Override
+    public void write(FieldBase field, LongFieldValue value) {
+        singleValueTag(field, value);
+    }
+
+    @Override
+    public void write(FieldBase field, Raw value) {
+        buffer.beginTag(field.getName());
+        XmlSerializationHelper.printRawXml(value, buffer);
+        buffer.endTag();
+    }
+
+    @Override
+    public void write(FieldBase field, PredicateFieldValue value) {
+        singleValueTag(field, value);
+    }
+
+    @Override
+    public void write(FieldBase field, StringFieldValue value) {
+        buffer.beginTag(field.getName());
+        XmlSerializationHelper.printStringXml(value, buffer);
+        buffer.endTag();
+    }
+
+    @Override
+    public void write(FieldBase field, TensorFieldValue value) {
+        throw new IllegalArgumentException("write() for tensor field value not implemented yet");
+    }
+
+    private void optionalWrapperStart(FieldBase field) {
+        if (field == null) {
+            return;
+        }
+
+        optionalWrapperMarker.addFirst(field);
+
+        buffer.beginTag(field.getName());
+    }
+
+    private void optionalWrapperEnd(FieldBase field) {
+        if (field == null) {
+            return;
+        }
+
+        if (optionalWrapperMarker.removeFirst() != field) {
+            throw new IllegalStateException("Unbalanced optional wrapper tags.");
+        }
+
+        buffer.endTag();
+    }
+
+    @Override
+    public void write(FieldBase field, Struct value) {
+        optionalWrapperStart(field);
+        XmlSerializationHelper.printStructXml(value, buffer);
+        optionalWrapperEnd(field);
+    }
+
+    @Override
+    public void write(FieldBase field, StructuredFieldValue value) {
+        buffer.beginTag(field.getName());
+        Iterator<Map.Entry<Field, FieldValue>> i = value.iterator();
+        while (i.hasNext()) {
+            Map.Entry<Field, FieldValue> v = i.next();
+            buffer.beginTag(v.getKey().getName());
+            v.getValue().printXml(buffer);
+            buffer.endTag();
+        }
+        buffer.endTag();
+    }
+
+    @Override
+    public <T extends FieldValue> void write(FieldBase field,
+            WeightedSet<T> value) {
+        buffer.beginTag(field.getName());
+        XmlSerializationHelper.printWeightedSetXml(value, buffer);
+        buffer.endTag();
+    }
+
+    @Override
+    public void write(FieldBase field, AnnotationReference value) {
+        // TODO Auto-generated method stub
+
+    }
+
+    @Override
+    public Serializer putByte(FieldBase field, byte value) {
+        singleValueTag(field, new ByteFieldValue(value));
+        return this;
+    }
+
+    @Override
+    public Serializer putShort(FieldBase field, short value) {
+        singleValueTag(field, new IntegerFieldValue(value));
+        return this;
+    }
+
+    @Override
+    public Serializer putInt(FieldBase field, int value) {
+        singleValueTag(field, new IntegerFieldValue(value));
+        return this;
+    }
+
+    @Override
+    public Serializer putLong(FieldBase field, long value) {
+        singleValueTag(field, new LongFieldValue(value));
+        return this;
+    }
+
+    @Override
+    public Serializer putFloat(FieldBase field, float value) {
+        singleValueTag(field, new FloatFieldValue(value));
+        return this;
+    }
+
+    @Override
+    public Serializer putDouble(FieldBase field, double value) {
+        singleValueTag(field, new DoubleFieldValue(value));
+        return this;
+    }
+
+    @Override
+    public Serializer put(FieldBase field, byte[] value) {
+        write(field, new Raw(value));
+        return this;
+    }
+
+    @Override
+    public Serializer put(FieldBase field, ByteBuffer value) {
+        write(field, new Raw(value));
+        return this;
+    }
+
+    @Override
+    public Serializer put(FieldBase field, String value) {
+        write(field, new StringFieldValue(value));
+        return this;
+    }
+
+    @Override
+    public void write(Document document) {
+        buffer = new XmlStream();
+        buffer.setIndent(indent);
+        optionalWrapperMarker.clear();
+        write(new Field(document.getDataType().getName(), 0, document.getDataType(), true), document);
+    }
+
+    @Override
+    public void write(DocumentId id) {
+        throw new UnsupportedOperationException("Writing a DocumentId as XML is not implemented.");
+    }
+
+    @Override
+    public void write(DocumentType type) {
+        throw new UnsupportedOperationException("Writing a DocumentId as XML is not implemented.");
+
+    }
+
+    public String lastRendered() {
+        return buffer.toString();
+    }
+
+}
diff --git a/document/src/main/java/com/yahoo/document/serialization/XmlSerializationHelper.java b/document/src/main/java/com/yahoo/document/serialization/XmlSerializationHelper.java
new file mode 100644
index 00000000000..d7362095cf3
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/serialization/XmlSerializationHelper.java
@@ -0,0 +1,128 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.serialization;
+
+import com.yahoo.document.Document;
+import com.yahoo.document.Field;
+import com.yahoo.document.datatypes.*;
+import com.yahoo.text.Utf8;
+import org.apache.commons.codec.binary.Base64;
+
+import java.io.UnsupportedEncodingException;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Date: Apr 17, 2008
+ *
+ * @author <a href="mailto:humbe@yahoo-inc.com">H&aring;kon Humberset</a>
+ */
+public class XmlSerializationHelper {
+
+    public static void printArrayXml(Array array, XmlStream xml) {
+        List<FieldValue> lst = array.getValues();
+        for (FieldValue value : lst) {
+            xml.beginTag("item");
+            value.printXml(xml);
+            xml.endTag();
+        }
+    }
+
+    public static <K extends FieldValue, V extends FieldValue> void printMapXml(MapFieldValue<K, V> map, XmlStream xml) {
+        for (Map.Entry<K, V> e : map.entrySet()) {
+            FieldValue key = e.getKey();
+            FieldValue val = e.getValue();
+            xml.beginTag("item");
+            xml.beginTag("key");
+            key.printXml(xml);
+            xml.endTag();
+            xml.beginTag("value");
+            val.printXml(xml);
+            xml.endTag();
+            xml.endTag();
+        }
+    }
+
+    public static void printByteXml(ByteFieldValue b, XmlStream xml) {
+        xml.addContent(b.toString());
+    }
+
+    public static void printDocumentXml(Document doc, XmlStream xml) {
+        xml.addAttribute("documenttype", doc.getDataType().getName());
+        xml.addAttribute("documentid", doc.getId());
+        final java.lang.Long lastModified = doc.getLastModified();
+        if (lastModified != null) {
+            xml.addAttribute("lastmodifiedtime", lastModified);
+        }
+        doc.getHeader().printXml(xml);
+        doc.getBody().printXml(xml);
+    }
+
+    public static void printDoubleXml(DoubleFieldValue d, XmlStream xml) {
+        xml.addContent(d.toString());
+    }
+
+    public static void printFloatXml(FloatFieldValue f, XmlStream xml) {
+        xml.addContent(f.toString());
+    }
+
+    public static void printIntegerXml(IntegerFieldValue f, XmlStream xml) {
+        xml.addContent(f.toString());
+    }
+
+    public static void printLongXml(LongFieldValue l, XmlStream xml) {
+        xml.addContent(l.toString());
+    }
+
+    public static void printRawXml(Raw r, XmlStream xml) {
+        xml.addAttribute("binaryencoding", "base64");
+        xml.addContent(new Base64(0).encodeToString(r.getByteBuffer().array()));
+    }
+
+    public static void printStringXml(StringFieldValue s, XmlStream xml) {
+        String content = s.getString();
+        if (containsNonPrintableCharactersString(content)) {
+            byte[] bytecontent = Utf8.toBytes(content);
+            xml.addAttribute("binaryencoding", "base64");
+            xml.addContent(new Base64(0).encodeToString(bytecontent));
+        } else {
+            xml.addContent(content);
+        }
+    }
+
+    public static void printStructXml(Struct s, XmlStream xml) {
+        Iterator<Map.Entry<Field, FieldValue>> it = s.iterator();
+        while (it.hasNext()) {
+            Map.Entry<Field, FieldValue> val = it.next();
+            xml.beginTag(val.getKey().getName());
+            val.getValue().printXml(xml);
+            xml.endTag();
+        }
+    }
+
+    public static void printWeightedSetXml(WeightedSet ws, XmlStream xml) {
+        Iterator<FieldValue> it = ws.fieldValueIterator();
+        while (it.hasNext()) {
+            FieldValue val = it.next();
+            xml.beginTag("item");
+            xml.addAttribute("weight", ws.get(val));
+            val.printXml(xml);
+            xml.endTag();
+        }
+    }
+
+    private static boolean containsNonPrintableCharactersByte(final byte[] buffer) {
+        for (byte b : buffer) {
+            if (b < 32 && (b != 9 && b != 10 && b != 13)) return true;
+        }
+        return false;
+    }
+
+    private static boolean containsNonPrintableCharactersString(final CharSequence buffer) {
+        for (int i = 0; i < buffer.length(); i++) {
+            char b = buffer.charAt(i);
+            if (b < 32 && (b != 9 && b != 10 && b != 13)) return true;
+        }
+        return false;
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/serialization/XmlStream.java b/document/src/main/java/com/yahoo/document/serialization/XmlStream.java
new file mode 100644
index 00000000000..f0c8451cd2a
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/serialization/XmlStream.java
@@ -0,0 +1,215 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.serialization;
+
+import com.yahoo.text.XML;
+
+import java.io.StringWriter;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Deque;
+import java.util.List;
+import java.util.ListIterator;
+
+/**
+ * Class for writing XML in a simplified way.
+ * <p>
+ * Give a writer for it to write the XML directly to. If none is given a
+ * StringWriter is used so you can call toString() on the class to get the
+ * XML.
+ * <p>
+ * You build XML by calling beginTag(name), addAttribute(id, value), endTag().
+ * Remember to close all your tags, or you'll get an exception when calling
+ * toString(). If writing directly to a writer, call isFinalized to verify that
+ * all tags have been closed.
+ * <p>
+ * The XML escaping tools only give an interface for escape from and to a string
+ * value. Thus writing of all data here is also just available through strings.
+ *
+ * @author <a href="humbe@yahoo-inc.com">Haakon Humberset</a>
+ */
+public class XmlStream {
+
+    // Utility class to hold attributes internally until it's time to write them
+    private static class Attribute {
+        final String name;
+        final String value;
+
+        public Attribute(String name, Object value) {
+            this.name = name;
+            this.value = value.toString();
+        }
+    }
+
+    private final StringWriter writer; // Writer to output XML to.
+    private final Deque<String> tags = new ArrayDeque<String>();
+    private String indent = "";
+    // We write tags lazily for several reasons:
+    //   - To allow recursive methods to have both parents and child add
+    //     attributes to last tag, without giving child responsibility of
+    //     closing or creating the tag.
+    //   - Be able to check content before adding whitespace, such that we
+    //     can add newlines if content is new tags for instance.
+    // The cached variables here will be written with the flush() method.
+    private String cachedTag = null;
+    private final List<Attribute> cachedAttribute = new ArrayList<Attribute>();
+    private final List<String> cachedContent = new ArrayList<String>();
+
+    /**
+     * Create an XmlStream writing to a StringWriter.
+     * Fetch XML through toString() once you're done creating it.
+     */
+    public XmlStream() {
+        writer = new StringWriter();
+    }
+
+    /**
+     * Set an indent to use for pretty printing of XML. Default is no indent.
+     *
+     * @param indent the initial indentation
+     */
+    public void setIndent(String indent) {
+        this.indent = indent;
+    }
+
+    /**
+     * Check if all tags have been properly closed.
+     *
+     * @return true if all tags are closed
+     */
+    public boolean isFinalized() {
+        return (tags.isEmpty() && cachedTag == null);
+    }
+
+    public String toString() {
+        if (!isFinalized()) {
+            throw new IllegalStateException("There are still" + " tag(s) that are not closed.");
+        }
+        StringWriter sw = writer; // Ensure we have string writer
+        return sw.toString();
+    }
+
+    /**
+     * Add a new XML tag with the given name.
+     *
+     * @param name the tag name
+     */
+    public void beginTag(String name) {
+        if (!XML.isName(name)) {
+            throw new IllegalArgumentException("The name '" + name
+                    + "' cannot be used as an XML tag name. Legal names must adhere to"
+                    + "http://www.w3.org/TR/2006/REC-xml11-20060816/#sec-common-syn");
+        }
+        if (cachedTag != null) flush(false);
+        cachedTag = name;
+    }
+
+    /**
+     * Add a new XML attribute to the last tag started.
+     * The tag cannot already have had content added to it, or been ended.
+     * If a null value is added, the attribute will be skipped.
+     *
+     * @param key   the attribute name
+     * @param value the attribute value
+     */
+    public void addAttribute(String key, Object value) {
+        if (value == null) {
+            return;
+        }
+        if (cachedTag == null) {
+            throw new IllegalStateException("There is no open tag to add attributes to.");
+        }
+        if (!XML.isName(key)) {
+            throw new IllegalArgumentException("The name '" + key
+                    + "' cannot be used as an XML attribute name. Legal names must adhere to"
+                    + " http://www.w3.org/TR/2006/REC-xml11-20060816/#sec-common-syn");
+        }
+        cachedAttribute.add(new Attribute(key, value));
+    }
+
+    /**
+     * Add content to the last tag.
+     *
+     * @param content the content to add to the last tag
+     */
+    public void addContent(String content) {
+        if (cachedTag != null) {
+            cachedContent.add(XML.xmlEscape(content, false));
+        } else if (tags.isEmpty()) {
+            throw new IllegalStateException("There is no open tag to add content to.");
+        } else {
+            for (int i = 0; i < tags.size(); ++i) {
+                writer.write(indent);
+            }
+            writer.write(XML.xmlEscape(content, false));
+            writer.write('\n');
+        }
+    }
+
+    /**
+     * Ends the last tag created.
+     *
+     */
+    public void endTag() {
+        if (cachedTag != null) {
+            flush(true);
+        } else if (tags.isEmpty()) {
+            throw new IllegalStateException("Cannot end non-existing tag");
+        } else {
+            for (int i = 1; i < tags.size(); ++i) {
+                writer.write(indent);
+            }
+            writer.write("</");
+            writer.write(tags.removeFirst());
+            writer.write(">\n");
+        }
+    }
+
+    // Utility function to write whatever is cached.
+    private void flush(boolean endTag) {
+        if (cachedTag == null) {
+            throw new IllegalStateException("Cannot write non-existing tag");
+        }
+        for (int i = 0; i < tags.size(); ++i) {
+            writer.write(indent);
+        }
+        writer.write('<');
+        writer.write(cachedTag);
+        for (ListIterator<Attribute> it = cachedAttribute.listIterator(); it.hasNext();) {
+            Attribute attr = it.next();
+            writer.write(' ');
+            writer.write(attr.name);
+            writer.write("=\"");
+            writer.write(XML.xmlEscape(attr.value, true));
+            writer.write('"');
+        }
+        cachedAttribute.clear();
+        if (cachedContent.isEmpty() && endTag) {
+            writer.write("/>\n");
+        } else if (cachedContent.isEmpty()) {
+            writer.write(">\n");
+            tags.addFirst(cachedTag);
+        } else {
+            writer.write(">");
+            if (!endTag) {
+                writer.write('\n');
+                for (int i = 0; i <= tags.size(); ++i) {
+                    writer.write(indent);
+                }
+            }
+            for (String content : cachedContent) {
+                writer.write(content);
+            }
+            cachedContent.clear();
+            if (endTag) {
+                writer.write("</");
+                writer.write(cachedTag);
+                writer.write(">\n");
+            } else {
+                writer.write('\n');
+                tags.addFirst(cachedTag);
+            }
+        }
+        cachedTag = null;
+    }
+
+}
diff --git a/document/src/main/java/com/yahoo/document/serialization/package-info.java b/document/src/main/java/com/yahoo/document/serialization/package-info.java
new file mode 100644
index 00000000000..711990c5d9a
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/serialization/package-info.java
@@ -0,0 +1,7 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+@PublicApi
+package com.yahoo.document.serialization;
+
+import com.yahoo.api.annotations.PublicApi;
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/document/src/main/java/com/yahoo/document/update/AddValueUpdate.java b/document/src/main/java/com/yahoo/document/update/AddValueUpdate.java
new file mode 100644
index 00000000000..1ad2941c80c
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/update/AddValueUpdate.java
@@ -0,0 +1,102 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.update;
+
+import com.yahoo.document.CollectionDataType;
+import com.yahoo.document.DataType;
+import com.yahoo.document.datatypes.CollectionFieldValue;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.WeightedSet;
+import com.yahoo.document.serialization.DocumentUpdateWriter;
+
+/**
+ * <p>Value update representing an addition of a value (possibly with an associated weight)
+ * to a multi-valued data type.</p>
+ *
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class AddValueUpdate extends ValueUpdate {
+    protected FieldValue value;
+    protected Integer weight;
+
+    AddValueUpdate(FieldValue value) {
+        super(ValueUpdateClassID.ADD);
+        setValue(value, 1);
+    }
+
+    public AddValueUpdate(FieldValue key, int weight) {
+        super(ValueUpdateClassID.ADD);
+        setValue(key, weight);
+    }
+
+    private void setValue(FieldValue key, int weight) {
+        this.value = key;
+        this.weight = weight;
+    }
+
+    /**
+     * Returns the value of this value update.
+     *
+     * @return the value of this ValueUpdate
+     * @see com.yahoo.document.DataType
+     */
+    public FieldValue getValue() { return value; }
+
+    public void setValue(FieldValue value) { this.value=value; }
+
+    /**
+     * Return the associated weight of this value update.
+     *
+     * @return the weight of this value update, or 1 if unset
+     */
+    public int getWeight() {
+        return weight;
+    }
+
+    @Override
+    public FieldValue applyTo(FieldValue val) {
+        if (val instanceof WeightedSet) {
+            WeightedSet wset = (WeightedSet) val;
+            wset.put((FieldValue) value, weight);
+        } else if (val instanceof CollectionFieldValue) {
+            CollectionFieldValue fval = (CollectionFieldValue) val;
+            fval.add((FieldValue) value);
+        } else {
+            throw new IllegalStateException("Cannot add "+value+" to field of type " + val.getClass().getName());
+        }
+        return val;
+    }
+
+    @Override
+    protected void checkCompatibility(DataType fieldType) {
+        if (!(fieldType instanceof CollectionDataType)) {
+            throw new UnsupportedOperationException("Expected collection, got " + fieldType.getName() + ".");
+        }
+        fieldType = ((CollectionDataType)fieldType).getNestedType();
+        if (value != null && !value.getDataType().equals(fieldType)) {
+            throw new IllegalArgumentException("Expected " + fieldType.getName() + ", got " +
+                                               value.getDataType().getName());
+        }
+    }
+
+    @Override
+    public void serialize(DocumentUpdateWriter data, DataType superType) {
+        data.write(this, superType);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        return o instanceof AddValueUpdate && super.equals(o) && value.equals(((AddValueUpdate) o).value) &&
+                weight.equals(((AddValueUpdate) o).weight);
+    }
+
+    @Override
+    public int hashCode() {
+        return super.hashCode() + value.hashCode() + weight;
+    }
+
+    @Override
+    public String toString() {
+        return super.toString() + " " + value + " " + weight;
+    }
+
+}
diff --git a/document/src/main/java/com/yahoo/document/update/ArithmeticValueUpdate.java b/document/src/main/java/com/yahoo/document/update/ArithmeticValueUpdate.java
new file mode 100644
index 00000000000..c4e677cab66
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/update/ArithmeticValueUpdate.java
@@ -0,0 +1,159 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.update;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.NumericDataType;
+import com.yahoo.document.datatypes.DoubleFieldValue;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.NumericFieldValue;
+import com.yahoo.document.serialization.DocumentUpdateWriter;
+
+/**
+ * <p>Value update representing an arithmetic operation on a numeric data type.</p>
+ *
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class ArithmeticValueUpdate extends ValueUpdate<DoubleFieldValue> {
+    protected Operator operator;
+    protected DoubleFieldValue operand;
+
+    public ArithmeticValueUpdate(Operator operator, DoubleFieldValue operand) {
+        super(ValueUpdateClassID.ARITHMETIC);
+        this.operator = operator;
+        this.operand = operand;
+    }
+
+    public ArithmeticValueUpdate(Operator operator, Number operand) {
+        this(operator, new DoubleFieldValue(operand.doubleValue()));
+    }
+
+    /**
+     * Returns the operator of this arithmatic value update.
+     *
+     * @return the operator
+     * @see com.yahoo.document.update.ArithmeticValueUpdate.Operator
+     */
+    public Operator getOperator() {
+        return operator;
+    }
+
+    /**
+     * Returns the operand of this arithmetic value update.
+     *
+     * @return the operand
+     */
+    public Number getOperand() {
+        return operand.getDouble();
+    }
+
+    /** Returns the operand */
+    public DoubleFieldValue getValue() { return operand; }
+
+    /** Sets the operand */
+    public void setValue(DoubleFieldValue value) { operand=value; }
+
+    @Override
+    public FieldValue applyTo(FieldValue oldValue) {
+        if (oldValue instanceof NumericFieldValue) {
+            Number number = (Number) oldValue.getWrappedValue();
+            oldValue.assign(calculate(number));
+        } else {
+            throw new IllegalStateException("Cannot use arithmetic value update on non-numeric datatype "+oldValue.getClass().getName());
+        }
+        return oldValue;
+    }
+
+    @Override
+    protected void checkCompatibility(DataType fieldType) {
+        if (!(fieldType instanceof NumericDataType)) {
+            throw new UnsupportedOperationException("Expected numeric type, got " + fieldType.getName() + ".");
+        }
+    }
+
+    private double calculate(Number operand2) {
+        switch (operator) {
+            case ADD:
+                return operand2.doubleValue() + operand.getDouble();
+            case DIV:
+                return operand2.doubleValue() / operand.getDouble();
+            case MUL:
+                return operand2.doubleValue() * operand.getDouble();
+            case SUB:
+                return operand2.doubleValue() - operand.getDouble();
+        }
+        return 0d;
+    }
+
+    @Override
+    public void serialize(DocumentUpdateWriter data, DataType superType) {
+        data.write(this);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        return o instanceof ArithmeticValueUpdate && super.equals(o) &&
+                operator == ((ArithmeticValueUpdate) o).operator && operand.equals(((ArithmeticValueUpdate) o).operand);
+    }
+
+    @Override
+    public int hashCode() {
+        return super.hashCode() + operator.hashCode() + operand.hashCode();
+    }
+
+    @Override
+    public String toString() {
+        return super.toString() + " " + operator.name + " " + operand;
+    }
+
+    /**
+     * Lists valid operations that can be performed by an ArithmeticValueUpdate.
+     */
+    public enum Operator {
+        /**
+         * Add the operand to the value.
+         */
+        ADD(0, "add"),
+        /**
+         * Divide the value by the operand.
+         */
+        DIV(1, "divide"),
+        /**
+         * Multiply the value by the operand.
+         */
+        MUL(2, "multiply"),
+        /**
+         * Subtract the operand from the value.
+         */
+        SUB(3, "subtract");
+
+        /**
+         * The numeric ID of the operator, used for serialization.
+         */
+        public final int id;
+        /**
+         * The name of the operator, mainly used in toString() methods.
+         */
+        public final String name;
+
+        Operator(int id, String name) {
+            this.id = id;
+            this.name = name;
+        }
+
+        /**
+         * Returns the operator with the specified ID.
+         *
+         * @param id the ID to search for
+         * @return the Operator with the specified ID, or null if it does not exist.
+         */
+        public static Operator getID(int id) {
+            for (Operator operator : Operator.values()) {
+                if (operator.id == id) {
+                    return operator;
+                }
+            }
+            return null;
+        }
+    }
+}
+
diff --git a/document/src/main/java/com/yahoo/document/update/AssignValueUpdate.java b/document/src/main/java/com/yahoo/document/update/AssignValueUpdate.java
new file mode 100644
index 00000000000..f1157d272e2
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/update/AssignValueUpdate.java
@@ -0,0 +1,87 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.update;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.serialization.DocumentUpdateWriter;
+
+/**
+ * <p>Value update that represents assigning a new value.</p>
+ *
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class AssignValueUpdate extends ValueUpdate {
+    protected FieldValue value;
+
+    public AssignValueUpdate(FieldValue value) {
+        super(ValueUpdateClassID.ASSIGN);
+        this.value = value;
+    }
+
+    /**
+     * <p>Returns the value of this value update.</p>
+     *
+     * <p>The type of the value is defined by the type of this field
+     * in this documents DocumentType - a java.lang primitive wrapper for single value types,
+     * java.util.List for arrays and {@link com.yahoo.document.datatypes.WeightedSet WeightedSet}
+     * for weighted sets.</p>
+     *
+     * @return the value of this ValueUpdate
+     * @see com.yahoo.document.DataType
+     */
+    public FieldValue getValue() { return value; }
+
+    /**
+     * <p>Sets the value to assign.</p>
+     *
+     * <p>The type of the value must match the type of this field
+     * in this documents DocumentType - a java.lang primitive wrapper for single value types,
+     * java.util.List for arrays and {@link com.yahoo.document.datatypes.WeightedSet WeightedSet}
+     * for weighted sets.</p>
+     */
+    public void setValue(FieldValue value) { this.value=value; }
+
+    @Override
+    public FieldValue applyTo(FieldValue fval) {
+        if (value == null) return null;
+        fval.assign(value);
+        return fval;
+    }
+
+    @Override
+    protected void checkCompatibility(DataType fieldType) {
+        if (value != null && !value.getDataType().equals(fieldType)) {
+            throw new IllegalArgumentException("Expected " + fieldType.getName() + ", got " +
+                                               value.getDataType().getName());
+        }
+    }
+
+    @Override
+    public void serialize(DocumentUpdateWriter data, DataType superType) {
+        data.write(this, superType);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        boolean baseEquals = o instanceof AssignValueUpdate && super.equals(o);
+
+        if (!baseEquals) return false;
+
+        if (value == null && ((AssignValueUpdate) o).value == null) {
+            return true;
+        } else if (value != null && value.equals(((AssignValueUpdate) o).value)) {
+            return true;
+        }
+        return false;
+    }
+
+    @Override
+    public int hashCode() {
+        return super.hashCode() + (value == null ? 0 : value.hashCode());
+    }
+
+    @Override
+    public String toString() {
+        return super.toString() + " " + value;
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/update/ClearValueUpdate.java b/document/src/main/java/com/yahoo/document/update/ClearValueUpdate.java
new file mode 100644
index 00000000000..a9a0ad271ef
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/update/ClearValueUpdate.java
@@ -0,0 +1,42 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.update;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.serialization.DocumentUpdateWriter;
+
+/**
+ * <p>Value update that represents clearing a field. Clearing a field mean removing it.</p>
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class ClearValueUpdate extends ValueUpdate {
+
+    public ClearValueUpdate() {
+        super(ValueUpdateClassID.CLEAR);
+    }
+
+    @Override
+    public FieldValue applyTo(FieldValue fval) {
+        return null;
+    }
+
+    @Override
+    protected void checkCompatibility(DataType fieldType) {
+        // empty
+    }
+
+    @Override
+    public FieldValue getValue() {
+        return null;
+    }
+
+    @Override
+    public void setValue(FieldValue value) {
+        // empty
+    }
+
+    @Override
+    public void serialize(DocumentUpdateWriter data, DataType superType) {
+        data.write(this, superType);
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/update/FieldUpdate.java b/document/src/main/java/com/yahoo/document/update/FieldUpdate.java
new file mode 100644
index 00000000000..c4250a33f28
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/update/FieldUpdate.java
@@ -0,0 +1,624 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.update;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.Document;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.Field;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.WeightedSet;
+import com.yahoo.document.serialization.DocumentSerializerFactory;
+import com.yahoo.document.serialization.DocumentUpdateReader;
+import com.yahoo.document.serialization.DocumentUpdateWriter;
+import com.yahoo.io.GrowableByteBuffer;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * <p>A field update holds a list of value updates that will be applied atomically to a field in a document.</p>
+ * <p>To create a field update that contains only a single value update, use the static factory methods provided by this
+ * class.</p> <p>Example:</p>
+ * <pre>
+ * FieldUpdate clearFieldUpdate = FieldUpdate.createClearField(field);
+ * </pre>
+ * <p>It is also possible to create a field update that holds more than one value update.</p>
+ * <p>Example:</p>
+ * <pre>
+ * FieldUpdate fieldUpdate = FieldUpdate.create(field);
+ * ValueUpdate incrementValue = ValueUpdate.createIncrement("foo", 130d);
+ * ValueUpdate addValue = ValueUpdate.createAdd("bar", 100);
+ * fieldUpdate.addValueUpdate(incrementValue);
+ * fieldUpdate.addValueUpdate(addValue);</pre>
+ * <p>Note that the addValueUpdate() method returns a reference to itself to support chaining, so the last two
+ * lines could be written as one:</p>
+ * <pre>
+ * fieldUpdate.addValueUpdate(incrementValue).addValueUpdate(addValue);
+ * </pre>
+ * <p>Note also that the second example above is equivalent to:</p>
+ * <pre>
+ * FieldUpdate fieldUpdate = FieldUpdate.createIncrement(field, "foo", 130d);
+ * ValueUpdate addValue = ValueUpdate.createAdd("bar", 100);
+ * fieldUpdate.addValueUpdate(addValue);
+ * </pre>
+ * <p>Note that even though updates take fields as arguments, those fields are not necessarily a field of a document
+ * type - any name/value pair which existing in an updatable structure can be addressed by creating the Fields as
+ * needed. For example:
+ * <pre>
+ * FieldUpdate field=FieldUpdate.createIncrement(new Field("myattribute",DataType.INT),130);
+ * </pre>
+ *
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ * @see com.yahoo.document.update.ValueUpdate
+ * @see com.yahoo.document.DocumentUpdate
+ */
+public class FieldUpdate {
+
+    protected Field field;
+    protected List<ValueUpdate> valueUpdates = new ArrayList<>();
+
+    // Used only while deserializing.
+    private DocumentType documentType = null;
+    private int serializationVersion = 0;
+
+    FieldUpdate(Field field) {
+        this.field = field;
+    }
+
+    FieldUpdate(Field field, ValueUpdate valueUpd) {
+        this(field);
+        addValueUpdate(valueUpd);
+    }
+
+    FieldUpdate(Field field, List<ValueUpdate> valueUpdates) {
+        this(field);
+        addValueUpdates(valueUpdates);
+    }
+
+    public FieldUpdate(DocumentUpdateReader reader, DocumentType type, int serializationVersion) {
+        documentType = type;
+        this.serializationVersion = serializationVersion;
+        reader.read(this);
+    }
+
+    public DocumentType getDocumentType() {
+        return documentType;
+    }
+
+    public int getSerializationVersion() {
+        return serializationVersion;
+    }
+
+    /** Returns the field that this field update applies to */
+    public Field getField() {
+        return field;
+    }
+
+    /**
+     * Sets the field this update applies to. Note that this does not need to be
+     * a field of the document type in question - a field is just the name and type
+     * of some value to be updated.
+     */
+    public void setField(Field field) {
+        this.field = field;
+    }
+
+    /**
+     * Applies this field update.
+     *
+     * @param doc the document to apply the update to
+     * @return a reference to itself
+     */
+    public FieldUpdate applyTo(Document doc) {
+        for (ValueUpdate vupd : valueUpdates) {
+            DataType dataType = field.getDataType();
+            FieldValue oldValue = doc.getFieldValue(field);
+            boolean existed = (oldValue != null);
+
+            if (!existed) {
+                oldValue = dataType.createFieldValue();
+            }
+
+            FieldValue newValue = vupd.applyTo(oldValue);
+
+            if (newValue == null) {
+                if (existed) {
+                    doc.removeFieldValue(field);
+                }
+            } else {
+                doc.setFieldValue(field, newValue);
+            }
+        }
+        return this;
+    }
+
+    /**
+     * Adds a value update to the list of value updates.
+     *
+     * @param valueUpdate the ValueUpdate to add
+     * @return a reference to itself
+     * @throws IllegalArgumentException if the data type of the value update is not equal to the data type of this field
+     */
+    public FieldUpdate addValueUpdate(ValueUpdate valueUpdate) {
+        valueUpdate.checkCompatibility(field.getDataType());  //will throw exception
+        valueUpdates.add(valueUpdate);
+        return this;
+    }
+
+    /**
+     * Adds a value update to the list of value updates.
+     *
+     * @param index the index where this value update should be added
+     * @param valueUpdate the ValueUpdate to add
+     * @return a reference to itself
+     * @throws IllegalArgumentException if the data type of the value update is not equal to the data type of this field
+     */
+    public FieldUpdate addValueUpdate(int index, ValueUpdate valueUpdate) {
+        valueUpdate.checkCompatibility(field.getDataType());  //will throw exception
+        valueUpdates.add(index, valueUpdate);
+        return this;
+    }
+
+    /**
+     * Adds a list of value updates to the list of value updates.
+     *
+     * @param valueUpdates a list containing the value updates to add
+     * @return a reference to itself
+     * @throws IllegalArgumentException if the data type of the value update is not equal to the data type of this field
+     */
+    public FieldUpdate addValueUpdates(List<ValueUpdate> valueUpdates) {
+        for (ValueUpdate vupd : valueUpdates) {
+            addValueUpdate(vupd);
+        }
+        return this;
+    }
+
+    /**
+     * Removes the value update at the specified position in the list of value updates.
+     *
+     * @param index the index of the ValueUpdate to remove
+     * @return the ValueUpdate previously at the specified position
+     * @throws IndexOutOfBoundsException if index is out of range
+     */
+    public ValueUpdate removeValueUpdate(int index) {
+        return valueUpdates.remove(index);
+    }
+
+    /**
+     * Replaces the value update at the specified position in the list of value updates
+     * with the specified value update.
+     *
+     * @param index index of value update to replace
+     * @param update value update to be stored at the specified position
+     * @return the value update previously at the specified position
+     * @throws IndexOutOfBoundsException if index out of range (index &lt; 0 || index &gt;= size())
+     */
+    public ValueUpdate setValueUpdate(int index, ValueUpdate update) {
+        return valueUpdates.set(index, update);
+    }
+
+    /**
+     * Get the number of value updates in this field update.
+     *
+     * @return the size of the List of FieldUpdates
+     */
+    public int size() {
+        return valueUpdates.size();
+    }
+
+    /** Removes all value updates from the list of value updates. */
+    public void clearValueUpdates() {
+        valueUpdates.clear();
+    }
+
+    /**
+     * Get the value update at the specified index in the list of value updates.
+     *
+     * @param index the index of the ValueUpdate to return
+     * @return the ValueUpdate at the specified index
+     * @throws IndexOutOfBoundsException if index is out of range
+     */
+    public ValueUpdate getValueUpdate(int index) {
+        return valueUpdates.get(index);
+    }
+
+    /**
+     * Get an unmodifiable list of all value updates that this field update specifies.
+     *
+     * @return a list of all ValueUpdates in this FieldUpdate
+     */
+    public List<ValueUpdate> getValueUpdates() {
+        return Collections.unmodifiableList(valueUpdates);
+    }
+
+    /**
+     * Get value updates with the specified valueUpdateClassID. The caller gets ownership of the returned list, and
+     * subsequent modifications to the list does not change the state of this object.
+     *
+     * @param classID the classID of ValueUpdates to return
+     * @return a List of ValueUpdates of the specified classID (possibly empty, but not null)
+     */
+    public List<ValueUpdate> getValueUpdates(ValueUpdate.ValueUpdateClassID classID) {
+        List<ValueUpdate> updateList = new ArrayList<>();
+        for (ValueUpdate vupd : valueUpdates) {
+            if (vupd.getValueUpdateClassID() == classID) {
+                updateList.add(vupd);
+            }
+        }
+        return updateList;
+    }
+
+    /**
+     * Returns whether this field update contains (at least) one update of the given type
+     *
+     * @param classID the classID of ValueUpdates to check for
+     * @return true if there is at least one value update of the given type in this
+     */
+    public boolean hasValueUpdate(ValueUpdate.ValueUpdateClassID classID) {
+        for (ValueUpdate vupd : valueUpdates) {
+            if (vupd.getValueUpdateClassID() == classID) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    /**
+     * Returns whether or not this field update contains any value updates.
+     *
+     * @return True if this update is empty.
+     */
+    public boolean isEmpty() {
+        return valueUpdates.isEmpty();
+    }
+
+    /**
+     * Adds all the {@link ValueUpdate}s of the given FieldUpdate to this. If the given FieldUpdate refers to a
+     * different {@link Field} than this, this method throws an exception.
+     *
+     * @param update The update whose content to add to this.
+     * @throws IllegalArgumentException If the {@link Field} of the given FieldUpdate does not match this.
+     */
+    public void addAll(FieldUpdate update) {
+        if (update == null) {
+            return;
+        }
+        if (!field.equals(update.field)) {
+            throw new IllegalArgumentException("Expected " + field + ", got " + update.field + ".");
+        }
+        addValueUpdates(update.valueUpdates);
+    }
+
+    public final void serialize(GrowableByteBuffer buf) {
+        serialize(DocumentSerializerFactory.create42(buf));
+    }
+
+    public void serialize(DocumentUpdateWriter data) {
+        data.write(this);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        return o instanceof FieldUpdate && field.equals(((FieldUpdate)o).field) &&
+               valueUpdates.equals(((FieldUpdate)o).valueUpdates);
+    }
+
+    @Override
+    public int hashCode() {
+        return field.getId(Document.SERIALIZED_VERSION) + valueUpdates.hashCode();
+    }
+
+    @Override
+    public String toString() {
+        return "'" + field.getName() + "' " + valueUpdates;
+    }
+
+    /**
+     * Creates a new, empty field update with no encapsulated value updates. Use this method to add an arbitrary
+     * set of value updates using the FieldUpdate.addValueUpdate() method.
+     *
+     * @param field the Field to alter
+     * @return a new, empty FieldUpdate
+     * @see com.yahoo.document.update.ValueUpdate
+     * @see FieldUpdate#addValueUpdate(ValueUpdate)
+     */
+    public static FieldUpdate create(Field field) {
+        return new FieldUpdate(field);
+    }
+
+    /**
+     * Creates a new field update that clears the field. This operation removes the value/field completely.
+     *
+     * @param field the Field to clear
+     * @return a FieldUpdate specifying the clearing
+     * @see com.yahoo.document.update.FieldUpdate#createClear(Field)
+     */
+    public static FieldUpdate createClearField(Field field) {
+        return new FieldUpdate(field, ValueUpdate.createClear());
+    }
+
+    /**
+     * Creates a new field update, with one encapsulated value update
+     * specifying an addition of a value to an array or a key to a weighted set (with default weight 1).
+     *
+     * @param field the Field to add a value to
+     * @param value the value to add to the array, or key to add to the weighted set
+     * @return a FieldUpdate specifying the addition
+     * @throws IllegalArgumentException      if the runtime type of newValue does not match the type required by field
+     * @throws UnsupportedOperationException if the field type is not array or weighted set
+     * @see com.yahoo.document.update.ValueUpdate#createAdd(FieldValue)
+     */
+    public static FieldUpdate createAdd(Field field, FieldValue value) {
+        return new FieldUpdate(field, ValueUpdate.createAdd(value));
+    }
+
+    /**
+     * Creates a new field update, with one encapsulated value update
+     * specifying an addition of a key (with a specified weight) to a weighted set. If this
+     * method is used on an array data type, the weight will be omitted.
+     *
+     * @param field  the Field to a add a key to
+     * @param key    the key to add
+     * @param weight the weight to associate with the given key
+     * @return a FieldUpdate specifying the addition
+     * @throws IllegalArgumentException      if the runtime type of key does not match the type required by field
+     * @throws UnsupportedOperationException if the field type is not array or weighted set
+     * @see com.yahoo.document.update.ValueUpdate#createAdd(FieldValue,Integer)
+     */
+    public static FieldUpdate createAdd(Field field, FieldValue key, Integer weight) {
+        return new FieldUpdate(field, ValueUpdate.createAdd(key, weight));
+    }
+
+    /**
+     * Creates a new field update, with encapsulated value updates,
+     * specifying an addition of all values in a given list to an array. If this method is used on a weighted set data
+     * type, the default weights will be 1.
+     *
+     * @param field  the Field to add an array of values to
+     * @param values a List containing the values to add
+     * @return a FieldUpdate specifying the addition
+     * @throws IllegalArgumentException      if the runtime type of values does not match the type required by field
+     * @throws UnsupportedOperationException if the field type is not array or weighted set
+     * @see com.yahoo.document.update.ValueUpdate#createAddAll(java.util.List)
+     * @see ValueUpdate#createAdd(FieldValue)
+     */
+    public static FieldUpdate createAddAll(Field field, List<? extends FieldValue> values) {
+        return new FieldUpdate(field, ValueUpdate.createAddAll(values));
+    }
+
+    /**
+     * Creates a new field update, with encapsulated value updates,
+     * specifying an addition of all key/weight pairs in a weighted set to a weighted set. If this method
+     * is used on an array data type, the weights will be omitted.
+     *
+     * @param field the Field to add the key/weight pairs in a weighted set to
+     * @param set a WeightedSet containing the key/weight pairs to add
+     * @return a FieldUpdate specifying the addition
+     * @throws IllegalArgumentException      if the runtime type of values does not match the type required by field
+     * @throws UnsupportedOperationException if the field type is not weighted set or array
+     * @see ValueUpdate#createAdd(FieldValue, Integer)
+     * @see com.yahoo.document.update.ValueUpdate#createAddAll(com.yahoo.document.datatypes.WeightedSet)
+     */
+    public static FieldUpdate createAddAll(Field field, WeightedSet<? extends FieldValue> set) {
+        return new FieldUpdate(field, ValueUpdate.createAddAll(set));
+    }
+
+    /**
+     * Creates a new field update, with one encapsulated value update that increments a value.
+     * Note that the data type must be a numeric type.
+     *
+     * @param field     the field to increment the value of
+     * @param increment the number to increment by
+     * @return a FieldUpdate specifying the increment
+     * @throws UnsupportedOperationException if the data type is non-numeric
+     * @see ValueUpdate#createIncrement(Number)
+     */
+    public static FieldUpdate createIncrement(Field field, Number increment) {
+        return new FieldUpdate(field, ValueUpdate.createIncrement(increment));
+    }
+
+    /**
+     * Creates a new field update, with one encapsulated value update that increments a weight in a weighted set.
+     *
+     * @param field     the field to increment one of the weights of
+     * @param key the key whose weight in the weighted set to increment
+     * @param increment the number to increment by
+     * @return a FieldUpdate specifying the increment
+     * @throws IllegalArgumentException if key is not equal to the nested type of the weighted set
+     * @see ValueUpdate#createIncrement(Number)
+     * @see ValueUpdate#createMap(FieldValue, ValueUpdate)
+     */
+    public static FieldUpdate createIncrement(Field field, FieldValue key, Number increment) {
+        return new FieldUpdate(field, ValueUpdate.createIncrement(key, increment));
+    }
+
+    /**
+     * Creates a new field update, with one encapsulated value update that decrements a value.
+     * Note that the data type must be a numeric type.
+     *
+     * @param field     the field to decrement the value of
+     * @param decrement the number to decrement by
+     * @return a FieldUpdate specifying the decrement
+     * @throws UnsupportedOperationException if the data type is non-numeric
+     * @see ValueUpdate#createDecrement(Number)
+     */
+    public static FieldUpdate createDecrement(Field field, Number decrement) {
+        return new FieldUpdate(field, ValueUpdate.createDecrement(decrement));
+    }
+
+    /**
+     * Creates a new field update, with one encapsulated value update that decrements a weight in a weighted set.
+     *
+     * @param field     the field to decrement one of the weights of
+     * @param key the key whose weight in the weighted set to decrement
+     * @param decrement the number to decrement by
+     * @return a FieldUpdate specifying the decrement
+     * @throws IllegalArgumentException if key is not equal to the nested type of the weighted set
+     * @see ValueUpdate#createDecrement(Number)
+     * @see ValueUpdate#createMap(FieldValue, ValueUpdate)
+     */
+    public static FieldUpdate createDecrement(Field field, FieldValue key, Number decrement) {
+        return new FieldUpdate(field, ValueUpdate.createDecrement(key, decrement));
+    }
+
+    /**
+     * Creates a new field update, with one encapsulated value update that multiplies a value.
+     * Note that the data type must be a numeric type.
+     *
+     * @param field     the field to multiply the value of
+     * @param factor the number to multiply by
+     * @return a FieldUpdate specifying the multiplication
+     * @throws UnsupportedOperationException if the data type is non-numeric
+     * @see ValueUpdate#createMultiply(Number)
+     */
+    public static FieldUpdate createMultiply(Field field, Number factor) {
+        return new FieldUpdate(field, ValueUpdate.createMultiply(factor));
+    }
+
+    /**
+     * Creates a new field update, with one encapsulated value update that multiplies a weight in a weighted set.
+     *
+     * @param field     the field to multiply one of the weights of
+     * @param key the key whose weight in the weighted set to multiply
+     * @param factor the number to multiply by
+     * @return a FieldUpdate specifying the multiplication
+     * @throws IllegalArgumentException if key is not equal to the nested type of the weighted set
+     * @see ValueUpdate#createMultiply(Number)
+     * @see ValueUpdate#createMap(FieldValue, ValueUpdate)
+     */
+    public static FieldUpdate createMultiply(Field field, FieldValue key, Number factor) {
+        return new FieldUpdate(field, ValueUpdate.createMultiply(key, factor));
+    }
+
+    /**
+     * Creates a new field update, with one encapsulated value update that divides a value.
+     * Note that the data type must be a numeric type.
+     *
+     * @param field     the field to divide the value of
+     * @param divisor the number to divide by
+     * @return a FieldUpdate specifying the division
+     * @throws UnsupportedOperationException if the data type is non-numeric
+     * @see ValueUpdate#createDivide(Number)
+     */
+    public static FieldUpdate createDivide(Field field, Number divisor) {
+        return new FieldUpdate(field, ValueUpdate.createDivide(divisor));
+    }
+
+    /**
+     * Creates a new field update, with one encapsulated value update that divides a weight in a weighted set.
+     *
+     * @param field     the field to divide one of the weights of
+     * @param key the key whose weight in the weighted set to divide
+     * @param divisor the number to divide by
+     * @return a FieldUpdate specifying the division
+     * @throws IllegalArgumentException if key is not equal to the nested type of the weighted set
+     * @see ValueUpdate#createDivide(Number)
+     * @see ValueUpdate#createMap(FieldValue, ValueUpdate)
+     */
+    public static FieldUpdate createDivide(Field field, FieldValue key, Number divisor) {
+        return new FieldUpdate(field, ValueUpdate.createDivide(key, divisor));
+    }
+
+    /**
+     * Creates a new field update, with one encapsulated value update,
+     * that assigns a new value, completely overwriting the previous value. Note that it is possible to pass
+     * newValue=null to this method to remove the value completely.
+     *
+     * @param field    the Field to assign a new value to
+     * @param newValue the value to assign
+     * @return a FieldUpdate specifying the assignment
+     * @throws IllegalArgumentException if the runtime type of newValue does not match the type required by field
+     * @see com.yahoo.document.update.ValueUpdate#createAssign(FieldValue)
+     */
+    public static FieldUpdate createAssign(Field field, FieldValue newValue) {
+        return new FieldUpdate(field, ValueUpdate.createAssign(newValue));
+    }
+
+    /**
+     * Creates a new field update, with one encapsulated value update,
+     * that clears the value; see documentation for ClearValueUpdate to see behavior
+     * for the individual data types. Note that clearing the value is not the same
+     * clearing the field; this method leaves an empty value, whereas clearing the
+     * field will completely remove the value.
+     *
+     * @param field the field to clear the value of
+     * @return a FieldUpdate specifying the clearing
+     * @see com.yahoo.document.update.ClearValueUpdate
+     * @see ValueUpdate#createClear()
+     * @see FieldUpdate#createClearField(com.yahoo.document.Field)
+     */
+    public static FieldUpdate createClear(Field field) {
+        return new FieldUpdate(field, ValueUpdate.createClear());
+    }
+
+    /**
+     * Creates a new field update, with one encapsulated value update, which
+     * is able to map an update to a value to a subvalue in an array or a
+     * weighted set. If this update is to be applied to an array, the value parameter must be an integer specifying
+     * the index in the array that the update parameter is to be applied to, and the update parameter must be
+     * compatible with the sub-datatype of the array. If this update is to be applied on a weighted set, the value
+     * parameter must be the key in the set that the update parameter is to be applied to, and the update parameter
+     * must be compatible with the INT data type.
+     *
+     * @param field the field to modify the subvalue of
+     * @param value the index in case of array, or key in case of weighted set
+     * @param update the update to apply to the target sub-value
+     * @throws IllegalArgumentException in case data type is an array type and value is not an Integer; in case data type is a weighted set type and value is not equal to the nested type of the weighted set; or the encapsulated update throws such an exception
+     * @throws UnsupportedOperationException if superType is a single-value type, or anything else than array or weighted set; or the encapsulated update throws such an exception
+     * @return a FieldUpdate specifying the sub-update
+     * @see ValueUpdate#createMap(FieldValue, ValueUpdate)
+     */
+    public static FieldUpdate createMap(Field field, FieldValue value, ValueUpdate update) {
+        return new FieldUpdate(field, ValueUpdate.createMap(value, update));
+    }
+
+    /**
+     * Creates a new field update, with one encapsulated value update,
+     * specifying the removal of a value from an array or a key/weight from a weighted set.
+     *
+     * @param field the field to remove a value from
+     * @param value the value to remove from the array, or key to remove from the weighted set
+     * @return a FieldUpdate specifying the removal
+     * @throws IllegalArgumentException      if the runtime type of newValue does not match the type required
+     * @throws UnsupportedOperationException if the field type is not array or weighted set
+     * @see ValueUpdate#createRemove(FieldValue)
+     */
+    public static FieldUpdate createRemove(Field field, FieldValue value) {
+        return new FieldUpdate(field, ValueUpdate.createRemove(value));
+    }
+
+    /**
+     * Creates a new field update, with encapsulated value updates,
+     * specifying the removal of all values in a given list from an array or weighted set. Note that this method
+     * is just a convenience method, it simply iterates
+     * through the list and creates value updates by calling createRemove() for each element.
+     *
+     * @param field the field to remove values from
+     * @param values a List containing the values to remove
+     * @return a FieldUpdate specifying the removal
+     * @throws IllegalArgumentException      if the runtime type of values does not match the type required by field
+     * @throws UnsupportedOperationException if the field type is not array or weighted set
+     * @see ValueUpdate#createRemoveAll(java.util.List)
+     */
+    public static FieldUpdate createRemoveAll(Field field, List<? extends FieldValue> values) {
+        return new FieldUpdate(field, ValueUpdate.createRemoveAll(values));
+    }
+
+    /**
+     * Creates a new field update, with encapsulated value updates,
+     * specifying the removal of all values in a given list from an array or weighted set. Note that this method
+     * is just a convenience method, it simply iterates
+     * through the list and creates value updates by calling createRemove() for each element.
+     *
+     * @param field the field to remove values from
+     * @param values a List containing the values to remove
+     * @return a FieldUpdate specifying the removal
+     * @throws IllegalArgumentException      if the runtime type of values does not match the type required by field
+     * @throws UnsupportedOperationException if the field type is not array or weighted set
+     * @see ValueUpdate#createRemoveAll(java.util.List)
+     */
+    public static FieldUpdate createRemoveAll(Field field, WeightedSet<? extends FieldValue> values) {
+        return new FieldUpdate(field, ValueUpdate.createRemoveAll(values));
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/update/MapValueUpdate.java b/document/src/main/java/com/yahoo/document/update/MapValueUpdate.java
new file mode 100644
index 00000000000..37b4329c934
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/update/MapValueUpdate.java
@@ -0,0 +1,128 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.update;
+
+import com.yahoo.document.ArrayDataType;
+import com.yahoo.document.DataType;
+import com.yahoo.document.Field;
+import com.yahoo.document.StructuredDataType;
+import com.yahoo.document.WeightedSetDataType;
+import com.yahoo.document.datatypes.Array;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.IntegerFieldValue;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.document.datatypes.WeightedSet;
+import com.yahoo.document.serialization.DocumentUpdateWriter;
+
+/**
+ * <p>Value update that represents performing an encapsulated value update on a subvalue. Currently, there are
+ * two multi-value data types in Vespa, <em>array</em> and <em>weighted set</em>. </p>
+ *
+ * <ul>
+ * <li>For an array, the value
+ * must be an Integer, and the update must represent a legal operation on the subtype of the array. </li>
+ * <li>For a
+ * weighted set, the value must be a key of the same type as the subtype of the weighted set, and the update
+ * must represent a legal operation on an integer value.</li>
+ * </ul>
+ *
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class MapValueUpdate extends ValueUpdate {
+    protected FieldValue value;
+    protected ValueUpdate update;
+
+    public MapValueUpdate(FieldValue value, ValueUpdate update) {
+        super(ValueUpdateClassID.MAP);
+        this.value = value;
+        this.update = update;
+    }
+
+    /** Returns the key of the nested update */
+    public FieldValue getValue() {
+        return value;
+    }
+
+    /** Sets the key of the nested update */
+    public void setValue(FieldValue value) {
+        this.value=value;
+    }
+
+    public ValueUpdate getUpdate() {
+        return update;
+    }
+
+    @Override
+    public FieldValue applyTo(FieldValue fval) {
+        if (fval instanceof Array) {
+            Array array = (Array) fval;
+            FieldValue element = array.getFieldValue(((IntegerFieldValue) value).getInteger());
+            element = update.applyTo(element);
+            array.set(((IntegerFieldValue) value).getInteger(), element);
+        } else if (fval instanceof WeightedSet) {
+            WeightedSet wset = (WeightedSet) fval;
+            WeightedSetDataType wtype = wset.getDataType();
+            Integer weight = wset.get(value);
+            if (weight == null) {
+                if (wtype.createIfNonExistent() && update instanceof ArithmeticValueUpdate) {
+                    weight = 0;
+                } else {
+                    return fval;
+                }
+            }
+            weight = (Integer) update.applyTo(new IntegerFieldValue(weight)).getWrappedValue();
+            wset.put((FieldValue) value, weight);
+            if (wtype.removeIfZero() && update instanceof ArithmeticValueUpdate && weight == 0) {
+                wset.remove(value);
+            }
+        }
+        return fval;
+    }
+
+    @Override
+    protected void checkCompatibility(DataType fieldType) {
+        if (fieldType instanceof ArrayDataType) {
+            if (!(value instanceof IntegerFieldValue)) {
+                throw new IllegalArgumentException("Expected integer, got " + value.getClass().getName() + ".");
+            }
+            update.checkCompatibility(((ArrayDataType)fieldType).getNestedType());
+        } else if (fieldType instanceof WeightedSetDataType) {
+            ((WeightedSetDataType)fieldType).getNestedType().createFieldValue().assign(value);
+            update.checkCompatibility(DataType.INT);
+        } else if (fieldType instanceof StructuredDataType) {
+            if (!(value instanceof StringFieldValue)) {
+                throw new IllegalArgumentException("Expected string, got " + value.getClass().getName() + ".");
+            }
+            Field field = ((StructuredDataType)fieldType).getField(((StringFieldValue)value).getString());
+            if (field == null) {
+                throw new IllegalArgumentException("Field '" + value + "' not found.");
+            }
+            update.checkCompatibility(field.getDataType());
+        } else {
+            throw new UnsupportedOperationException("Field type " + fieldType.getName() + " not supported.");
+        }
+    }
+
+
+    @Override
+    public void serialize(DocumentUpdateWriter data, DataType superType) {
+        data.write(this, superType);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        return o instanceof MapValueUpdate && super.equals(o) && value.equals(((MapValueUpdate) o).value) &&
+                update.equals(((MapValueUpdate) o).update);
+    }
+
+    @Override
+    public int hashCode() {
+        return super.hashCode() + value.hashCode() + update.hashCode();
+    }
+
+    @Override
+    public String toString() {
+        return super.toString() + " " + value + " " + update;
+    }
+
+}
+
diff --git a/document/src/main/java/com/yahoo/document/update/RemoveValueUpdate.java b/document/src/main/java/com/yahoo/document/update/RemoveValueUpdate.java
new file mode 100644
index 00000000000..e83d19803b1
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/update/RemoveValueUpdate.java
@@ -0,0 +1,70 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.update;
+
+import com.yahoo.document.CollectionDataType;
+import com.yahoo.document.DataType;
+import com.yahoo.document.datatypes.CollectionFieldValue;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.serialization.DocumentUpdateWriter;
+
+/**
+ * <p>Value update representing a removal of a value (and its associated weight, if any)
+ * from a multi-valued data type.</p>
+ * Deprecated: Use RemoveFieldPathUpdate instead.
+ *
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class RemoveValueUpdate extends ValueUpdate {
+    protected FieldValue value;
+
+    public RemoveValueUpdate(FieldValue value) {
+        super(ValueUpdateClassID.REMOVE);
+        this.value = value;
+    }
+
+    /** Sets the key this should remove */
+    public FieldValue getValue() { return value; }
+
+    public void setValue(FieldValue value) { this.value=value; }
+
+    @Override
+    public FieldValue applyTo(FieldValue fval) {
+        if (fval instanceof CollectionFieldValue) {
+            CollectionFieldValue val = (CollectionFieldValue) fval;
+            val.removeValue(value);
+        }
+        return fval;
+    }
+
+    @Override
+    protected void checkCompatibility(DataType fieldType) {
+        if (!(fieldType instanceof CollectionDataType)) {
+            throw new UnsupportedOperationException("Expected collection, got " + fieldType.getName() + ".");
+        }
+        fieldType = ((CollectionDataType)fieldType).getNestedType();
+        if (value != null && !value.getDataType().equals(fieldType)) {
+            throw new IllegalArgumentException("Expected " + fieldType.getName() + ", got " +
+                                               value.getDataType().getName());
+        }
+    }
+
+    @Override
+    public void serialize(DocumentUpdateWriter data, DataType superType) {
+        data.write(this, superType);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        return o instanceof RemoveValueUpdate && super.equals(o) && value.equals(((RemoveValueUpdate) o).value);
+    }
+
+    @Override
+    public int hashCode() {
+        return super.hashCode() + value.hashCode();
+    }
+
+    @Override
+    public String toString() {
+        return super.toString() + " " + value;
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/update/ValueUpdate.java b/document/src/main/java/com/yahoo/document/update/ValueUpdate.java
new file mode 100644
index 00000000000..5c6f5b22dbe
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/update/ValueUpdate.java
@@ -0,0 +1,364 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.update;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.WeightedSet;
+import com.yahoo.document.serialization.DocumentUpdateWriter;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+/**
+ * A value update represents some action to perform to a value.
+ *
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ * @see com.yahoo.document.update.FieldUpdate
+ * @see com.yahoo.document.DocumentUpdate
+ * @see AddValueUpdate
+ * @see com.yahoo.document.update.ArithmeticValueUpdate
+ * @see com.yahoo.document.update.AssignValueUpdate
+ * @see com.yahoo.document.update.ClearValueUpdate
+ * @see com.yahoo.document.update.MapValueUpdate
+ * @see com.yahoo.document.update.RemoveValueUpdate
+ */
+public abstract class ValueUpdate<T extends FieldValue> {
+
+    protected ValueUpdateClassID valueUpdateClassID;
+
+    protected ValueUpdate(ValueUpdateClassID valueUpdateClassID) {
+        this.valueUpdateClassID = valueUpdateClassID;
+    }
+
+    /**
+     * Returns the valueUpdateClassID of this value update.
+     *
+     * @return the valueUpdateClassID of this ValueUpdate
+     */
+    public ValueUpdateClassID getValueUpdateClassID() {
+        return valueUpdateClassID;
+    }
+
+    protected abstract void checkCompatibility(DataType fieldType);
+
+    public abstract void serialize(DocumentUpdateWriter data, DataType superType);
+
+    @Override
+    public boolean equals(Object o) {
+        return o instanceof ValueUpdate && valueUpdateClassID == ((ValueUpdate) o).valueUpdateClassID;
+    }
+
+    @Override
+    public int hashCode() {
+        return valueUpdateClassID.id;
+    }
+
+    @Override
+    public String toString() {
+        return valueUpdateClassID.name;
+    }
+
+    public abstract FieldValue applyTo(FieldValue oldValue);
+
+    /**
+     * Creates a new value update specifying an addition of a value to an array or a key to a weighted set (with default weight 1).
+     *
+     * @param value the value to add to the array, or key to add to the weighted set
+     * @return a ValueUpdate specifying the addition
+     * @throws IllegalArgumentException      if the runtime type of newValue does not match the type required
+     * @throws UnsupportedOperationException if the field type is not array or weighted set
+     */
+    public static ValueUpdate createAdd(FieldValue value) {
+        return new AddValueUpdate(value);
+    }
+
+    /**
+     * Creates a new value update specifying an addition of a key (with a specified weight) to a weighted set. If this
+     * method is used on an array data type, the weight will be omitted.
+     *
+     * @param key    the key to add
+     * @param weight the weight to associate with the given key
+     * @return a ValueUpdate specifying the addition
+     * @throws IllegalArgumentException      if the runtime type of key does not match the type required
+     * @throws UnsupportedOperationException if the field type is not array or weighted set
+     */
+    public static ValueUpdate createAdd(FieldValue key, Integer weight) {
+        return new AddValueUpdate(key, weight);
+    }
+
+    /**
+     * Creates a new value update
+     * specifying an addition of all values in a given list to an array. If this method is used on a weighted set data
+     * type, the default weights will be 1. Note that this method is just a convenience method, it simply iterates
+     * through the list and creates value updates by calling createAdd() for each element.
+     *
+     * @param values a List containing the values to add
+     * @return a List of ValueUpdates specifying the addition
+     * @throws IllegalArgumentException      if the runtime type of values does not match the type required
+     * @throws UnsupportedOperationException if the field type is not array or weighted set
+     * @see ValueUpdate#createAdd(FieldValue)
+     */
+    public static List<ValueUpdate> createAddAll(List<? extends FieldValue> values) {
+        List<ValueUpdate> vupds = new ArrayList<>();
+        for (FieldValue value : values) {
+            vupds.add(ValueUpdate.createAdd(value));
+        }
+        return vupds;
+    }
+
+    /**
+     * Creates a new value update
+     * specifying an addition of all key/weight pairs in a weighted set to a weighted set. If this method
+     * is used on an array data type, the weights will be omitted. Note that this method is just a convenience method,
+     * it simply iterates through the set and creates value updates by calling createAdd() for each element.
+     *
+     * @param set a WeightedSet containing the key/weight pairs to add
+     * @return a ValueUpdate specifying the addition
+     * @throws IllegalArgumentException      if the runtime type of values does not match the type required
+     * @throws UnsupportedOperationException if the field type is not weighted set or array
+     * @see ValueUpdate#createAdd(FieldValue, Integer)
+     */
+    public static List<ValueUpdate> createAddAll(WeightedSet<? extends FieldValue> set) {
+        List<ValueUpdate> vupds = new ArrayList<>();
+        Iterator<? extends FieldValue> it = set.fieldValueIterator();
+        while (it.hasNext()) {
+            FieldValue key = it.next();
+            vupds.add(ValueUpdate.createAdd(key, set.get(key)));
+        }
+        return vupds;
+    }
+
+    /**
+     * Creates a new value update that increments a value. Note that the data type must be a numeric
+     * type.
+     *
+     * @param increment the number to increment by
+     * @return a ValueUpdate specifying the increment
+     * @throws UnsupportedOperationException if the data type is non-numeric
+     */
+    public static ValueUpdate createIncrement(Number increment) {
+        return new ArithmeticValueUpdate(ArithmeticValueUpdate.Operator.ADD, increment);
+    }
+
+    /**
+     * Creates a new value update that increments a weight in a weighted set. Note that this method is just a convenience
+     * method, it simply creates an increment value update by calling createIncrement() and then creates a map value
+     * update by calling createMap() with the key and the increment value update as parameters.
+     *
+     * @param key the key whose weight in the weighted set to increment
+     * @param increment the number to increment by
+     * @return a ValueUpdate specifying the increment
+     * @see ValueUpdate#createIncrement(Number)
+     * @see ValueUpdate#createMap(FieldValue, ValueUpdate)
+     */
+    public static ValueUpdate createIncrement(FieldValue key, Number increment) {
+        return createMap(key, createIncrement(increment));
+    }
+
+    /**
+     * Creates a new value update that decrements a value. Note that the data type must be a numeric
+     * type.
+     *
+     * @param decrement the number to decrement by
+     * @return a ValueUpdate specifying the decrement
+     * @throws UnsupportedOperationException if the data type is non-numeric
+     */
+    public static ValueUpdate createDecrement(Number decrement) {
+        return new ArithmeticValueUpdate(ArithmeticValueUpdate.Operator.SUB, decrement);
+    }
+
+    /**
+     * Creates a new value update that decrements a weight in a weighted set. Note that this method is just a convenience
+     * method, it simply creates a decrement value update by calling createDecrement() and then creates a map value
+     * update by calling createMap() with the key and the decrement value update as parameters.
+     *
+     * @param key the key whose weight in the weighted set to decrement
+     * @param decrement the number to decrement by
+     * @return a ValueUpdate specifying the decrement
+     * @see ValueUpdate#createDecrement(Number)
+     * @see ValueUpdate#createMap(FieldValue, ValueUpdate)
+     */
+    public static ValueUpdate createDecrement(FieldValue key, Number decrement) {
+        return createMap(key, createDecrement(decrement));
+    }
+
+    /**
+     * Creates a new value update that multiplies a value. Note that the data type must be a numeric
+     * type.
+     *
+     * @param factor the number to multiply by
+     * @return a ValueUpdate specifying the multiplication
+     * @throws UnsupportedOperationException if the data type is non-numeric
+     */
+    public static ValueUpdate createMultiply(Number factor) {
+        return new ArithmeticValueUpdate(ArithmeticValueUpdate.Operator.MUL, factor);
+    }
+
+    /**
+     * Creates a new value update that multiplies a weight in a weighted set. Note that this method is just a convenience
+     * method, it simply creates a multiply value update by calling createMultiply() and then creates a map value
+     * update by calling createMap() with the key and the multiply value update as parameters.
+     *
+     * @param key the key whose weight in the weighted set to multiply
+     * @param factor the number to multiply by
+     * @return a ValueUpdate specifying the multiplication
+     * @see ValueUpdate#createMultiply(Number)
+     * @see ValueUpdate#createMap(FieldValue, ValueUpdate)
+     */
+    public static ValueUpdate createMultiply(FieldValue key, Number factor) {
+        return createMap(key, createMultiply(factor));
+    }
+
+
+    /**
+     * Creates a new value update that divides a value. Note that the data type must be a numeric
+     * type.
+     *
+     * @param divisor the number to divide by
+     * @return a ValueUpdate specifying the division
+     * @throws UnsupportedOperationException if the data type is non-numeric
+     */
+    public static ValueUpdate createDivide(Number divisor) {
+        return new ArithmeticValueUpdate(ArithmeticValueUpdate.Operator.DIV, divisor);
+    }
+
+    /**
+     * Creates a new value update that divides a weight in a weighted set. Note that this method is just a convenience
+     * method, it simply creates a divide value update by calling createDivide() and then creates a map value
+     * update by calling createMap() with the key and the divide value update as parameters.
+     *
+     * @param key the key whose weight in the weighted set to divide
+     * @param divisor the number to divide by
+     * @return a ValueUpdate specifying the division
+     * @see ValueUpdate#createDivide(Number)
+     * @see ValueUpdate#createMap(FieldValue, ValueUpdate)
+     */
+    public static ValueUpdate createDivide(FieldValue key, Number divisor) {
+        return createMap(key, createDivide(divisor));
+    }
+
+    /**
+     * Creates a new value update that assigns a new value, completely overwriting
+     * the previous value.
+     *
+     * @param newValue the value to assign
+     * @return a ValueUpdate specifying the assignment
+     * @throws IllegalArgumentException if the runtime type of newValue does not match the type required
+     */
+    public static ValueUpdate createAssign(FieldValue newValue) {
+        return new AssignValueUpdate(newValue);
+    }
+
+    /**
+     * Creates a new value update that clears the field fromthe document.
+     *
+     * @return a ValueUpdate specifying the removal
+     */
+    public static ValueUpdate createClear() {
+        return new ClearValueUpdate();
+    }
+
+    /**
+     * Creates a map value update, which is able to map an update to a value to a subvalue in an array or a
+     * weighted set. If this update is to be applied to an array, the value parameter must be an integer specifying
+     * the index in the array that the update parameter is to be applied to, and the update parameter must be
+     * compatible with the sub-datatype of the array. If this update is to be applied on a weighted set, the value
+     * parameter must be the key in the set that the update parameter is to be applied to, and the update parameter
+     * must be compatible with the INT data type.
+     *
+     * @param value the index in case of array, or key in case of weighted set
+     * @param update the update to apply to the target sub-value
+     * @throws IllegalArgumentException in case data type is an array type and value is not an Integer; in case data type is a weighted set type and value is not equal to the nested type of the weighted set; or the encapsulated update throws such an exception
+     * @throws UnsupportedOperationException if superType is a single-value type, or anything else than array or weighted set; or the encapsulated update throws such an exception
+     * @return a ValueUpdate specifying the sub-update
+     */
+    public static ValueUpdate createMap(FieldValue value, ValueUpdate update) {
+        return new MapValueUpdate(value, update);
+    }
+
+    /**
+     * Creates a new value update specifying the removal of a value from an array or a key/weight from a weighted set.
+     *
+     * @param value the value to remove from the array, or key to remove from the weighted set
+     * @return a ValueUpdate specifying the removal
+     * @throws IllegalArgumentException      if the runtime type of newValue does not match the type required
+     * @throws UnsupportedOperationException if the field type is not array or weighted set
+     */
+    public static ValueUpdate createRemove(FieldValue value) {
+        return new RemoveValueUpdate(value);
+    }
+
+    /**
+     * Creates a new value update
+     * specifying the removal of all values in a given list from an array or weighted set. Note that this method
+     * is just a convenience method, it simply iterates
+     * through the list and creates value updates by calling createRemove() for each element.
+     *
+     * @param values a List containing the values to remove
+     * @return a List of ValueUpdates specifying the removal
+     * @throws IllegalArgumentException      if the runtime type of values does not match the type required
+     * @throws UnsupportedOperationException if the field type is not array or weighted set
+     * @see ValueUpdate#createRemove(FieldValue)
+     */
+    public static List<ValueUpdate> createRemoveAll(List<? extends FieldValue> values) {
+        List<ValueUpdate> vupds = new ArrayList<>();
+        for (FieldValue value : values) {
+            vupds.add(ValueUpdate.createRemove(value));
+        }
+        return vupds;
+    }
+
+    /**
+     * Creates a new value update
+     * specifying the removal of all values in a given list from an array or weighted set. Note that this method
+     * is just a convenience method, it simply iterates
+     * through the list and creates value updates by calling createRemove() for each element.
+     *
+     * @param values a List containing the values to remove
+     * @return a List of ValueUpdates specifying the removal
+     * @throws IllegalArgumentException      if the runtime type of values does not match the type required
+     * @throws UnsupportedOperationException if the field type is not array or weighted set
+     * @see ValueUpdate#createRemove(FieldValue)
+     */
+    public static List<ValueUpdate> createRemoveAll(WeightedSet<? extends FieldValue> values) {
+        List<ValueUpdate> vupds = new ArrayList<>();
+        for (FieldValue value : values.keySet()) {
+            vupds.add(ValueUpdate.createRemove(value));
+        }
+        return vupds;
+    }
+
+    /** Returns the primary "value" of this update, or null if this kind of update has no value */
+    public abstract T getValue();
+
+    /** Sets the value of this. Ignored by update who have no value */
+    public abstract void setValue(T value);
+
+    public enum ValueUpdateClassID {
+        //DO NOT change anything here unless you change src/vespa/document/util/identifiableid.h as well!!
+        ADD(25, "add"),
+        ARITHMETIC(26, "arithmetic"),
+        ASSIGN(27, "assign"),
+        CLEAR(28, "clear"),
+        MAP(29, "map"),
+        REMOVE(30, "remove");
+
+        public final int id;
+        public final String name;
+
+        ValueUpdateClassID(int id, String name) {
+            this.id = 0x1000 + id;
+            this.name = name;
+        }
+
+        public static ValueUpdateClassID getID(int id) {
+            for (ValueUpdateClassID vucid : ValueUpdateClassID.values()) {
+                if (vucid.id == id) {
+                    return vucid;
+                }
+            }
+            return null;
+        }
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/update/package-info.java b/document/src/main/java/com/yahoo/document/update/package-info.java
new file mode 100644
index 00000000000..3699dec32ef
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/update/package-info.java
@@ -0,0 +1,7 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+@PublicApi
+package com.yahoo.document.update;
+
+import com.yahoo.api.annotations.PublicApi;
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/document/src/main/java/com/yahoo/documentmodel/.gitignore b/document/src/main/java/com/yahoo/documentmodel/.gitignore
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/document/src/main/java/com/yahoo/documentmodel/.gitignore
diff --git a/document/src/main/java/com/yahoo/vespaxmlparser/FeedReader.java b/document/src/main/java/com/yahoo/vespaxmlparser/FeedReader.java
new file mode 100644
index 00000000000..e97bd43d9bf
--- /dev/null
+++ b/document/src/main/java/com/yahoo/vespaxmlparser/FeedReader.java
@@ -0,0 +1,21 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespaxmlparser;
+
+import com.yahoo.vespaxmlparser.VespaXMLFeedReader.Operation;
+
+/**
+ * Minimal interface for reading operations from a stream for a feeder.
+ *
+ * Interface extracted from VespaXMLFeedReader to enable JSON feeding.
+ *
+ * @author steinar
+ */
+public interface FeedReader {
+
+    /**
+     * Reads the next operation from the stream.
+     * @param operation The operation to fill in. Operation is unchanged if none was found.
+     */
+    public abstract void read(Operation operation) throws Exception;
+
+}
+\ No newline at end of file
diff --git a/document/src/main/java/com/yahoo/vespaxmlparser/VespaXMLDocumentReader.java b/document/src/main/java/com/yahoo/vespaxmlparser/VespaXMLDocumentReader.java
new file mode 100644
index 00000000000..a5ea5983e29
--- /dev/null
+++ b/document/src/main/java/com/yahoo/vespaxmlparser/VespaXMLDocumentReader.java
@@ -0,0 +1,49 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespaxmlparser;
+
+import com.yahoo.document.Document;
+import com.yahoo.document.DocumentTypeManager;
+import com.yahoo.document.serialization.DocumentReader;
+
+import javax.xml.stream.XMLStreamReader;
+import java.io.InputStream;
+
+/**
+ * XML parser that reads Vespa documents from an XML stream.
+ *
+ * @author thomasg
+ */
+public class VespaXMLDocumentReader extends VespaXMLFieldReader implements DocumentReader {
+
+    /**
+     * Creates a reader that reads from the given file.
+     */
+    public VespaXMLDocumentReader(String fileName, DocumentTypeManager docTypeManager) throws Exception {
+        super(fileName, docTypeManager);
+    }
+
+    /**
+     * Creates a reader that reads from the given stream.
+     */
+    public VespaXMLDocumentReader(InputStream stream, DocumentTypeManager docTypeManager) throws Exception {
+        super(stream, docTypeManager);
+    }
+
+    /**
+     * Creates a reader that reads using the given reader. This is useful if the document is part of a greater
+     * XML stream.
+     */
+    public VespaXMLDocumentReader(XMLStreamReader reader, DocumentTypeManager docTypeManager) {
+        super(reader, docTypeManager);
+    }
+
+    /**
+     * Reads one document from the stream. Function assumes that the current element in the stream is
+     * the start tag for the document.
+     *
+     * @param document the document to be read
+     */
+    public void read(Document document) {
+        read(null, document);
+    }
+}
diff --git a/document/src/main/java/com/yahoo/vespaxmlparser/VespaXMLFeedReader.java b/document/src/main/java/com/yahoo/vespaxmlparser/VespaXMLFeedReader.java
new file mode 100644
index 00000000000..0c8b9b22961
--- /dev/null
+++ b/document/src/main/java/com/yahoo/vespaxmlparser/VespaXMLFeedReader.java
@@ -0,0 +1,313 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespaxmlparser;
+
+import com.yahoo.document.Document;
+import com.yahoo.document.DocumentId;
+import com.yahoo.document.DocumentTypeManager;
+import com.yahoo.document.DocumentUpdate;
+import com.yahoo.document.TestAndSetCondition;
+
+import javax.xml.stream.XMLStreamException;
+import javax.xml.stream.XMLStreamReader;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+import java.util.Optional;
+
+/**
+ * XML parser for Vespa document XML.
+ *
+ * Parses an entire document "feed", which consists of a vespafeed element containing
+ * zero or more instances of documents, updates or removes.
+ *
+ * Standard usage is to create an Operation object and call read(Operation) until
+ * operation.getType() returns OperationType.INVALID.
+ *
+ * If you are looking to parse only a single document or update, use VespaXMLDocumentReader
+ * or VespaXMLUpdateReader respectively.
+ */
+public class VespaXMLFeedReader extends VespaXMLReader implements FeedReader {
+
+    /**
+     * Creates a reader that reads from the given file.
+     */
+    public VespaXMLFeedReader(String fileName, DocumentTypeManager docTypeManager) throws Exception {
+        super(fileName, docTypeManager);
+        readInitial();
+    }
+
+    /**
+     * Creates a reader that reads from the given stream.
+     */
+    public VespaXMLFeedReader(InputStream stream, DocumentTypeManager docTypeManager) throws Exception {
+        super(stream, docTypeManager);
+        readInitial();
+    }
+
+    /**
+     * Creates a reader that uses the given reader to read - this can be used if the vespa feed
+     * is part of a larger XML document.
+     */
+    public VespaXMLFeedReader(XMLStreamReader reader, DocumentTypeManager manager) throws Exception {
+        super(reader, manager);
+        readInitial();
+    }
+
+    /**
+     * Skips the initial "vespafeed" tag.
+     */
+    void readInitial() throws Exception {
+        boolean found = false;
+
+        while (reader.hasNext()) {
+            int type = reader.next();
+            if (type == XMLStreamReader.START_ELEMENT) {
+                if ("vespafeed".equals(reader.getName().toString())) {
+                    found = true;
+                    break;
+                }
+            }
+        }
+
+        if (!found) {
+            throw newDeserializeException("Feed information must be contained within a \"vespafeed\" element");
+        }
+    }
+
+    public enum OperationType {
+        DOCUMENT,
+        REMOVE,
+        UPDATE,
+        INVALID
+    }
+
+    /**
+     * Represents a feed operation found by the parser. Can be one of the following types:
+     * - getType() == DOCUMENT: getDocument() is valid.
+     * - getType() == REMOVE: getRemove() is valid.
+     * - getType() == UPDATE: getUpdate() is valid.
+     */
+    public static class Operation {
+
+        private OperationType type;
+        private Document doc;
+        private DocumentId remove;
+        private DocumentUpdate docUpdate;
+        private FeedOperation feedOperation;
+        private TestAndSetCondition condition;
+
+        public Operation() {
+            setInvalid();
+        }
+
+        public void setInvalid() {
+            type = OperationType.INVALID;
+            doc = null;
+            remove = null;
+            docUpdate = null;
+            feedOperation = null;
+            condition = null;
+        }
+
+        public OperationType getType() {
+            return type;
+        }
+
+        public Document getDocument() {
+            return doc;
+        }
+
+        public void setDocument(Document doc) {
+            this.type = OperationType.DOCUMENT;
+            this.doc = doc;
+        }
+
+        public DocumentId getRemove() {
+            return remove;
+        }
+
+        public void setRemove(DocumentId remove) {
+            this.type = OperationType.REMOVE;
+            this.remove = remove;
+        }
+
+        public DocumentUpdate getDocumentUpdate() {
+            return docUpdate;
+        }
+
+        public void setDocumentUpdate(DocumentUpdate docUpdate) {
+            this.type = OperationType.UPDATE;
+            this.docUpdate = docUpdate;
+        }
+
+        public FeedOperation getFeedOperation() {
+            return feedOperation;
+        }
+
+        public void setCondition(TestAndSetCondition condition) {
+            this.condition = condition;
+        }
+
+        public TestAndSetCondition getCondition() {
+            return condition;
+        }
+
+        @Override
+        public String toString() {
+            return "Operation{" +
+                   "type=" + type +
+                   ", doc=" + doc +
+                   ", remove=" + remove +
+                   ", docUpdate=" + docUpdate +
+                   ", feedOperation=" + feedOperation +
+                   '}';
+        }
+    }
+
+    public static class FeedOperation {
+
+        private String name;
+        private Integer generation;
+        private Integer increment;
+
+        public String getName() {
+            return name;
+        }
+
+        public void setName(String name) {
+            this.name = name;
+        }
+
+        public Integer getGeneration() {
+            return generation;
+        }
+
+        public void setGeneration(int generation) {
+            this.generation = generation;
+        }
+
+        public Integer getIncrement() {
+            return increment;
+        }
+
+        public void setIncrement(int increment) {
+            this.increment = increment;
+        }
+    }
+
+    /**
+     * <p>Reads all operations from the XML stream and puts into a list. Note
+     * that if the XML stream is large, this may cause out of memory errors, so
+     * make sure to use this only with small streams.</p>
+     *
+     * @return The list of all read operations.
+     */
+    public List<Operation> readAll() throws Exception {
+        List<Operation> list = new ArrayList<Operation>();
+        while (true) {
+            Operation op = new Operation();
+            read(op);
+            if (op.getType() == OperationType.INVALID) {
+                return list;
+            } else {
+                list.add(op);
+            }
+        }
+    }
+
+    /* (non-Javadoc)
+     * @see com.yahoo.vespaxmlparser.FeedReader#read(com.yahoo.vespaxmlparser.VespaXMLFeedReader.Operation)
+     */
+    @Override
+    public void read(Operation operation) throws Exception {
+        String startTag = null;
+        operation.setInvalid();
+
+        try {
+            while (reader.hasNext()) {
+                int type = reader.next();
+
+                if (type == XMLStreamReader.START_ELEMENT) {
+                    startTag = reader.getName().toString();
+
+                    if ("document".equals(startTag)) {
+                        VespaXMLDocumentReader documentReader = new VespaXMLDocumentReader(reader, docTypeManager);
+                        Document document = new Document(documentReader);
+                        operation.setDocument(document);
+                        operation.setCondition(TestAndSetCondition.fromConditionString(documentReader.getCondition()));
+                        return;
+                    } else if ("update".equals(startTag)) {
+                        VespaXMLUpdateReader updateReader = new VespaXMLUpdateReader(reader, docTypeManager);
+                        DocumentUpdate update = new DocumentUpdate(updateReader);
+                        operation.setDocumentUpdate(update);
+                        operation.setCondition(TestAndSetCondition.fromConditionString(updateReader.getCondition()));
+                        return;
+                    } else if ("remove".equals(startTag)) {
+                        boolean documentIdFound = false;
+
+                        Optional<String> condition = Optional.empty();
+                        for (int i = 0; i < reader.getAttributeCount(); i++) {
+                            final String attributeName = reader.getAttributeName(i).toString();
+                            if ("documentid".equals(attributeName) || "id".equals(attributeName)) {
+                                operation.setRemove(new DocumentId(reader.getAttributeValue(i)));
+                                documentIdFound = true;
+                            } else if ("condition".equals(attributeName)) {
+                                condition = Optional.of(reader.getAttributeValue(i));
+                            }
+                        }
+
+                        if (!documentIdFound) {
+                            throw newDeserializeException("Missing \"documentid\" attribute for remove operation");
+                        }
+
+                        operation.setCondition(TestAndSetCondition.fromConditionString(condition));
+
+                        return;
+                    } else {
+                        throw newDeserializeException("Element \"" + startTag + "\" not allowed in this context");
+                    }
+                }
+            }
+        } catch (XMLStreamException e) {
+            throw(e);
+            // Skip to end of current tag with other exceptions.
+        } catch (Exception e) {
+            try {
+                if (startTag != null) {
+                    skipToEnd(startTag);
+                }
+            } catch (Exception ignore) {
+            }
+
+            throw(e);
+        }
+    }
+
+    public void read(FeedOperation fo) throws XMLStreamException {
+        while (reader.hasNext()) {
+            int type = reader.next();
+
+            if (type == XMLStreamReader.START_ELEMENT) {
+                if ("name".equals(reader.getName().toString())) {
+                    fo.setName(reader.getElementText().toString());
+                    skipToEnd("name");
+                } else if ("generation".equals(reader.getName().toString())) {
+                    fo.setGeneration(Integer.parseInt(reader.getElementText().toString()));
+                    skipToEnd("generation");
+                } else if ("increment".equals(reader.getName().toString())) {
+                    String text = reader.getElementText();
+                    if ("autodetect".equals(text)) {
+                        fo.setIncrement(-1);
+                    } else {
+                        fo.setIncrement(Integer.parseInt(text));
+                    }
+                    skipToEnd("increment");
+                }
+            } else if (type == XMLStreamReader.END_ELEMENT) {
+                return;
+            }
+        }
+    }
+
+}
diff --git a/document/src/main/java/com/yahoo/vespaxmlparser/VespaXMLFieldReader.java b/document/src/main/java/com/yahoo/vespaxmlparser/VespaXMLFieldReader.java
new file mode 100644
index 00000000000..cdc676eca5f
--- /dev/null
+++ b/document/src/main/java/com/yahoo/vespaxmlparser/VespaXMLFieldReader.java
@@ -0,0 +1,520 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespaxmlparser;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.Document;
+import com.yahoo.document.DocumentId;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.DocumentTypeManager;
+import com.yahoo.document.Field;
+import com.yahoo.document.MapDataType;
+import com.yahoo.document.PositionDataType;
+import com.yahoo.document.annotation.AnnotationReference;
+import com.yahoo.document.datatypes.*;
+import com.yahoo.document.predicate.Predicate;
+import com.yahoo.document.serialization.DeserializationException;
+import com.yahoo.document.serialization.FieldReader;
+import com.yahoo.text.Utf8;
+import com.yahoo.vespa.objects.FieldBase;
+import org.apache.commons.codec.binary.Base64;
+
+import javax.xml.stream.XMLStreamException;
+import javax.xml.stream.XMLStreamReader;
+import java.io.InputStream;
+import java.math.BigInteger;
+import java.util.Optional;
+
+/**
+ * XML parser that reads document fields from an XML stream.
+ *
+ * All read methods assume that the stream is currently positioned at the start element of the relevant field.
+ *
+ */
+public class VespaXMLFieldReader extends VespaXMLReader implements FieldReader {
+    private static final BigInteger UINT_MAX = new BigInteger("4294967296");
+    private static final BigInteger ULONG_MAX = new BigInteger("18446744073709551616");
+
+    public VespaXMLFieldReader(String fileName, DocumentTypeManager docTypeManager) throws Exception {
+        super(fileName, docTypeManager);
+    }
+
+    public VespaXMLFieldReader(InputStream stream, DocumentTypeManager docTypeManager) throws Exception {
+        super(stream, docTypeManager);
+    }
+
+    public VespaXMLFieldReader(XMLStreamReader reader, DocumentTypeManager docTypeManager) {
+        super(reader, docTypeManager);
+    }
+
+    /**
+     * Optional test and set condition. Common for document/update/remove elements
+     * This variable is either set in VespaXMLFieldReader#read (reader for document)
+     * or in VespaXMLUpdateReader#read (reader for update).
+     */
+    private Optional<String> condition = Optional.empty();
+
+    public Optional<String> getCondition() {
+        return condition;
+    }
+
+    public void read(FieldBase field, Document document) {
+        try {
+            //workaround for documents inside array <item>
+            if (reader.getEventType() != XMLStreamReader.START_ELEMENT || !"document".equals(reader.getName().toString())) {
+                while (reader.hasNext()) {
+                    if (reader.getEventType() == XMLStreamReader.START_ELEMENT && "document".equals(reader.getName().toString())) {
+                        break;
+                    }
+                    reader.next();
+                }
+            }
+
+            // First fetch attributes.
+            String typeName = null;
+
+            for (int i = 0; i < reader.getAttributeCount(); i++) {
+                final String attributeName = reader.getAttributeName(i).toString();
+                if ("documentid".equals(attributeName) || "id".equals(attributeName)) {
+                    document.setId(new DocumentId(reader.getAttributeValue(i)));
+                } else if ("documenttype".equals(attributeName) || "type".equals(attributeName)) {
+                    typeName = reader.getAttributeValue(i);
+                } else if ("condition".equals(attributeName)) {
+                    condition = Optional.of(reader.getAttributeValue(i));
+                }
+            }
+
+            if (document.getId() != null) {
+                if (field == null) {
+                    field = new FieldBase(document.getId().toString());
+                }
+            }
+
+            DocumentType doctype = docTypeManager.getDocumentType(typeName);
+            if (doctype == null) {
+                throw newDeserializeException(field, "Must specify an existing document type, not '" + typeName + "'");
+            } else {
+                document.setDataType(doctype);
+            }
+
+            // Then fetch fields
+            while (reader.hasNext()) {
+                int type = reader.next();
+
+                if (type == XMLStreamReader.START_ELEMENT) {
+                    Field f = doctype.getField(reader.getName().toString());
+
+                    if (f == null) {
+                        throw newDeserializeException(field, "Field " + reader.getName() + " not found.");
+                    }
+
+                    FieldValue fv = f.getDataType().createFieldValue();
+                    fv.deserialize(f, this);
+                    document.setFieldValue(f, fv);
+                    skipToEnd(f.getName());
+                } else if (type == XMLStreamReader.END_ELEMENT) {
+                    return;
+                }
+            }
+        } catch (XMLStreamException e) {
+            throw newException(field, e);
+        }
+    }
+
+    public <T extends FieldValue> void read(FieldBase field, Array<T> value) {
+        try {
+            while (reader.hasNext()) {
+                int type = reader.next();
+
+                if (type == XMLStreamReader.START_ELEMENT) {
+                    if ("item".equals(reader.getName().toString())) {
+                        FieldValue fv = (value.getDataType()).getNestedType().createFieldValue();
+                        deserializeFieldValue(field, fv);
+                        // noinspection unchecked
+                        value.add((T)fv);
+                        skipToEnd("item");
+                    }
+                } else if (type == XMLStreamReader.END_ELEMENT) {
+                    return;
+                }
+            }
+        } catch (XMLStreamException e) {
+            throw newException(field, e);
+        }
+    }
+
+    class KeyAndValue {
+        FieldValue key = null;
+        FieldValue value = null;
+    }
+
+    void readKeyAndValue(FieldBase field, KeyAndValue val, MapDataType dt) throws XMLStreamException {
+        while (reader.hasNext()) {
+            int type = reader.next();
+
+            if (type == XMLStreamReader.START_ELEMENT) {
+                if ("key".equals(reader.getName().toString())) {
+                    val.key = dt.getKeyType().createFieldValue();
+                    deserializeFieldValue(field, val.key);
+                    skipToEnd("key");
+                } else if ("value".equals(reader.getName().toString())) {
+                    val.value = dt.getValueType().createFieldValue();
+                    deserializeFieldValue(field, val.value);
+                    skipToEnd("value");
+                } else {
+                    throw newDeserializeException("Illegal element inside map item: " + reader.getName());
+                }
+            } else if (type == XMLStreamReader.END_ELEMENT) {
+                return;
+            }
+        }
+    }
+
+    public <K extends FieldValue, V extends FieldValue> void read(FieldBase field, MapFieldValue<K, V> map) {
+        try {
+            MapDataType dt = map.getDataType();
+
+            while (reader.hasNext()) {
+                int type = reader.next();
+
+                if (type == XMLStreamReader.START_ELEMENT) {
+                    if ("item".equals(reader.getName().toString())) {
+                        KeyAndValue kv = new KeyAndValue();
+                        readKeyAndValue(field, kv, dt);
+
+                        if (kv.key == null || kv.value == null) {
+                            throw newDeserializeException(field, "Map items must specify both key and value");
+                        }
+                        // noinspection unchecked
+                        map.put((K)kv.key, (V)kv.value);
+                        skipToEnd("item");
+                    } else {
+                        throw newDeserializeException(field, "Illegal tag " + reader.getName() + " expected 'item'");
+                    }
+                } else if (type == XMLStreamReader.END_ELEMENT) {
+                    return;
+                }
+            }
+        } catch (XMLStreamException e) {
+            throw newException(field, e);
+        }
+    }
+
+    public void read(FieldBase field, Struct value) {
+        try {
+            boolean base64 = isBase64EncodedElement(reader);
+            boolean foundField = false;
+            StringBuilder positionBuilder = null;
+            while (reader.hasNext()) {
+                int type = reader.next();
+                if (type == XMLStreamReader.START_ELEMENT) {
+                    Field structField = value.getField(reader.getName().toString());
+                    if (structField == null) {
+                        throw newDeserializeException(field, "Field " + reader.getName() + " not found.");
+                    }
+                    FieldValue fieldValue = structField.getDataType().createFieldValue();
+                    fieldValue.deserialize(structField, this);
+                    value.setFieldValue(structField, fieldValue);
+                    skipToEnd(structField.getName());
+                    foundField = true;
+                } else if (type == XMLStreamReader.CHARACTERS) {
+                    if (foundField) {
+                        continue;
+                    }
+                    // The text of an XML element may be output using 1-n CHARACTERS
+                    // events, so we have to buffer up until the end of the element to
+                    // ensure we get everything.
+                    String chars = reader.getText();
+                    if (positionBuilder == null) {
+                        positionBuilder = new StringBuilder(chars);
+                    } else {
+                        positionBuilder.append(chars);
+                    }
+                } else if (type == XMLStreamReader.END_ELEMENT) {
+                    if (positionBuilder != null) {
+                        assignPositionFieldFromStringIfNonEmpty(value, positionBuilder.toString(), base64);
+                    }
+                    break;
+                }
+            }
+        } catch (XMLStreamException e) {
+            throw newException(field, e);
+        }
+    }
+
+    private void assignPositionFieldFromStringIfNonEmpty(Struct value, String elementText, boolean base64) {
+        String str = base64 ? Utf8.toString(new Base64().decode(elementText)) : elementText;
+        str = str.trim();
+        if (str.isEmpty()) {
+            return;
+        }
+        DataType valueType = value.getDataType();
+        if (valueType.equals(PositionDataType.INSTANCE)) {
+            value.assign(PositionDataType.fromString(str));
+        }
+    }
+
+    public <T extends FieldValue> void read(FieldBase field, WeightedSet<T> value) {
+        try {
+            while (reader.hasNext()) {
+                int type = reader.next();
+
+                if (type == XMLStreamReader.START_ELEMENT) {
+                    if ("item".equals(reader.getName().toString())) {
+                        FieldValue fv = value.getDataType().getNestedType().createFieldValue();
+
+                        int weight = 1;
+                        for (int i = 0; i < reader.getAttributeCount(); i++) {
+                            if ("weight".equals(reader.getAttributeName(i).toString())) {
+                                weight = Integer.parseInt(reader.getAttributeValue(i));
+                            }
+                        }
+
+                        deserializeFieldValue(field, fv);
+                        // noinspection unchecked
+                        value.put((T)fv, weight);
+                        skipToEnd("item");
+                    } else {
+                        throw newDeserializeException(field, "Illegal tag " + reader.getName() + " expected 'item'");
+                    }
+                } else if (type == XMLStreamReader.END_ELEMENT) {
+                    return;
+                }
+            }
+        } catch (XMLStreamException e) {
+            throw newException(field, e);
+        }
+    }
+
+    public void read(FieldBase field, ByteFieldValue value) {
+        try {
+            String dataParsed = reader.getElementText();
+            try {
+                value.assign(new Byte(dataParsed));
+            } catch (Exception e) {
+                throw newDeserializeException(field, "Invalid byte \"" + dataParsed + "\".");
+            }
+        } catch (XMLStreamException e) {
+            throw newException(field, e);
+        }
+    }
+
+    public void read(FieldBase field, DoubleFieldValue value) {
+        try {
+            String dataParsed = reader.getElementText();
+            try {
+                value.assign(new Double(dataParsed));
+            } catch (Exception e) {
+                throw newDeserializeException(field, "Invalid double \"" + dataParsed + "\".");
+            }
+        } catch (XMLStreamException e) {
+            throw newException(field, e);
+        }
+    }
+
+    public void read(FieldBase field, FloatFieldValue value) {
+        try {
+            String dataParsed = reader.getElementText();
+            try {
+                value.assign(new Float(dataParsed));
+            } catch (Exception e) {
+                throw newDeserializeException(field, "Invalid float \"" + dataParsed + "\".");
+            }
+        } catch (XMLStreamException e) {
+            throw newException(field, e);
+        }
+    }
+
+    private RuntimeException newDeserializeException(FieldBase field, String msg) {
+        return newDeserializeException("Field '" + ((field == null) ? "null" : field.getName()) + "': " + msg);
+    }
+    private RuntimeException newException(FieldBase field, Exception e) {
+        return newDeserializeException("Field '" + ((field == null) ? "null" : field.getName()) + "': " + e.getMessage());
+    }
+    public void read(FieldBase field, IntegerFieldValue value) {
+        try {
+            String dataParsed = reader.getElementText();
+
+            BigInteger val;
+            try {
+                if (dataParsed.startsWith("0x")) {
+                    val = new BigInteger(dataParsed.substring(2), 16);
+                } else if (dataParsed.startsWith("0") && dataParsed.length() > 1) {
+                    val = new BigInteger(dataParsed.substring(1), 8);
+                } else {
+                    val = new BigInteger(dataParsed);
+                }
+            } catch (Exception e) {
+                throw newDeserializeException(field, "Invalid integer \"" + dataParsed + "\".");
+            }
+            if (val.bitLength() > 32) {
+                throw newDeserializeException(field, "Invalid integer \"" + dataParsed + "\". Out of range.");
+            }
+            if (val.bitLength() == 32) {
+                if (val.compareTo(BigInteger.ZERO) == 1) {
+                    // Flip to negative
+                    val = val.subtract(UINT_MAX);
+                } else {
+                    throw newDeserializeException(field, "Invalid integer \"" + dataParsed + "\". Out of range.");
+                }
+            }
+
+            value.assign(val.intValue());
+        } catch (XMLStreamException e) {
+            throw newException(field, e);
+        }
+    }
+
+    public void read(FieldBase field, LongFieldValue value) {
+        try {
+            String dataParsed = reader.getElementText();
+
+            BigInteger val;
+            try {
+                if (dataParsed.startsWith("0x")) {
+                    val = new BigInteger(dataParsed.substring(2), 16);
+                } else if (dataParsed.startsWith("0") && dataParsed.length() > 1) {
+                    val = new BigInteger(dataParsed.substring(1), 8);
+                } else {
+                    val = new BigInteger(dataParsed);
+                }
+            } catch (Exception e) {
+                throw newDeserializeException(field, "Invalid long \"" + dataParsed + "\".");
+            }
+            if (val.bitLength() > 64) {
+                throw newDeserializeException(field, "Invalid long \"" + dataParsed + "\". Out of range.");
+            }
+            if (val.compareTo(BigInteger.ZERO) == 1 && val.bitLength() == 64) {
+                // Flip to negative
+                val = val.subtract(ULONG_MAX);
+            }
+            value.assign(val.longValue());
+        } catch (XMLStreamException e) {
+            throw newException(field, e);
+        }
+    }
+
+    public void read(FieldBase field, Raw value) {
+        try {
+            if (isBase64EncodedElement(reader)) {
+                value.assign(new Base64().decode(reader.getElementText()));
+            } else {
+                value.assign(reader.getElementText().getBytes());
+            }
+        } catch (XMLStreamException e) {
+            throw newException(field, e);
+        }
+    }
+
+    @Override
+    public void read(FieldBase field, PredicateFieldValue value) {
+        try {
+            if (isBase64EncodedElement(reader)) {
+                value.assign(Predicate.fromBinary(new Base64().decode(reader.getElementText())));
+            } else {
+                value.assign(Predicate.fromString(reader.getElementText()));
+            }
+        } catch (XMLStreamException e) {
+            throw newException(field, e);
+        }
+    }
+
+    public void read(FieldBase field, StringFieldValue value) {
+        try {
+            if (isBase64EncodedElement(reader)) {
+                throw new IllegalArgumentException("Attribute binaryencoding=base64 is not allowed for fields of type 'string'. To represent binary data, use type 'raw'.");
+            } else {
+                value.assign(reader.getElementText());
+            }
+        } catch (XMLStreamException | IllegalArgumentException e) {
+            throw newException(field, e);
+        }
+    }
+
+    @Override
+    public void read(FieldBase field, TensorFieldValue value) {
+        throw new DeserializationException("Field '"+ (field != null ? field.getName() : "null") + "': "
+                + "XML input for fields of type TENSOR is not supported. Please use JSON input instead.");
+    }
+
+    public void read(FieldBase field, AnnotationReference value) {
+        System.out.println("Annotation value read!");
+    }
+
+    private void deserializeFieldValue(FieldBase field, FieldValue value) {
+        value.deserialize(field instanceof Field ? (Field)field : null, this);
+    }
+
+    /***********************************************************************/
+    /*                   UNUSED METHODS                                    */
+    /***********************************************************************/
+
+    @SuppressWarnings("UnusedDeclaration")
+    public DocumentId readDocumentId() {
+        return null;
+    }
+
+    @SuppressWarnings("UnusedDeclaration")
+    public DocumentType readDocumentType() {
+        return null;  //To change body of implemented methods use File | Settings | File Templates.
+    }
+
+    @SuppressWarnings("UnusedDeclaration")
+    public DocumentTypeManager getDocumentTypeManager() {
+        return docTypeManager;
+    }
+
+    @Override
+    public <T extends FieldValue> void read(FieldBase field, CollectionFieldValue<T> value) {
+        System.out.println("Should not be called!!!");
+    }
+
+    @Override
+    public void read(FieldBase field, StructuredFieldValue value) {
+        System.out.println("Should not be called!!!");
+    }
+
+    @Override
+    public void read(FieldBase field, FieldValue value) {
+        System.out.println("SHOULD NEVER BE CALLED? " + field.toString());
+    }
+
+    @Override
+    public byte getByte(FieldBase fieldBase) {
+        return 0;
+    }
+
+    @Override
+    public short getShort(FieldBase fieldBase) {
+        return 0;
+    }
+
+    @Override
+    public int getInt(FieldBase fieldBase) {
+        return 0;
+    }
+
+    @Override
+    public long getLong(FieldBase fieldBase) {
+        return 0;
+    }
+
+    @Override
+    public float getFloat(FieldBase fieldBase) {
+        return 0;
+    }
+
+    @Override
+    public double getDouble(FieldBase fieldBase) {
+        return 0;
+    }
+
+    @Override
+    public byte[] getBytes(FieldBase fieldBase, int i) {
+        return new byte[0];
+    }
+
+    @Override
+    public String getString(FieldBase fieldBase) {
+        return null;
+    }
+}
diff --git a/document/src/main/java/com/yahoo/vespaxmlparser/VespaXMLReader.java b/document/src/main/java/com/yahoo/vespaxmlparser/VespaXMLReader.java
new file mode 100644
index 00000000000..10c3676a965
--- /dev/null
+++ b/document/src/main/java/com/yahoo/vespaxmlparser/VespaXMLReader.java
@@ -0,0 +1,69 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespaxmlparser;
+
+import com.yahoo.document.DocumentTypeManager;
+import com.yahoo.document.serialization.DeserializationException;
+
+import javax.xml.stream.XMLInputFactory;
+import javax.xml.stream.XMLStreamException;
+import javax.xml.stream.XMLStreamReader;
+import java.io.FileInputStream;
+import java.io.InputStream;
+
+/**
+ * @author thomasg
+ */
+public class VespaXMLReader {
+    DocumentTypeManager docTypeManager;
+    XMLStreamReader reader;
+
+    public VespaXMLReader(String fileName, DocumentTypeManager docTypeManager) throws Exception {
+        this(new FileInputStream(fileName), docTypeManager);
+    }
+
+    public VespaXMLReader(InputStream stream, DocumentTypeManager docTypeManager) throws Exception {
+        this.docTypeManager = docTypeManager;
+        XMLInputFactory xmlInputFactory = XMLInputFactory.newInstance();
+        xmlInputFactory.setProperty("javax.xml.stream.isSupportingExternalEntities", Boolean.FALSE);
+        reader = xmlInputFactory.createXMLStreamReader(stream);
+    }
+
+    public VespaXMLReader(XMLStreamReader reader, DocumentTypeManager docTypeManager) {
+        this.docTypeManager = docTypeManager;
+        this.reader = reader;
+    }
+
+    protected RuntimeException newDeserializeException(String message) {
+        return new DeserializationException(message + " (at line " + reader.getLocation().getLineNumber() + ", column " + reader.getLocation().getColumnNumber() + ")");
+    }
+
+    protected RuntimeException newException(Exception e) {
+        return new DeserializationException(e.getMessage() + " (at line " + reader.getLocation().getLineNumber() + ", column " + reader.getLocation().getColumnNumber() + ")", e);
+    }
+
+    protected void skipToEnd(String tagName) throws XMLStreamException {
+        while (reader.hasNext()) {
+            if (reader.getEventType() == XMLStreamReader.END_ELEMENT && tagName.equals(reader.getName().toString())) {
+                return;
+            }
+            reader.next();
+        }
+        throw new DeserializationException("Missing end tag for element '" + tagName + "'" + reader.getLocation());
+    }
+
+    public static boolean isBase64EncodingAttribute(String attributeName, String attributeValue) {
+        return "binaryencoding".equals(attributeName) &&
+               "base64".equalsIgnoreCase(attributeValue);
+    }
+
+    public static boolean isBase64EncodedElement(XMLStreamReader reader) {
+        for (int i = 0; i < reader.getAttributeCount(); i++) {
+            if (isBase64EncodingAttribute(reader.getAttributeName(i).toString(),
+                                          reader.getAttributeValue(i)))
+            {
+                return true;
+            }
+        }
+        return false;
+    }
+}
diff --git a/document/src/main/java/com/yahoo/vespaxmlparser/VespaXMLUpdateReader.java b/document/src/main/java/com/yahoo/vespaxmlparser/VespaXMLUpdateReader.java
new file mode 100644
index 00000000000..a4d334848d5
--- /dev/null
+++ b/document/src/main/java/com/yahoo/vespaxmlparser/VespaXMLUpdateReader.java
@@ -0,0 +1,379 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespaxmlparser;
+
+import com.yahoo.document.*;
+import com.yahoo.document.datatypes.Array;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.IntegerFieldValue;
+import com.yahoo.document.datatypes.WeightedSet;
+import com.yahoo.document.fieldpathupdate.AddFieldPathUpdate;
+import com.yahoo.document.fieldpathupdate.AssignFieldPathUpdate;
+import com.yahoo.document.fieldpathupdate.FieldPathUpdate;
+import com.yahoo.document.fieldpathupdate.RemoveFieldPathUpdate;
+import com.yahoo.document.select.parser.ParseException;
+import com.yahoo.document.serialization.DocumentUpdateReader;
+import com.yahoo.document.update.FieldUpdate;
+import com.yahoo.document.update.ValueUpdate;
+
+import javax.xml.stream.XMLStreamException;
+import javax.xml.stream.XMLStreamReader;
+import java.io.InputStream;
+import java.util.List;
+import java.util.Optional;
+
+public class VespaXMLUpdateReader extends VespaXMLFieldReader implements DocumentUpdateReader {
+    public VespaXMLUpdateReader(String fileName, DocumentTypeManager docTypeManager) throws Exception {
+        super(fileName, docTypeManager);
+    }
+
+    public VespaXMLUpdateReader(InputStream stream, DocumentTypeManager docTypeManager) throws Exception {
+        super(stream, docTypeManager);
+    }
+
+    public VespaXMLUpdateReader(XMLStreamReader reader, DocumentTypeManager docTypeManager) {
+        super(reader, docTypeManager);
+    }
+
+    private Optional<String> condition = Optional.empty();
+
+    public Optional<String> getCondition() {
+        return condition;
+    }
+
+    public boolean hasFieldPath() {
+        for (int i = 0; i < reader.getAttributeCount(); i++) {
+            if (reader.getAttributeName(i).toString().equals("fieldpath")) {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
+    public void read(DocumentUpdate update) {
+       try {
+           // First fetch attributes.
+           DocumentType doctype = null;
+
+           for (int i = 0; i < reader.getAttributeCount(); i++) {
+               final String attributeName = reader.getAttributeName(i).toString();
+               final String attributeValue = reader.getAttributeValue(i);
+
+               if ("documentid".equals(attributeName) || "id".equals(attributeName)) {
+                   update.setId(new DocumentId(attributeValue));
+               } else if ("documenttype".equals(attributeName) || "type".equals(attributeName)) {
+                   doctype = docTypeManager.getDocumentType(attributeValue);
+                   update.setDocumentType(doctype);
+               } else if ("create-if-non-existent".equals(attributeName)) {
+                   if ("true".equals(attributeValue)) {
+                       update.setCreateIfNonExistent(true);
+                   } else if ("false".equals(attributeValue)) {
+                       update.setCreateIfNonExistent(false);
+                   } else {
+                       throw newDeserializeException("'create-if-non-existent' must be either 'true' or 'false', was '" + attributeValue +"'");
+                   }
+               } else if ("condition".equals(attributeName)) {
+                   condition = Optional.of(attributeValue);
+               }
+           }
+
+           if (doctype == null) {
+               throw newDeserializeException("Must specify document type. " + reader.getLocation());
+           }
+
+           // Then fetch fields
+           while (reader.hasNext()) {
+               int type = reader.next();
+
+               if (type == XMLStreamReader.START_ELEMENT) {
+                   final String currentName = reader.getName().toString();
+                   if (hasFieldPath()) {
+                       if ("assign".equals(currentName)) {
+                           update.addFieldPathUpdate(new AssignFieldPathUpdate(doctype, this));
+                           skipToEnd("assign");
+                       } else if ("add".equals(currentName)) {
+                           update.addFieldPathUpdate(new AddFieldPathUpdate(doctype, this));
+                           skipToEnd("add");
+                       } else if ("remove".equals(currentName)) {
+                           update.addFieldPathUpdate(new RemoveFieldPathUpdate(doctype, this));
+                           skipToEnd("remove");
+                       } else {
+                           throw newDeserializeException("Unknown field path update operation " + reader.getName());
+                       }
+                   } else {
+                       if ("assign".equals(currentName)) {
+                           update.addFieldUpdate(readAssign(update));
+                           skipToEnd("assign");
+                       } else if ("add".equals(currentName)) {
+                           update.addFieldUpdate(readAdd(update));
+                           skipToEnd("add");
+                       } else if ("remove".equals(currentName)) {
+                           update.addFieldUpdate(readRemove(update));
+                           skipToEnd("remove");
+                       } else if ("alter".equals(currentName)) {
+                           update.addFieldUpdate(readAlter(update));
+                           skipToEnd("alter");
+                       } else if ("increment".equals(currentName) ||
+                               "decrement".equals(currentName) ||
+                               "multiply".equals(currentName) ||
+                               "divide".equals(currentName)) {
+                           update.addFieldUpdate(readArithmeticField(update, currentName));
+                           skipToEnd(currentName);
+                       } else {
+                           throw newDeserializeException("Unknown update operation " + reader.getName());
+                       }
+                   }
+               } else if (type == XMLStreamReader.END_ELEMENT) {
+                   return;
+               }
+           }
+       } catch (XMLStreamException e) {
+           throw newException(e);
+       }
+    }
+
+    FieldUpdate readAdd(DocumentUpdate update) throws XMLStreamException {
+        for (int i = 0; i < reader.getAttributeCount(); i++) {
+            if ("field".equals(reader.getAttributeName(i).toString())) {
+                Field f = update.getDocumentType().getField(reader.getAttributeValue(i));
+
+                FieldValue value = f.getDataType().createFieldValue();
+                value.deserialize(f, this);
+
+                if (value instanceof Array) {
+                    List<FieldValue> l = ((Array)value).getValues();
+                    return FieldUpdate.createAddAll(f, l);
+                } else if (value instanceof WeightedSet) {
+                    return FieldUpdate.createAddAll(f, ((WeightedSet) value));
+                } else {
+                    throw newDeserializeException("Add operation only applicable to multivalue lists");
+                }
+
+            }
+        }
+        throw newDeserializeException("Add update without field attribute");
+    }
+
+
+    FieldUpdate readRemove(DocumentUpdate update) throws XMLStreamException {
+        for (int i = 0; i < reader.getAttributeCount(); i++) {
+            if ("field".equals(reader.getAttributeName(i).toString())) {
+                Field f = update.getDocumentType().getField(reader.getAttributeValue(i));
+
+                FieldValue value = f.getDataType().createFieldValue();
+                value.deserialize(f, this);
+
+                if (value instanceof Array) {
+                    List<FieldValue> l = ((Array)value).getValues();
+                    return FieldUpdate.createRemoveAll(f, l);
+                } else if (value instanceof WeightedSet) {
+                    return FieldUpdate.createRemoveAll(f, ((WeightedSet)value));
+                } else {
+                    throw newDeserializeException("Remove operation only applicable to multivalue lists");
+                }
+
+            }
+        }
+        throw newDeserializeException("Remove update without field attribute");
+    }
+
+    FieldUpdate readAssign(DocumentUpdate update) throws XMLStreamException {
+        for (int i = 0; i < reader.getAttributeCount(); i++) {
+            if ("field".equals(reader.getAttributeName(i).toString())) {
+                Field f = update.getDocumentType().getField(reader.getAttributeValue(i));
+
+                if (f == null) {
+                    throw newDeserializeException("Field " + reader.getAttributeValue(i) + " not found.");
+                }
+
+                FieldValue value = f.getDataType().createFieldValue();
+                value.deserialize(f, this);
+                return FieldUpdate.createAssign(f, value);
+            }
+        }
+        throw newDeserializeException("Assignment update without field attribute");
+    }
+
+
+    FieldUpdate readAlter(DocumentUpdate update) throws XMLStreamException {
+        Field f = null;
+        for (int i = 0; i < reader.getAttributeCount(); i++) {
+            if ("field".equals(reader.getAttributeName(i).toString())) {
+                f = update.getDocumentType().getField(reader.getAttributeValue(i));
+            }
+        }
+
+        if (f == null) {
+            throw newDeserializeException("Alter update without \"field\" attribute");
+        }
+
+        FieldUpdate fu = FieldUpdate.create(f);
+
+        while (reader.hasNext()) {
+            int type = reader.next();
+            if (type == XMLStreamReader.START_ELEMENT) {
+                if ("increment".equals(reader.getName().toString()) ||
+                    "decrement".equals(reader.getName().toString()) ||
+                    "multiply".equals(reader.getName().toString()) ||
+                    "divide".equals(reader.getName().toString())) {
+                    update.addFieldUpdate(readArithmetic(update, reader.getName().toString(), f, fu));
+                    skipToEnd(reader.getName().toString());
+                } else {
+                    throw newDeserializeException("Element \"" + reader.getName() + "\" not appropriate within alter element");
+                }
+            } else if (type == XMLStreamReader.END_ELEMENT) {
+                break;
+            }
+        }
+
+        return fu;
+    }
+
+    FieldUpdate readArithmeticField(DocumentUpdate update, String type) throws XMLStreamException {
+        Field f = null;
+        for (int i = 0; i < reader.getAttributeCount(); i++) {
+            if ("field".equals(reader.getAttributeName(i).toString())) {
+                f = update.getDocumentType().getField(reader.getAttributeValue(i));
+            }
+        }
+
+        if (f == null) {
+            throw newDeserializeException("Assignment update without \"field\" attribute");
+        }
+
+        FieldUpdate fu = FieldUpdate.create(f);
+        readArithmetic(update, type, f, fu);
+        return fu;
+    }
+
+    FieldUpdate readArithmetic(DocumentUpdate update, String type, Field f, FieldUpdate fu) throws XMLStreamException {
+        Double by = null;
+
+        for (int i = 0; i < reader.getAttributeCount(); i++) {
+            if ("by".equals(reader.getAttributeName(i).toString())) {
+                by = Double.parseDouble(reader.getAttributeValue(i));
+            }
+        }
+
+        if (by == null) {
+            throw newDeserializeException("Assignment update without \"by\" attribute");
+        }
+
+        FieldValue key = null;
+        do {
+            reader.next();
+            if (reader.getEventType() == XMLStreamReader.START_ELEMENT) {
+                if ("key".equals(reader.getName().toString())) {
+                    if (f.getDataType() instanceof WeightedSetDataType) {
+                        DataType nestedType = ((WeightedSetDataType)f.getDataType()).getNestedType();
+                        key = nestedType.createFieldValue();
+                        key.deserialize(this);
+                    } else if (f.getDataType() instanceof MapDataType) {
+                        key = ((MapDataType)f.getDataType()).getKeyType().createFieldValue();
+                        key.deserialize(this);
+                    } else if (f.getDataType() instanceof ArrayDataType) {
+                        key = new IntegerFieldValue(Integer.parseInt(reader.getElementText()));
+                    } else {
+                        throw newDeserializeException("Key tag only applicable for weighted sets and maps");
+                    }
+                    skipToEnd("key");
+                } else {
+                    throw newDeserializeException("\"" + reader.getName() + "\" not appropriate within " + type + " element.");
+                }
+            }
+        } while (reader.getEventType() != XMLStreamReader.END_ELEMENT);
+
+        if (key != null) {
+            if ("increment".equals(type)) { fu.addValueUpdate(ValueUpdate.createIncrement(key, by)); }
+            if ("decrement".equals(type)) { fu.addValueUpdate(ValueUpdate.createDecrement(key, by)); }
+            if ("multiply".equals(type)) { fu.addValueUpdate(ValueUpdate.createMultiply(key, by)); }
+            if ("divide".equals(type)) { fu.addValueUpdate(ValueUpdate.createDivide(key, by)); }
+        } else {
+            if ("increment".equals(type)) { fu.addValueUpdate(ValueUpdate.createIncrement(by)); }
+            if ("decrement".equals(type)) { fu.addValueUpdate(ValueUpdate.createDecrement(by)); }
+            if ("multiply".equals(type)) { fu.addValueUpdate(ValueUpdate.createMultiply(by)); }
+            if ("divide".equals(type)) { fu.addValueUpdate(ValueUpdate.createDivide(by)); }
+        }
+
+        return fu;
+    }
+
+    public void read(FieldUpdate update) {
+    }
+
+    public void read(FieldPathUpdate update) {
+        String whereClause = null;
+        String fieldPath = null;
+
+        for (int i = 0; i < reader.getAttributeCount(); i++) {
+            if (reader.getAttributeName(i).toString().equals("where")) {
+                whereClause = reader.getAttributeValue(i);
+            } else if (reader.getAttributeName(i).toString().equals("fieldpath")) {
+                fieldPath = reader.getAttributeValue(i);
+            }
+        }
+
+        if (fieldPath != null) {
+            update.setFieldPath(fieldPath);
+        } else {
+            throw newDeserializeException("Field path is required for document updates.");
+        }
+
+        if (whereClause != null) {
+            try {
+                update.setWhereClause(whereClause);
+            } catch (ParseException e) {
+                throw newException(e);
+            }
+        }
+    }
+
+    public void read(AssignFieldPathUpdate update) {
+        try {
+            for (int i = 0; i < reader.getAttributeCount(); i++) {
+                if (reader.getAttributeName(i).toString().equals("removeifzero")) {
+                    update.setRemoveIfZero(Boolean.parseBoolean(reader.getAttributeValue(i)));
+                } else if (reader.getAttributeName(i).toString().equals("createmissingpath")) {
+                    update.setCreateMissingPath(Boolean.parseBoolean(reader.getAttributeValue(i)));
+                }
+            }
+            DataType dt = update.getFieldPath().getResultingDataType();
+
+            if (dt instanceof NumericDataType) {
+                update.setExpression(reader.getElementText());
+            } else {
+                FieldValue fv = dt.createFieldValue();
+                fv.deserialize(resolveField(update), this);
+                update.setNewValue(fv);
+            }
+        } catch (XMLStreamException e) {
+            throw newException(e);
+        }
+    }
+
+    public void read(AddFieldPathUpdate update) {
+        DataType dt = update.getFieldPath().getResultingDataType();
+        FieldValue fv = dt.createFieldValue();
+        fv.deserialize(resolveField(update), this);
+        update.setNewValues((Array)fv);
+    }
+
+    public void read(RemoveFieldPathUpdate update) {
+    }
+
+    private static Field resolveField(FieldPathUpdate update) {
+        String orig = update.getOriginalFieldPath();
+        if (orig == null) {
+            return null;
+        }
+        FieldPath path = update.getFieldPath();
+        if (path == null) {
+            return null;
+        }
+        DataType type = path.getResultingDataType();
+        if (type == null) {
+            return null;
+        }
+        return new Field(orig, type);
+    }
+}
diff --git a/document/src/main/java/com/yahoo/vespaxmlparser/package-info.java b/document/src/main/java/com/yahoo/vespaxmlparser/package-info.java
new file mode 100644
index 00000000000..eae7e1320a2
--- /dev/null
+++ b/document/src/main/java/com/yahoo/vespaxmlparser/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespaxmlparser;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/document/src/main/java/net/jpountz/lz4/package-info.java b/document/src/main/java/net/jpountz/lz4/package-info.java
new file mode 100644
index 00000000000..45fc2e031ab
--- /dev/null
+++ b/document/src/main/java/net/jpountz/lz4/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage(version = @Version(major = 1, minor = 3, micro = 0))
+package net.jpountz.lz4;
+import com.yahoo.osgi.annotation.ExportPackage;
+import com.yahoo.osgi.annotation.Version;
diff --git a/document/src/main/javacc/SelectParser.jj b/document/src/main/javacc/SelectParser.jj
new file mode 100755
index 00000000000..2464365196c
--- /dev/null
+++ b/document/src/main/javacc/SelectParser.jj
@@ -0,0 +1,268 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * @author  <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ * @version $Id$
+ */
+options {
+    CACHE_TOKENS       = true;
+    DEBUG_PARSER       = false;
+    ERROR_REPORTING    = true;
+    IGNORE_CASE        = true;
+    STATIC             = false;
+    UNICODE_INPUT      = true;
+    USER_CHAR_STREAM   = true;
+    USER_TOKEN_MANAGER = false;
+}
+
+PARSER_BEGIN(SelectParser)
+
+package com.yahoo.document.select.parser;
+
+import com.yahoo.document.select.rule.*;
+
+import java.math.BigInteger;
+import java.util.List;
+import java.util.ArrayList;
+
+public class SelectParser {
+
+
+}
+
+PARSER_END(SelectParser)
+
+SKIP :
+{
+    " " | "\f" | "\n" | "\r" | "\t"
+}
+
+TOKEN :
+{
+    <INTEGER: <DECIMAL> (["l","L"])? | <HEX> (["l","L"])? | <OCTAL> (["l","L"])?> |
+        <#DECIMAL: ["1"-"9"] (["0"-"9"])*> |
+        <#HEX: "0" ["x","X"] (["0"-"9","a"-"f","A"-"F"])+> |
+        <#OCTAL: "0" (["0"-"7"])*> |
+    <FLOAT: (["0"-"9"])+ ("." (["0"-"9"])*)? (<EXPONENT>)? (["f","F","d","D"])?> |  
+        <#EXPONENT: ["e","E"] (["+","-"])? (["0"-"9"])+>
+}
+
+TOKEN :
+{
+    <LBRACE: "("> |
+    <RBRACE: ")"> |
+    <ADD: "+"> |
+    <SUB: "-"> |
+    <MUL: "*"> |
+    <DIV: "/"> |
+    <MOD: "%"> |
+    <DOT: "."> |
+    <COMMA: ","> |
+    <NE: "!="> |
+    <GE: ">="> |
+    <LE: "<="> |
+    <REGEX: "=~"> |
+    <EQ: "=="> |
+    <GLOB: "="> |
+    <GT: ">"> |
+    <LT: "<"> |
+    <DOLLAR: "$"> |
+    <STRING: ("\"" (~["\""] | "\\\"")* "\"") |
+             ("'" (~["'"] | "\\'")* "'")> |
+    <ID: "id"> |
+    <SEARCHCOLUMN: "searchcolumn"> |
+    <ID_SCHEME: "scheme"> |
+    <ID_TYPE: "type"> |
+    <ID_NAMESPACE: "namespace"> |
+    <ID_SPECIFIC: "specific"> |
+    <ID_USER: "user"> |
+    <ID_GROUP: "group"> |
+    <ID_ORDER: "order"> |
+    <ID_BUCKET: "bucket"> |
+    <NULL: "null"> |
+    <NOW: "now"> |
+    <TRUE: "true"> |
+    <FALSE: "false"> |
+    <AND: "and"> |
+    <OR: "or"> |
+    <NOT: "not"> |
+    <IDENTIFIER: ["a"-"z","A"-"Z", "_"] (["a"-"z","A"-"Z","0"-"9","_","-","@","$","[","]","{","}"])*>
+}
+
+ExpressionNode expression() :
+{
+    ExpressionNode ret = new LiteralNode(true);
+}
+{
+    ( [ ret = logic() ] <EOF> )
+    { return ret; }
+}
+
+ExpressionNode logic() :
+{
+    LogicNode lst = new LogicNode();
+    String op = null;
+    ExpressionNode node;
+}
+{
+    ( node = negation() { lst.add(null, node); }
+      ( ( <AND> | <OR> ) { op = token.image; }
+        node = negation() { lst.add(op, node); } )* )
+    { return op != null ? lst : node; }
+}
+
+ExpressionNode negation() :
+{
+    boolean not = false;
+    ExpressionNode node;
+}
+{
+    ( [ <NOT> { not = true; } ] node = relational() )
+    { return not ? new NegationNode(node) : node; }
+}
+
+NowNode now() : { }
+{
+    ( <NOW> <LBRACE> <RBRACE> )
+    { return new NowNode(); }
+}
+
+ExpressionNode relational() :
+{
+    ExpressionNode lhs, rhs = null;
+    String op = null;
+}
+{
+    ( lhs = arithmetic()
+      [ ( <EQ> | <NE> | <LT> | <GT> | <LE> | <GE> | <REGEX> | <GLOB> ) { op = token.image; }
+        rhs = arithmetic() ] )
+    { return rhs != null ? new ComparisonNode(lhs, op, rhs) : lhs; }
+}
+
+ExpressionNode arithmetic() :
+{
+    ArithmeticNode lst = new ArithmeticNode();
+    String op = null;
+    ExpressionNode node;
+}
+{
+    ( node = attribute() { lst.add(null, node); }
+      ( ( <ADD> | <SUB> | <DIV> | <MUL> | <MOD> ) { op = token.image; }
+        node = attribute() { lst.add(op, node); } )* )
+    { return op != null ? lst : node; }
+}
+
+VariableNode variable() :
+{
+    VariableNode ret;
+}
+{
+    ( <DOLLAR> identifier() { ret = new VariableNode(token.image); } )
+    { return ret; }
+}
+
+ExpressionNode attribute() :
+{
+    ExpressionNode val;
+    AttributeNode.Item item;
+    List lst = new ArrayList();
+}
+{
+    ( val = value() ( <DOT> identifier()  { item = new AttributeNode.Item(token.image); }
+                      [ <LBRACE> <RBRACE> { item.setType(AttributeNode.Item.FUNCTION); } ]
+                                          { lst.add(item); } )* )
+    { return lst.size() > 0 ? new AttributeNode(val, lst) : val; }
+}
+
+ExpressionNode value() :
+{
+    ExpressionNode ret;
+}
+{
+    ( LOOKAHEAD(2)
+      ( ret = id() |
+        ret = searchColumn() |
+        ret = literal() |
+        ret = variable() |
+        ret = now() |
+        <LBRACE> ret = logic() <RBRACE> { ret = new EmbracedNode(ret); } ) |
+     ( ret = document() ) )
+    { return ret; }
+}
+
+DocumentNode document() :
+{
+    DocumentNode ret;
+}
+{
+    ( identifier() { ret = new DocumentNode(token.image); } )
+    { return ret; }
+}
+
+                                                                                                                                                                                                                                             
+void identifier() : { }
+{                                                                                                                                                                                                                                            
+    (  <ID> |
+       <SEARCHCOLUMN> |
+       <ID_SCHEME> |
+       <ID_TYPE> |
+       <ID_NAMESPACE> |
+       <ID_SPECIFIC> |
+       <ID_USER> |
+       <ID_GROUP> |
+       <ID_ORDER> |
+       <ID_BUCKET> |
+       <NULL> |
+       <NOW> |
+       <TRUE> |
+       <FALSE> |
+       <AND> |
+       <OR> |
+       // <NOT> |  Causes a choice conflict, but it is not such a good name anyway ...
+       <IDENTIFIER> )
+}  
+
+IdNode id() :
+{
+    IdNode ret = new IdNode();
+}
+{
+    ( <ID> [ LOOKAHEAD(2) <DOT>
+             ( <ID_SCHEME>     { ret.setField(token.image); } | 
+               <ID_TYPE>       { ret.setField(token.image); } |
+               <ID_NAMESPACE>  { ret.setField(token.image); } |
+               <ID_SPECIFIC>   { ret.setField(token.image); } |
+               <ID_USER>       { ret.setField(token.image); } | 
+               <ID_GROUP>      { ret.setField(token.image); } | 
+               <ID_BUCKET>     { ret.setField(token.image); } | 
+               ( <ID_ORDER>    { ret.setField(token.image); } <LBRACE> 
+     <INTEGER> { ret.setWidthBits(Short.parseShort(token.image)); } <COMMA> 
+     <INTEGER> { ret.setDivisionBits(Short.parseShort(token.image)); } <RBRACE> ) )
+              ] )
+    { return ret; }
+}
+
+SearchColumnNode searchColumn() :
+{
+    SearchColumnNode ret = new SearchColumnNode();
+}
+{
+    ( <SEARCHCOLUMN> [ LOOKAHEAD(2) <DOT> <INTEGER> { ret.setField(Integer.parseInt(token.image)); } ] )
+    { return ret; }
+}
+
+LiteralNode literal() :
+{
+    String sign = "";
+    Object ret = null;
+}
+{
+    ( ( [ <SUB>       { sign = "-"; } ]
+        ( <FLOAT>     { ret = Double.parseDouble(sign + token.image); } |
+          <INTEGER>   { ret = SelectParserUtils.decodeLong(sign + token.image); } ) ) |
+      ( <ADD> <FLOAT> { ret = Double.parseDouble(token.image); } ) |
+      ( <STRING>      { ret = SelectParserUtils.unquote(token.image); } ) |
+      ( <TRUE>        { ret = true; } ) |
+      ( <FALSE>       { ret = false; } ) |
+      ( <NULL>        { ret = null; } ) )
+    { return new LiteralNode(ret); }
+}
diff --git a/document/src/test/document/docindoc.cfg b/document/src/test/document/docindoc.cfg
new file mode 100644
index 00000000000..ee98e541faf
--- /dev/null
+++ b/document/src/test/document/docindoc.cfg
@@ -0,0 +1,62 @@
+datatype[6]
+datatype[0].id -2040625920
+datatype[0].arraytype[0]
+datatype[0].weightedsettype[0]
+datatype[0].structtype[1]
+datatype[0].structtype[0].name outerdoc.header
+datatype[0].structtype[0].version 0
+datatype[0].structtype[0].field[0]
+datatype[0].documenttype[0]
+datatype[1].id -1407012075
+datatype[1].arraytype[0]
+datatype[1].weightedsettype[0]
+datatype[1].structtype[1]
+datatype[1].structtype[0].name outerdoc.body
+datatype[1].structtype[0].version 0
+datatype[1].structtype[0].field[2]
+datatype[1].structtype[0].field[0].name stringfield
+datatype[1].structtype[0].field[0].id[0]
+datatype[1].structtype[0].field[0].datatype 2
+datatype[1].structtype[0].field[1].name docfield
+datatype[1].structtype[0].field[1].id[0]
+datatype[1].structtype[0].field[1].datatype 8
+datatype[1].documenttype[0]
+datatype[2].id 1748635999
+datatype[2].arraytype[0]
+datatype[2].weightedsettype[0]
+datatype[2].structtype[0]
+datatype[2].documenttype[1]
+datatype[2].documenttype[0].name outerdoc
+datatype[2].documenttype[0].version 0
+datatype[2].documenttype[0].inherits[0]
+datatype[2].documenttype[0].headerstruct -2040625920
+datatype[2].documenttype[0].bodystruct -1407012075
+datatype[3].id 1547813467
+datatype[3].arraytype[0]
+datatype[3].weightedsettype[0]
+datatype[3].structtype[1]
+datatype[3].structtype[0].name innerdoc.header
+datatype[3].structtype[0].version 0
+datatype[3].structtype[0].field[0]
+datatype[3].documenttype[0]
+datatype[4].id -858027216
+datatype[4].arraytype[0]
+datatype[4].weightedsettype[0]
+datatype[4].structtype[1]
+datatype[4].structtype[0].name innerdoc.body
+datatype[4].structtype[0].version 0
+datatype[4].structtype[0].field[1]
+datatype[4].structtype[0].field[0].name intfield
+datatype[4].structtype[0].field[0].id[0]
+datatype[4].structtype[0].field[0].datatype 0
+datatype[4].documenttype[0]
+datatype[5].id 828068964
+datatype[5].arraytype[0]
+datatype[5].weightedsettype[0]
+datatype[5].structtype[0]
+datatype[5].documenttype[1]
+datatype[5].documenttype[0].name innerdoc
+datatype[5].documenttype[0].version 0
+datatype[5].documenttype[0].inherits[0]
+datatype[5].documenttype[0].headerstruct 1547813467
+datatype[5].documenttype[0].bodystruct -858027216
diff --git a/document/src/test/document/documentmanager.annotationspolymorphy.cfg b/document/src/test/document/documentmanager.annotationspolymorphy.cfg
new file mode 100755
index 00000000000..869aea6cd2f
--- /dev/null
+++ b/document/src/test/document/documentmanager.annotationspolymorphy.cfg
@@ -0,0 +1,187 @@
+enablecompression false
+datatype[15]
+datatype[0].id -2014701853
+datatype[0].arraytype[0]
+datatype[0].weightedsettype[0]
+datatype[0].structtype[0]
+datatype[0].documenttype[0]
+datatype[0].annotationreftype[1]
+datatype[0].annotationreftype[0].annotation "super"
+datatype[1].id -169651851
+datatype[1].arraytype[0]
+datatype[1].weightedsettype[0]
+datatype[1].structtype[1]
+datatype[1].structtype[0].name "annotationspolymorphy_summary.header"
+datatype[1].structtype[0].version 0
+datatype[1].structtype[0].field[3]
+datatype[1].structtype[0].field[0].name "rankfeatures"
+datatype[1].structtype[0].field[0].datatype 2
+datatype[1].structtype[0].field[0].id[0]
+datatype[1].structtype[0].field[1].name "summaryfeatures"
+datatype[1].structtype[0].field[1].datatype 2
+datatype[1].structtype[0].field[1].id[0]
+datatype[1].structtype[0].field[2].name "documentid"
+datatype[1].structtype[0].field[2].datatype 2
+datatype[1].structtype[0].field[2].id[0]
+datatype[1].structtype[0].inherits[0]
+datatype[1].documenttype[0]
+datatype[1].annotationreftype[0]
+datatype[2].id -269870902
+datatype[2].arraytype[0]
+datatype[2].weightedsettype[0]
+datatype[2].structtype[1]
+datatype[2].structtype[0].name "annotationspolymorphy_summary.body"
+datatype[2].structtype[0].version 0
+datatype[2].structtype[0].field[0]
+datatype[2].structtype[0].inherits[0]
+datatype[2].documenttype[0]
+datatype[2].annotationreftype[0]
+datatype[3].id -197316982
+datatype[3].arraytype[0]
+datatype[3].weightedsettype[0]
+datatype[3].structtype[0]
+datatype[3].documenttype[1]
+datatype[3].documenttype[0].name "annotationspolymorphy_summary"
+datatype[3].documenttype[0].version 0
+datatype[3].documenttype[0].headerstruct -169651851
+datatype[3].documenttype[0].bodystruct -269870902
+datatype[3].documenttype[0].inherits[0]
+datatype[3].annotationreftype[0]
+datatype[4].id -1327132343
+datatype[4].arraytype[0]
+datatype[4].weightedsettype[0]
+datatype[4].structtype[1]
+datatype[4].structtype[0].name "annotationspolymorphy_index.header"
+datatype[4].structtype[0].version 0
+datatype[4].structtype[0].field[0]
+datatype[4].structtype[0].inherits[0]
+datatype[4].documenttype[0]
+datatype[4].annotationreftype[0]
+datatype[5].id -396214882
+datatype[5].arraytype[0]
+datatype[5].weightedsettype[0]
+datatype[5].structtype[1]
+datatype[5].structtype[0].name "annotationspolymorphy_index.body"
+datatype[5].structtype[0].version 0
+datatype[5].structtype[0].field[0]
+datatype[5].structtype[0].inherits[0]
+datatype[5].documenttype[0]
+datatype[5].annotationreftype[0]
+datatype[6].id 873354550
+datatype[6].arraytype[0]
+datatype[6].weightedsettype[0]
+datatype[6].structtype[0]
+datatype[6].documenttype[1]
+datatype[6].documenttype[0].name "annotationspolymorphy_index"
+datatype[6].documenttype[0].version 0
+datatype[6].documenttype[0].headerstruct -1327132343
+datatype[6].documenttype[0].bodystruct -396214882
+datatype[6].documenttype[0].inherits[0]
+datatype[6].annotationreftype[0]
+datatype[7].id -550571073
+datatype[7].arraytype[0]
+datatype[7].weightedsettype[0]
+datatype[7].structtype[1]
+datatype[7].structtype[0].name "annotationspolymorphy_attribute.header"
+datatype[7].structtype[0].version 0
+datatype[7].structtype[0].field[0]
+datatype[7].structtype[0].inherits[0]
+datatype[7].documenttype[0]
+datatype[7].annotationreftype[0]
+datatype[8].id -1418869356
+datatype[8].arraytype[0]
+datatype[8].weightedsettype[0]
+datatype[8].structtype[1]
+datatype[8].structtype[0].name "annotationspolymorphy_attribute.body"
+datatype[8].structtype[0].version 0
+datatype[8].structtype[0].field[0]
+datatype[8].structtype[0].inherits[0]
+datatype[8].documenttype[0]
+datatype[8].annotationreftype[0]
+datatype[9].id 241118080
+datatype[9].arraytype[0]
+datatype[9].weightedsettype[0]
+datatype[9].structtype[0]
+datatype[9].documenttype[1]
+datatype[9].documenttype[0].name "annotationspolymorphy_attribute"
+datatype[9].documenttype[0].version 0
+datatype[9].documenttype[0].headerstruct -550571073
+datatype[9].documenttype[0].bodystruct -1418869356
+datatype[9].documenttype[0].inherits[0]
+datatype[9].annotationreftype[0]
+datatype[10].id -592896846
+datatype[10].arraytype[0]
+datatype[10].weightedsettype[0]
+datatype[10].structtype[1]
+datatype[10].structtype[0].name "indexingdocument.header"
+datatype[10].structtype[0].version 0
+datatype[10].structtype[0].field[3]
+datatype[10].structtype[0].field[0].name "index"
+datatype[10].structtype[0].field[0].datatype 8
+datatype[10].structtype[0].field[0].id[0]
+datatype[10].structtype[0].field[1].name "summary"
+datatype[10].structtype[0].field[1].datatype 8
+datatype[10].structtype[0].field[1].id[0]
+datatype[10].structtype[0].field[2].name "attribute"
+datatype[10].structtype[0].field[2].datatype 8
+datatype[10].structtype[0].field[2].id[0]
+datatype[10].structtype[0].inherits[0]
+datatype[10].documenttype[0]
+datatype[10].annotationreftype[0]
+datatype[11].id -2093772985
+datatype[11].arraytype[0]
+datatype[11].weightedsettype[0]
+datatype[11].structtype[1]
+datatype[11].structtype[0].name "indexingdocument.body"
+datatype[11].structtype[0].version 0
+datatype[11].structtype[0].field[0]
+datatype[11].structtype[0].inherits[0]
+datatype[11].documenttype[0]
+datatype[11].annotationreftype[0]
+datatype[12].id -1831281171
+datatype[12].arraytype[0]
+datatype[12].weightedsettype[0]
+datatype[12].structtype[0]
+datatype[12].documenttype[1]
+datatype[12].documenttype[0].name "indexingdocument"
+datatype[12].documenttype[0].version 0
+datatype[12].documenttype[0].headerstruct -592896846
+datatype[12].documenttype[0].bodystruct -2093772985
+datatype[12].documenttype[0].inherits[0]
+datatype[12].annotationreftype[0]
+datatype[13].id -888007918
+datatype[13].arraytype[0]
+datatype[13].weightedsettype[0]
+datatype[13].structtype[1]
+datatype[13].structtype[0].name "annotation.blah"
+datatype[13].structtype[0].version 0
+datatype[13].structtype[0].field[1]
+datatype[13].structtype[0].field[0].name "a"
+datatype[13].structtype[0].field[0].datatype -2014701853
+datatype[13].structtype[0].field[0].id[0]
+datatype[13].structtype[0].inherits[0]
+datatype[13].documenttype[0]
+datatype[13].annotationreftype[0]
+datatype[14].id 264416387
+datatype[14].arraytype[0]
+datatype[14].weightedsettype[0]
+datatype[14].structtype[1]
+datatype[14].structtype[0].name "annotation.sub"
+datatype[14].structtype[0].version 0
+datatype[14].structtype[0].field[0]
+datatype[14].structtype[0].inherits[0]
+datatype[14].documenttype[0]
+datatype[14].annotationreftype[0]
+annotationtype[3]
+annotationtype[0].name "super"
+annotationtype[0].id 668095690
+annotationtype[0].inherits[0]
+annotationtype[1].name "sub"
+annotationtype[1].id 119710016
+annotationtype[1].datatype 264416387
+annotationtype[1].inherits[1]
+annotationtype[1].inherits[0].id 668095690
+annotationtype[2].name "blah"
+annotationtype[2].id -1793776935
+annotationtype[2].datatype -888007918
+annotationtype[2].inherits[0]
diff --git a/document/src/test/document/documentmanager.annotationtypes1.cfg b/document/src/test/document/documentmanager.annotationtypes1.cfg
new file mode 100755
index 00000000000..fe9a666a4ff
--- /dev/null
+++ b/document/src/test/document/documentmanager.annotationtypes1.cfg
@@ -0,0 +1,121 @@
+enablecompression false
+datatype[10]
+datatype[0].id 90768873
+datatype[0].arraytype[0]
+datatype[0].weightedsettype[0]
+datatype[0].structtype[0]
+datatype[0].documenttype[0]
+datatype[0].annotationreftype[1]
+datatype[0].annotationreftype[0].annotation "banana"
+datatype[1].id -1792356253
+datatype[1].arraytype[0]
+datatype[1].weightedsettype[0]
+datatype[1].structtype[0]
+datatype[1].documenttype[0]
+datatype[1].annotationreftype[1]
+datatype[1].annotationreftype[0].annotation "b"
+datatype[2].id 874655144
+datatype[2].arraytype[0]
+datatype[2].weightedsettype[0]
+datatype[2].structtype[0]
+datatype[2].documenttype[0]
+datatype[2].annotationreftype[1]
+datatype[2].annotationreftype[0].annotation "cyclic"
+datatype[3].id 571255414
+datatype[3].arraytype[0]
+datatype[3].weightedsettype[0]
+datatype[3].structtype[1]
+datatype[3].structtype[0].name "annotationsreference.header"
+datatype[3].structtype[0].version 0
+datatype[3].structtype[0].field[0]
+datatype[3].structtype[0].inherits[0]
+datatype[3].documenttype[0]
+datatype[3].annotationreftype[0]
+datatype[4].id 1692909067
+datatype[4].arraytype[0]
+datatype[4].weightedsettype[0]
+datatype[4].structtype[1]
+datatype[4].structtype[0].name "annotationsreference.body"
+datatype[4].structtype[0].version 0
+datatype[4].structtype[0].field[0]
+datatype[4].structtype[0].inherits[0]
+datatype[4].documenttype[0]
+datatype[4].annotationreftype[0]
+datatype[5].id -1448377175
+datatype[5].arraytype[0]
+datatype[5].weightedsettype[0]
+datatype[5].structtype[0]
+datatype[5].documenttype[1]
+datatype[5].documenttype[0].name "annotationsreference"
+datatype[5].documenttype[0].version 0
+datatype[5].documenttype[0].headerstruct 571255414
+datatype[5].documenttype[0].bodystruct 1692909067
+datatype[5].documenttype[0].inherits[0]
+datatype[5].annotationreftype[0]
+datatype[6].id 517946310
+datatype[6].arraytype[0]
+datatype[6].weightedsettype[0]
+datatype[6].structtype[1]
+datatype[6].structtype[0].name "annotation.banana"
+datatype[6].structtype[0].version 0
+datatype[6].structtype[0].field[1]
+datatype[6].structtype[0].field[0].name "brand"
+datatype[6].structtype[0].field[0].datatype 2
+datatype[6].structtype[0].field[0].id[0]
+datatype[6].structtype[0].inherits[0]
+datatype[6].documenttype[0]
+datatype[6].annotationreftype[0]
+datatype[7].id -770307521
+datatype[7].arraytype[0]
+datatype[7].weightedsettype[0]
+datatype[7].structtype[1]
+datatype[7].structtype[0].name "annotation.food"
+datatype[7].structtype[0].version 0
+datatype[7].structtype[0].field[1]
+datatype[7].structtype[0].field[0].name "what"
+datatype[7].structtype[0].field[0].datatype 90768873
+datatype[7].structtype[0].field[0].id[0]
+datatype[7].structtype[0].inherits[0]
+datatype[7].documenttype[0]
+datatype[7].annotationreftype[0]
+datatype[8].id 1781099546
+datatype[8].arraytype[0]
+datatype[8].weightedsettype[0]
+datatype[8].structtype[1]
+datatype[8].structtype[0].name "annotation.cyclic"
+datatype[8].structtype[0].version 0
+datatype[8].structtype[0].field[1]
+datatype[8].structtype[0].field[0].name "blah"
+datatype[8].structtype[0].field[0].datatype 874655144
+datatype[8].structtype[0].field[0].id[0]
+datatype[8].structtype[0].inherits[0]
+datatype[8].documenttype[0]
+datatype[8].annotationreftype[0]
+datatype[9].id 1443829412
+datatype[9].arraytype[0]
+datatype[9].weightedsettype[0]
+datatype[9].structtype[1]
+datatype[9].structtype[0].name "annotation.a"
+datatype[9].structtype[0].version 0
+datatype[9].structtype[0].field[1]
+datatype[9].structtype[0].field[0].name "foo"
+datatype[9].structtype[0].field[0].datatype -1792356253
+datatype[9].structtype[0].field[0].id[0]
+datatype[9].structtype[0].inherits[0]
+datatype[9].documenttype[0]
+datatype[9].annotationreftype[0]
+annotationtype[5]
+annotationtype[0].name "banana"
+annotationtype[0].id -269517759
+annotationtype[0].datatype 517946310
+annotationtype[1].name "b"
+annotationtype[1].id 1966167951
+annotationtype[2].name "food"
+annotationtype[2].id 1918102169
+annotationtype[2].datatype -770307521
+annotationtype[3].name "a"
+annotationtype[3].id 1769416289
+annotationtype[3].datatype 1443829412
+annotationtype[4].name "cyclic"
+annotationtype[4].id -1569831750
+annotationtype[4].datatype 1781099546
diff --git a/document/src/test/document/documentmanager.annotationtypes2.cfg b/document/src/test/document/documentmanager.annotationtypes2.cfg
new file mode 100755
index 00000000000..db24517ab47
--- /dev/null
+++ b/document/src/test/document/documentmanager.annotationtypes2.cfg
@@ -0,0 +1,153 @@
+enablecompression false
+datatype[10]
+datatype[0].id -1406250281
+datatype[0].arraytype[0]
+datatype[0].weightedsettype[0]
+datatype[0].structtype[1]
+datatype[0].structtype[0].name "annotationsinheritance.header"
+datatype[0].structtype[0].version 0
+datatype[0].structtype[0].field[0]
+datatype[0].structtype[0].inherits[0]
+datatype[0].documenttype[0]
+datatype[0].annotationreftype[0]
+datatype[1].id 1181354668
+datatype[1].arraytype[0]
+datatype[1].weightedsettype[0]
+datatype[1].structtype[1]
+datatype[1].structtype[0].name "annotationsinheritance.body"
+datatype[1].structtype[0].version 0
+datatype[1].structtype[0].field[0]
+datatype[1].structtype[0].inherits[0]
+datatype[1].documenttype[0]
+datatype[1].annotationreftype[0]
+datatype[2].id -748546200
+datatype[2].arraytype[0]
+datatype[2].weightedsettype[0]
+datatype[2].structtype[0]
+datatype[2].documenttype[1]
+datatype[2].documenttype[0].name "annotationsinheritance"
+datatype[2].documenttype[0].version 0
+datatype[2].documenttype[0].headerstruct -1406250281
+datatype[2].documenttype[0].bodystruct 1181354668
+datatype[2].documenttype[0].inherits[0]
+datatype[2].annotationreftype[0]
+datatype[3].id 517946310
+datatype[3].arraytype[0]
+datatype[3].weightedsettype[0]
+datatype[3].structtype[1]
+datatype[3].structtype[0].name "annotation.banana"
+datatype[3].structtype[0].version 0
+datatype[3].structtype[0].field[1]
+datatype[3].structtype[0].field[0].name "brand"
+datatype[3].structtype[0].field[0].datatype 2
+datatype[3].structtype[0].field[0].id[0]
+datatype[3].structtype[0].inherits[0]
+datatype[3].documenttype[0]
+datatype[3].annotationreftype[0]
+datatype[4].id -1047410193
+datatype[4].arraytype[0]
+datatype[4].weightedsettype[0]
+datatype[4].structtype[1]
+datatype[4].structtype[0].name "annotation.vehicle"
+datatype[4].structtype[0].version 0
+datatype[4].structtype[0].field[1]
+datatype[4].structtype[0].field[0].name "numwheels"
+datatype[4].structtype[0].field[0].datatype 0
+datatype[4].structtype[0].field[0].id[0]
+datatype[4].structtype[0].inherits[0]
+datatype[4].documenttype[0]
+datatype[4].annotationreftype[0]
+datatype[5].id 249059607
+datatype[5].arraytype[0]
+datatype[5].weightedsettype[0]
+datatype[5].structtype[1]
+datatype[5].structtype[0].name "annotation.car"
+datatype[5].structtype[0].version 0
+datatype[5].structtype[0].field[1]
+datatype[5].structtype[0].field[0].name "color"
+datatype[5].structtype[0].field[0].datatype 2
+datatype[5].structtype[0].field[0].id[0]
+datatype[5].structtype[0].inherits[1]
+datatype[5].structtype[0].inherits[0].name "annotation.vehicle"
+datatype[5].structtype[0].inherits[0].version 0
+datatype[5].documenttype[0]
+datatype[5].annotationreftype[0]
+datatype[6].id -1339036621
+datatype[6].arraytype[0]
+datatype[6].weightedsettype[0]
+datatype[6].structtype[1]
+datatype[6].structtype[0].name "annotation.intern"
+datatype[6].structtype[0].version 0
+datatype[6].structtype[0].field[1]
+datatype[6].structtype[0].field[0].name "enddate"
+datatype[6].structtype[0].field[0].datatype 4
+datatype[6].structtype[0].field[0].id[0]
+datatype[6].structtype[0].inherits[1]
+datatype[6].structtype[0].inherits[0].name "annotation.employee"
+datatype[6].structtype[0].inherits[0].version 0
+datatype[6].documenttype[0]
+datatype[6].annotationreftype[0]
+datatype[7].id -858216177
+datatype[7].arraytype[0]
+datatype[7].weightedsettype[0]
+datatype[7].structtype[1]
+datatype[7].structtype[0].name "annotation.employee"
+datatype[7].structtype[0].version 0
+datatype[7].structtype[0].field[1]
+datatype[7].structtype[0].field[0].name "employeeid"
+datatype[7].structtype[0].field[0].datatype 0
+datatype[7].structtype[0].field[0].id[0]
+datatype[7].structtype[0].inherits[1]
+datatype[7].structtype[0].inherits[0].name "annotation.worker"
+datatype[7].structtype[0].inherits[0].version 0
+datatype[7].documenttype[0]
+datatype[7].annotationreftype[0]
+datatype[8].id -1466283082
+datatype[8].arraytype[0]
+datatype[8].weightedsettype[0]
+datatype[8].structtype[1]
+datatype[8].structtype[0].name "annotation.person"
+datatype[8].structtype[0].version 0
+datatype[8].structtype[0].field[1]
+datatype[8].structtype[0].field[0].name "name"
+datatype[8].structtype[0].field[0].datatype 2
+datatype[8].structtype[0].field[0].id[0]
+datatype[8].structtype[0].inherits[0]
+datatype[8].documenttype[0]
+datatype[8].annotationreftype[0]
+datatype[9].id -1874092641
+datatype[9].arraytype[0]
+datatype[9].weightedsettype[0]
+datatype[9].structtype[1]
+datatype[9].structtype[0].name "annotation.worker"
+datatype[9].structtype[0].version 0
+datatype[9].structtype[0].field[0]
+datatype[9].structtype[0].inherits[1]
+datatype[9].structtype[0].inherits[0].name "annotation.person"
+datatype[9].structtype[0].inherits[0].version 0
+datatype[9].documenttype[0]
+datatype[9].annotationreftype[0]
+annotationtype[8]
+annotationtype[0].name "car"
+annotationtype[0].id -973728295
+annotationtype[0].datatype 249059607
+annotationtype[1].name "banana"
+annotationtype[1].id -269517759
+annotationtype[1].datatype 517946310
+annotationtype[2].name "fruit"
+annotationtype[2].id 877283632
+annotationtype[3].name "person"
+annotationtype[3].id 609952424
+annotationtype[3].datatype -1466283082
+annotationtype[4].name "vehicle"
+annotationtype[4].id 290814930
+annotationtype[4].datatype -1047410193
+annotationtype[5].name "intern"
+annotationtype[5].id 855102455
+annotationtype[5].datatype -1339036621
+annotationtype[6].name "worker"
+annotationtype[6].id 881692980
+annotationtype[6].datatype -1874092641
+annotationtype[7].name "employee"
+annotationtype[7].id 804106508
+annotationtype[7].datatype -858216177
diff --git a/document/src/test/document/documentmanager.cfg b/document/src/test/document/documentmanager.cfg
new file mode 100644
index 00000000000..d77c2b17460
--- /dev/null
+++ b/document/src/test/document/documentmanager.cfg
@@ -0,0 +1,108 @@
+datatype[11]
+datatype[0].id -1365874599
+datatype[0].arraytype[0]
+datatype[0].weightedsettype[0]
+datatype[0].structtype[1]
+datatype[0].structtype[0].name foobar.header
+datatype[0].structtype[0].version 9
+datatype[0].structtype[0].field[1]
+datatype[0].structtype[0].field[0].name foobarfield1
+datatype[0].structtype[0].field[0].id[0]
+datatype[0].structtype[0].field[0].datatype 4
+datatype[0].documenttype[0]
+datatype[1].id 278604398
+datatype[1].arraytype[0]
+datatype[1].weightedsettype[0]
+datatype[1].structtype[1]
+datatype[1].structtype[0].name foobar.body
+datatype[1].structtype[0].version 9
+datatype[1].structtype[0].field[1]
+datatype[1].structtype[0].field[0].name foobarfield0
+datatype[1].structtype[0].field[0].id[0]
+datatype[1].structtype[0].field[0].datatype 2
+datatype[1].documenttype[0]
+datatype[2].id 378030104
+datatype[2].arraytype[0]
+datatype[2].weightedsettype[0]
+datatype[2].structtype[0]
+datatype[2].documenttype[1]
+datatype[2].documenttype[0].name foobar
+datatype[2].documenttype[0].version 9
+datatype[2].documenttype[0].inherits[0]
+datatype[2].documenttype[0].headerstruct -1365874599
+datatype[2].documenttype[0].bodystruct 278604398
+datatype[3].id 673066331
+datatype[3].arraytype[0]
+datatype[3].weightedsettype[0]
+datatype[3].structtype[1]
+datatype[3].structtype[0].name banana.header
+datatype[3].structtype[0].version 234
+datatype[3].structtype[0].field[1]
+datatype[3].structtype[0].field[0].name bananafield0
+datatype[3].structtype[0].field[0].id[0]
+datatype[3].structtype[0].field[0].datatype 16
+datatype[3].documenttype[0]
+datatype[4].id -176986064
+datatype[4].arraytype[0]
+datatype[4].weightedsettype[0]
+datatype[4].structtype[1]
+datatype[4].structtype[0].name banana.body
+datatype[4].structtype[0].version 234
+datatype[4].structtype[0].field[0]
+datatype[4].documenttype[0]
+datatype[5].id 556449802
+datatype[5].arraytype[0]
+datatype[5].weightedsettype[0]
+datatype[5].structtype[0]
+datatype[5].documenttype[1]
+datatype[5].documenttype[0].name banana
+datatype[5].documenttype[0].version 234
+datatype[5].documenttype[0].inherits[1]
+datatype[5].documenttype[0].inherits[0].name foobar
+datatype[5].documenttype[0].inherits[0].version 9
+datatype[5].documenttype[0].headerstruct 673066331
+datatype[5].documenttype[0].bodystruct -176986064
+datatype[6].id -858669928
+datatype[6].arraytype[0]
+datatype[6].weightedsettype[0]
+datatype[6].structtype[1]
+datatype[6].structtype[0].name customtypes.header
+datatype[6].structtype[0].version 3
+datatype[6].structtype[0].field[0]
+datatype[6].documenttype[0]
+datatype[7].id 99
+datatype[7].arraytype[1]
+datatype[7].arraytype[0].datatype 1
+datatype[7].weightedsettype[0]
+datatype[7].structtype[0]
+datatype[7].documenttype[0]
+datatype[8].id 4003
+datatype[8].arraytype[1]
+datatype[8].arraytype[0].datatype 99
+datatype[8].weightedsettype[0]
+datatype[8].structtype[0]
+datatype[8].documenttype[0]
+datatype[9].id 2142817261
+datatype[9].arraytype[0]
+datatype[9].weightedsettype[0]
+datatype[9].structtype[1]
+datatype[9].structtype[0].name customtypes.body
+datatype[9].structtype[0].version 3
+datatype[9].structtype[0].field[2]
+datatype[9].structtype[0].field[0].name arrayfloat
+datatype[9].structtype[0].field[0].id[0]
+datatype[9].structtype[0].field[0].datatype 99
+datatype[9].structtype[0].field[1].name arrayarrayfloat
+datatype[9].structtype[0].field[1].id[0]
+datatype[9].structtype[0].field[1].datatype 4003
+datatype[9].documenttype[0]
+datatype[10].id -1500313747
+datatype[10].arraytype[0]
+datatype[10].weightedsettype[0]
+datatype[10].structtype[0]
+datatype[10].documenttype[1]
+datatype[10].documenttype[0].name customtypes
+datatype[10].documenttype[0].version 3
+datatype[10].documenttype[0].inherits[0]
+datatype[10].documenttype[0].headerstruct -858669928
+datatype[10].documenttype[0].bodystruct 2142817261
diff --git a/document/src/test/document/documentmanager.map.cfg b/document/src/test/document/documentmanager.map.cfg
new file mode 100644
index 00000000000..42b6e356571
--- /dev/null
+++ b/document/src/test/document/documentmanager.map.cfg
@@ -0,0 +1,9 @@
+datatype[2]
+datatype[0].id 1000
+datatype[0].maptype[1]
+datatype[0].maptype[0].keytype 2
+datatype[0].maptype[0].valtype 1001
+datatype[1].id 1001
+datatype[1].maptype[1]
+datatype[1].maptype[0].keytype 2
+datatype[1].maptype[0].valtype 2
diff --git a/document/src/test/document/documentmanager.sombrero1.cfg b/document/src/test/document/documentmanager.sombrero1.cfg
new file mode 100644
index 00000000000..1a8bfed7649
--- /dev/null
+++ b/document/src/test/document/documentmanager.sombrero1.cfg
@@ -0,0 +1,68 @@
+enablecompression false
+datatype[5]
+datatype[0].id 2131819585
+datatype[0].arraytype[0]
+datatype[0].weightedsettype[0]
+datatype[0].structtype[1]
+datatype[0].structtype[0].name "example.app.header"
+datatype[0].structtype[0].version 1
+datatype[0].structtype[0].field[1]
+datatype[0].structtype[0].field[0].name "text"
+datatype[0].structtype[0].field[0].datatype 2
+datatype[0].structtype[0].field[0].id[0]
+datatype[0].structtype[0].inherits[0]
+datatype[0].documenttype[0]
+datatype[0].annotationreftype[0]
+datatype[1].id 1636432470
+datatype[1].arraytype[0]
+datatype[1].weightedsettype[0]
+datatype[1].structtype[1]
+datatype[1].structtype[0].name "example.app.body"
+datatype[1].structtype[0].version 1
+datatype[1].structtype[0].field[0]
+datatype[1].structtype[0].inherits[0]
+datatype[1].documenttype[0]
+datatype[1].annotationreftype[0]
+datatype[2].id 112411168
+datatype[2].arraytype[0]
+datatype[2].weightedsettype[0]
+datatype[2].structtype[0]
+datatype[2].documenttype[1]
+datatype[2].documenttype[0].name "example.app"
+datatype[2].documenttype[0].version 1
+datatype[2].documenttype[0].headerstruct 2131819585
+datatype[2].documenttype[0].bodystruct 1636432470
+datatype[2].documenttype[0].inherits[0]
+datatype[2].annotationreftype[0]
+datatype[3].id -897633294
+datatype[3].arraytype[0]
+datatype[3].weightedsettype[0]
+datatype[3].structtype[1]
+datatype[3].structtype[0].name "annotation.base"
+datatype[3].structtype[0].version 0
+datatype[3].structtype[0].field[1]
+datatype[3].structtype[0].field[0].name "x"
+datatype[3].structtype[0].field[0].datatype 0
+datatype[3].structtype[0].field[0].id[0]
+datatype[3].structtype[0].inherits[0]
+datatype[3].documenttype[0]
+datatype[3].annotationreftype[0]
+datatype[4].id 1724438688
+datatype[4].arraytype[0]
+datatype[4].weightedsettype[0]
+datatype[4].structtype[1]
+datatype[4].structtype[0].name "annotation.derived"
+datatype[4].structtype[0].version 0
+datatype[4].structtype[0].field[0]
+datatype[4].structtype[0].inherits[1]
+datatype[4].structtype[0].inherits[0].name "annotation.base"
+datatype[4].structtype[0].inherits[0].version 0
+datatype[4].documenttype[0]
+datatype[4].annotationreftype[0]
+annotationtype[2]
+annotationtype[0].name "base"
+annotationtype[0].id 1091303828
+annotationtype[0].datatype -897633294
+annotationtype[1].name "derived"
+annotationtype[1].id 1799352802
+annotationtype[1].datatype 1724438688
diff --git a/document/src/test/document/documentmanager.structsanyorder.cfg b/document/src/test/document/documentmanager.structsanyorder.cfg
new file mode 100755
index 00000000000..1d649a79d3b
--- /dev/null
+++ b/document/src/test/document/documentmanager.structsanyorder.cfg
@@ -0,0 +1,88 @@
+enablecompression false
+datatype[7]
+datatype[0].id 97614088
+datatype[0].arraytype[0]
+datatype[0].weightedsettype[0]
+datatype[0].structtype[1]
+datatype[0].structtype[0].name "foo"
+datatype[0].structtype[0].version 0
+datatype[0].structtype[0].field[1]
+datatype[0].structtype[0].field[0].name "s1"
+datatype[0].structtype[0].field[0].datatype 0
+datatype[0].structtype[0].field[0].id[0]
+datatype[0].structtype[0].inherits[0]
+datatype[0].documenttype[0]
+datatype[0].annotationreftype[0]
+datatype[1].id 109267174
+datatype[1].arraytype[0]
+datatype[1].weightedsettype[0]
+datatype[1].structtype[1]
+datatype[1].structtype[0].name "sct"
+datatype[1].structtype[0].version 0
+datatype[1].structtype[0].field[4]
+datatype[1].structtype[0].field[0].name "s1"
+datatype[1].structtype[0].field[0].datatype 2
+datatype[1].structtype[0].field[0].id[0]
+datatype[1].structtype[0].field[1].name "s2"
+datatype[1].structtype[0].field[1].datatype 2
+datatype[1].structtype[0].field[1].id[0]
+datatype[1].structtype[0].field[2].name "s3"
+datatype[1].structtype[0].field[2].datatype 109267174
+datatype[1].structtype[0].field[2].id[0]
+datatype[1].structtype[0].field[3].name "s4"
+datatype[1].structtype[0].field[3].datatype 97614088
+datatype[1].structtype[0].field[3].id[0]
+datatype[1].structtype[0].inherits[0]
+datatype[1].documenttype[0]
+datatype[1].annotationreftype[0]
+datatype[2].id -1244829667
+datatype[2].arraytype[1]
+datatype[2].arraytype[0].datatype 109267174
+datatype[2].weightedsettype[0]
+datatype[2].structtype[0]
+datatype[2].documenttype[0]
+datatype[2].annotationreftype[0]
+datatype[3].id -364910881
+datatype[3].arraytype[0]
+datatype[3].weightedsettype[0]
+datatype[3].structtype[1]
+datatype[3].structtype[0].name "annotationsimplicitstruct.header"
+datatype[3].structtype[0].version 0
+datatype[3].structtype[0].field[2]
+datatype[3].structtype[0].field[0].name "structfield"
+datatype[3].structtype[0].field[0].datatype 109267174
+datatype[3].structtype[0].field[0].id[0]
+datatype[3].structtype[0].field[1].name "structarrayfield"
+datatype[3].structtype[0].field[1].datatype -1244829667
+datatype[3].structtype[0].field[1].id[0]
+datatype[3].structtype[0].inherits[0]
+datatype[3].documenttype[0]
+datatype[3].annotationreftype[0]
+datatype[4].id -1503592268
+datatype[4].arraytype[0]
+datatype[4].weightedsettype[0]
+datatype[4].structtype[1]
+datatype[4].structtype[0].name "annotationsimplicitstruct.body"
+datatype[4].structtype[0].version 0
+datatype[4].structtype[0].field[0]
+datatype[4].structtype[0].inherits[0]
+datatype[4].documenttype[0]
+datatype[4].annotationreftype[0]
+datatype[5].id -2099544992
+datatype[5].arraytype[0]
+datatype[5].weightedsettype[0]
+datatype[5].structtype[0]
+datatype[5].documenttype[1]
+datatype[5].documenttype[0].name "annotationsimplicitstruct"
+datatype[5].documenttype[0].version 0
+datatype[5].documenttype[0].headerstruct -364910881
+datatype[5].documenttype[0].bodystruct -1503592268
+datatype[5].documenttype[0].inherits[0]
+datatype[5].annotationreftype[0]
+datatype[6].id -1486737430
+datatype[6].arraytype[1]
+datatype[6].arraytype[0].datatype 2
+datatype[6].weightedsettype[0]
+datatype[6].structtype[0]
+datatype[6].documenttype[0]
+datatype[6].annotationreftype[0]
diff --git a/document/src/test/document/documentmanager.updated.cfg b/document/src/test/document/documentmanager.updated.cfg
new file mode 100644
index 00000000000..7e2a0bb682c
--- /dev/null
+++ b/document/src/test/document/documentmanager.updated.cfg
@@ -0,0 +1,108 @@
+datatype[11]
+datatype[0].id -1365874599
+datatype[0].arraytype[0]
+datatype[0].weightedsettype[0]
+datatype[0].structtype[1]
+datatype[0].structtype[0].name foobar.header
+datatype[0].structtype[0].version 9
+datatype[0].structtype[0].field[1]
+datatype[0].structtype[0].field[0].name foobarfield1
+datatype[0].structtype[0].field[0].id[0]
+datatype[0].structtype[0].field[0].datatype 4
+datatype[0].documenttype[0]
+datatype[1].id 278604398
+datatype[1].arraytype[0]
+datatype[1].weightedsettype[0]
+datatype[1].structtype[1]
+datatype[1].structtype[0].name foobar.body
+datatype[1].structtype[0].version 9
+datatype[1].structtype[0].field[1]
+datatype[1].structtype[0].field[0].name foobarfield0
+datatype[1].structtype[0].field[0].id[0]
+datatype[1].structtype[0].field[0].datatype 2
+datatype[1].documenttype[0]
+datatype[2].id 378030104
+datatype[2].arraytype[0]
+datatype[2].weightedsettype[0]
+datatype[2].structtype[0]
+datatype[2].documenttype[1]
+datatype[2].documenttype[0].name foobar
+datatype[2].documenttype[0].version 9
+datatype[2].documenttype[0].inherits[0]
+datatype[2].documenttype[0].headerstruct -1365874599
+datatype[2].documenttype[0].bodystruct 278604398
+datatype[3].id 673066331
+datatype[3].arraytype[0]
+datatype[3].weightedsettype[0]
+datatype[3].structtype[1]
+datatype[3].structtype[0].name banana.header
+datatype[3].structtype[0].version 234
+datatype[3].structtype[0].field[2]
+datatype[3].structtype[0].field[0].name bananafield0
+datatype[3].structtype[0].field[0].id[0]
+datatype[3].structtype[0].field[0].datatype 16
+datatype[3].structtype[0].field[1].name newfield
+datatype[3].structtype[0].field[1].id[0]
+datatype[3].structtype[0].field[1].datatype 2
+datatype[3].documenttype[0]
+datatype[4].id -176986064
+datatype[4].arraytype[0]
+datatype[4].weightedsettype[0]
+datatype[4].structtype[1]
+datatype[4].structtype[0].name banana.body
+datatype[4].structtype[0].version 234
+datatype[4].structtype[0].field[0]
+datatype[4].documenttype[0]
+datatype[5].id 556449802
+datatype[5].arraytype[0]
+datatype[5].weightedsettype[0]
+datatype[5].structtype[0]
+datatype[5].documenttype[1]
+datatype[5].documenttype[0].name banana
+datatype[5].documenttype[0].version 234
+datatype[5].documenttype[0].inherits[1]
+datatype[5].documenttype[0].inherits[0].name foobar
+datatype[5].documenttype[0].inherits[0].version 9
+datatype[5].documenttype[0].headerstruct 673066331
+datatype[5].documenttype[0].bodystruct -176986064
+datatype[6].id -858669928
+datatype[6].arraytype[0]
+datatype[6].weightedsettype[0]
+datatype[6].structtype[1]
+datatype[6].structtype[0].name customtypes.header
+datatype[6].structtype[0].version 3
+datatype[6].structtype[0].field[0]
+datatype[6].documenttype[0]
+datatype[7].id 99
+datatype[7].arraytype[1]
+datatype[7].arraytype[0].datatype 1
+datatype[7].weightedsettype[0]
+datatype[7].structtype[0]
+datatype[7].documenttype[0]
+datatype[8].id 4003
+datatype[8].arraytype[1]
+datatype[8].arraytype[0].datatype 99
+datatype[8].weightedsettype[0]
+datatype[8].structtype[0]
+datatype[8].documenttype[0]
+datatype[9].id 2142817261
+datatype[9].arraytype[0]
+datatype[9].weightedsettype[0]
+datatype[9].structtype[1]
+datatype[9].structtype[0].name customtypes.body
+datatype[9].structtype[0].version 3
+datatype[9].structtype[0].field[1]
+datatype[9].structtype[0].field[0].name arrayarrayfloat
+datatype[9].structtype[0].field[0].id[0]
+datatype[9].structtype[0].field[0].datatype 4003
+datatype[9].documenttype[0]
+datatype[10].id -1500313747
+datatype[10].arraytype[0]
+datatype[10].weightedsettype[0]
+datatype[10].structtype[0]
+datatype[10].documenttype[1]
+datatype[10].documenttype[0].name customtypes
+datatype[10].documenttype[0].version 3
+datatype[10].documenttype[0].inherits[0]
+datatype[10].documenttype[0].headerstruct -858669928
+datatype[10].documenttype[0].bodystruct 2142817261
diff --git a/document/src/test/document/serializecpp-lz4-level9.dat b/document/src/test/document/serializecpp-lz4-level9.dat
new file mode 100644
index 00000000000..9bc68803245
--- /dev/null
+++ b/document/src/test/document/serializecpp-lz4-level9.dat
diff --git a/document/src/test/document/serializecpp.dat b/document/src/test/document/serializecpp.dat
new file mode 100644
index 00000000000..7e50220a046
--- /dev/null
+++ b/document/src/test/document/serializecpp.dat
diff --git a/document/src/test/document/serializecppsplit_body.dat b/document/src/test/document/serializecppsplit_body.dat
new file mode 100755
index 00000000000..3f5db21dd18
--- /dev/null
+++ b/document/src/test/document/serializecppsplit_body.dat
diff --git a/document/src/test/document/serializecppsplit_header.dat b/document/src/test/document/serializecppsplit_header.dat
new file mode 100755
index 00000000000..8b938c499d4
--- /dev/null
+++ b/document/src/test/document/serializecppsplit_header.dat
diff --git a/document/src/test/document/serializeupdatecpp.dat b/document/src/test/document/serializeupdatecpp.dat
new file mode 100644
index 00000000000..fba49d562ec
--- /dev/null
+++ b/document/src/test/document/serializeupdatecpp.dat
diff --git a/document/src/test/files/.gitignore b/document/src/test/files/.gitignore
new file mode 100644
index 00000000000..770a2c59576
--- /dev/null
+++ b/document/src/test/files/.gitignore
@@ -0,0 +1,5 @@
+addfieldser.dat
+testser-split.body.dat
+testser-split.header.dat
+testser.dat
+updateser.dat
diff --git a/document/src/test/java/com/yahoo/document/BucketIdFactoryTestCase.java b/document/src/test/java/com/yahoo/document/BucketIdFactoryTestCase.java
new file mode 100644
index 00000000000..e511e4cf378
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/BucketIdFactoryTestCase.java
@@ -0,0 +1,130 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import com.yahoo.document.BucketIdFactory;
+import com.yahoo.document.BucketId;
+import com.yahoo.document.DocumentId;
+import com.yahoo.document.idstring.*;
+
+/**
+ * Date: Sep 7, 2007
+ *
+ * @author <a href="mailto:humbe@yahoo-inc.com">H&aring;kon Humberset</a>
+ */
+public class BucketIdFactoryTestCase extends junit.framework.TestCase {
+
+    public BucketIdFactoryTestCase(String name) {
+        super(name);
+    }
+
+    public void testNormalUsage() {
+        BucketIdFactory factory = new BucketIdFactory();
+        assertEquals(BucketId.COUNT_BITS, factory.getCountBitCount());
+
+        assertEquals(64, factory.getLocationBitCount() + factory.getGidBitCount() + BucketId.COUNT_BITS);
+    }
+
+    private class Hex {
+        long value;
+
+        Hex(long val) {
+            value = val;
+        }
+
+        public boolean equals(Object o) {
+            return (o instanceof Hex && value == ((Hex) o).value);
+        }
+
+        public String toString() {
+            return Long.toHexString(value);
+        }
+    }
+
+    public void testBucketGeneration() {
+        BucketIdFactory factory = new BucketIdFactory(32, 26, 6);
+        DocumentId doc1 = new DocumentId(new DocIdString("ns", "spec"));
+        DocumentId doc2 = new DocumentId(new DocIdString("ns2", "spec"));
+        DocumentId doc3 = new DocumentId(new DocIdString("ns", "spec2"));
+        DocumentId userDoc1 = new DocumentId(new UserDocIdString("ns", 0x12, "spec"));
+        DocumentId userDoc2 = new DocumentId(new UserDocIdString("ns2", 0x12, "spec2"));
+        DocumentId userDoc3 = new DocumentId(new UserDocIdString("ns", 0x13, "spec"));
+        DocumentId groupDoc1 = new DocumentId(new GroupDocIdString("ns", "yahoo.com", "spec"));
+        DocumentId groupDoc2 = new DocumentId(new GroupDocIdString("ns2", "yahoo.com", "spec2"));
+        DocumentId groupDoc3 = new DocumentId(new GroupDocIdString("ns", "yahoo", "spec"));
+        DocumentId orderDoc1 = new DocumentId(new OrderDocIdString("ns", "13", 31, 19, 1268182861, "foo"));
+        DocumentId orderDoc2 = new DocumentId(new OrderDocIdString("ns", "13", 31, 19, 1205110861, "foo"));
+        DocumentId orderDoc3 = new DocumentId(new OrderDocIdString("ns", "13", 31, 19, 1205715661, "foo"));
+        DocumentId orderDoc4 = new DocumentId(new OrderDocIdString("ns", "13", 4, 0, 2, "foo"));
+        DocumentId orderDoc5 = new DocumentId(new OrderDocIdString("ns", "13", 4, 0, 4, "foo"));
+        DocumentId orderDoc6 = new DocumentId(new OrderDocIdString("ns", "13", 4, 0, 11, "foo"));
+
+        BucketId docBucket1 = factory.getBucketId(doc1);
+        BucketId docBucket2 = factory.getBucketId(doc2);
+        BucketId docBucket3 = factory.getBucketId(doc3);
+        BucketId userDocBucket1 = factory.getBucketId(userDoc1);
+        BucketId userDocBucket2 = factory.getBucketId(userDoc2);
+        BucketId userDocBucket3 = factory.getBucketId(userDoc3);
+        BucketId groupDocBucket1 = factory.getBucketId(groupDoc1);
+        BucketId groupDocBucket2 = factory.getBucketId(groupDoc2);
+        BucketId groupDocBucket3 = factory.getBucketId(groupDoc3);
+        BucketId orderDocBucket1 = factory.getBucketId(orderDoc1);
+        BucketId orderDocBucket2 = factory.getBucketId(orderDoc2);
+        BucketId orderDocBucket3 = factory.getBucketId(orderDoc3);
+        BucketId orderDocBucket4 = factory.getBucketId(orderDoc4);
+        BucketId orderDocBucket5 = factory.getBucketId(orderDoc5);
+        BucketId orderDocBucket6 = factory.getBucketId(orderDoc6);
+
+        assertEquals(new Hex(0xe99703f200000012l), new Hex(userDocBucket1.getRawId()));
+        assertEquals(new Hex(0xebfa518a00000012l), new Hex(userDocBucket2.getRawId()));
+        assertEquals(new Hex(0xeac1850800000013l), new Hex(userDocBucket3.getRawId()));
+
+        assertEquals(new Hex(0xe90ce4b09a1acd50l), new Hex(groupDocBucket1.getRawId()));
+        assertEquals(new Hex(0xe9cedaa49a1acd50l), new Hex(groupDocBucket2.getRawId()));
+        assertEquals(new Hex(0xe8cdb18bafe81f24l), new Hex(groupDocBucket3.getRawId()));
+
+        assertEquals(new Hex(0xe980c9abd5fd8d11l), new Hex(docBucket1.getRawId()));
+        assertEquals(new Hex(0xeafe870c5f9c37b9l), new Hex(docBucket2.getRawId()));
+        assertEquals(new Hex(0xeaebe9473ecbcd69l), new Hex(docBucket3.getRawId()));
+
+        assertEquals(new Hex(0xeae764e90000000dl), new Hex(orderDocBucket1.getRawId()));
+        assertEquals(new Hex(0xeacb85f10000000dl), new Hex(orderDocBucket2.getRawId()));
+        assertEquals(new Hex(0xea68ddf10000000dl), new Hex(orderDocBucket3.getRawId()));
+
+        assertEquals(new Hex(0xe87526540000000dl), new Hex(orderDocBucket4.getRawId()));
+        assertEquals(new Hex(0xea59f8f20000000dl), new Hex(orderDocBucket5.getRawId()));
+        assertEquals(new Hex(0xe9eb703d0000000dl), new Hex(orderDocBucket6.getRawId()));
+    }
+
+    //Actually a BucketId testcase ...
+    public void testBidContainsBid() {
+        BucketId bid = new BucketId(18, 0x123456789L);
+
+        assert(bid.contains(new BucketId(20, 0x123456789L)));
+        assert(bid.contains(new BucketId(18, 0x888f56789L)));
+        assert(bid.contains(new BucketId(24, 0x888456789L)));
+        assert(!bid.contains(new BucketId(24, 0x888886789L)));
+        assert(!bid.contains(new BucketId(16, 0x123456789L)));
+    }
+
+    public void testBidContainsDocId() {
+        DocumentId docId = new DocumentId("userdoc:recovery:18:99999");
+        BucketIdFactory factory = new BucketIdFactory(32, 26, 6);
+        BucketId bid = new BucketId(16, 0x12L);
+        assert(bid.contains(docId, factory));
+        //split on '0'
+        bid = new BucketId(17, 0x12L);
+        assert(bid.contains(docId, factory));
+        //split on '1'
+        bid = new BucketId(17, (1L<<16) + 0x12L);
+        assert(!bid.contains(docId, factory));
+    }
+
+    public void testBucketIdSerializationAndCompare() {
+        BucketId bid = new BucketId(18, 0x123456789L);
+
+        assertEquals(bid, new BucketId(bid.toString()));
+        assertEquals(0, bid.compareTo(new BucketId(18, 0x123456789L)));
+    }
+
+}
+
diff --git a/document/src/test/java/com/yahoo/document/DataTypeNameTestCase.java b/document/src/test/java/com/yahoo/document/DataTypeNameTestCase.java
new file mode 100644
index 00000000000..c7d9ac93592
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/DataTypeNameTestCase.java
@@ -0,0 +1,23 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class DataTypeNameTestCase {
+
+    @Test
+    @SuppressWarnings("deprecation")
+    public void requireThatAccessorsWork() {
+        DataTypeName name = new DataTypeName("foo");
+        assertEquals("foo", name.getName());
+        assertEquals("foo", name.toString());
+        assertFalse(name.equals(new Object()));
+    }
+
+}
diff --git a/document/src/test/java/com/yahoo/document/DataTypeTestCase.java b/document/src/test/java/com/yahoo/document/DataTypeTestCase.java
new file mode 100644
index 00000000000..89f22e97d5d
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/DataTypeTestCase.java
@@ -0,0 +1,146 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import com.yahoo.document.datatypes.*;
+
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+/**
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public class DataTypeTestCase {
+
+    @Test
+    public void testWeightedSetTypes() {
+        DataType stringDefault = DataType.getWeightedSet(DataType.STRING);
+        DataType stringTag=DataType.getWeightedSet(DataType.STRING,true,true);
+        assertTrue(stringDefault.equals(stringDefault));
+        assertTrue(stringTag.equals(stringTag));
+        assertFalse(stringDefault.equals(stringTag));
+        assertEquals("WeightedSet<string>",stringDefault.getName());
+        assertEquals(18, stringTag.getCode());
+        //assertEquals("WeightedSet<string>;Add;Remove",stringTag.getName());
+        assertEquals("tag",stringTag.getName());
+    }
+
+    @Test
+    public void testTagType() {
+        DocumentTypeManager manager = new DocumentTypeManager();
+        //assertEquals(DataType.getWeightedSet(DataType.STRING,true,true),DataType.TAG.getBaseType());
+        assertNotNull(manager.getDataType("tag"));
+        assertEquals(DataType.TAG, manager.getDataType("tag"));
+        assertEquals(manager.getDataType("tag").getCode(), 18);
+        assertEquals(DataType.getWeightedSet(DataType.STRING,true,true).getCode(), 18);
+    }
+
+    @Test
+    public void requireThatPredicateDataTypeIsNamedPredicate() {
+        assertEquals("predicate", DataType.PREDICATE.getName());
+    }
+
+    @Test
+    public void requireThatPredicateDataTypeHasId20() {
+        assertEquals(20, DataType.PREDICATE.getId());
+    }
+
+    @Test
+    public void requireThatPredicateDataTypeYieldsPredicateFieldValue() {
+        assertEquals(PredicateFieldValue.class, DataType.PREDICATE.createFieldValue().getClass());
+    }
+
+    @Test
+    public void testCreateFieldValueWithArg() {
+        {
+            ByteFieldValue bfv = (ByteFieldValue) DataType.BYTE.createFieldValue((byte) 4);
+            assertEquals((byte) 4, bfv.getByte());
+        }
+        {
+            ByteFieldValue bfv2 = (ByteFieldValue) DataType.BYTE.createFieldValue(4);
+            assertEquals((byte) 4, bfv2.getByte());
+        }
+        {
+            DoubleFieldValue dfv = (DoubleFieldValue) DataType.DOUBLE.createFieldValue(5.5d);
+            assertEquals(5.5d, dfv.getDouble(), 1E-6);
+        }
+        {
+            FloatFieldValue ffv = (FloatFieldValue) DataType.FLOAT.createFieldValue(5.5f);
+            assertEquals(5.5f, ffv.getFloat(), 1E-6);
+        }
+        {
+            IntegerFieldValue ifv = (IntegerFieldValue) DataType.INT.createFieldValue(5);
+            assertEquals(5, ifv.getInteger());
+        }
+        {
+            LongFieldValue lfv = (LongFieldValue) DataType.LONG.createFieldValue(34L);
+            assertEquals(34L, lfv.getLong());
+        }
+        {
+            StringFieldValue sfv = (StringFieldValue) DataType.STRING.createFieldValue("foo");
+            assertEquals("foo", sfv.getString());
+        }
+        // TODO: the 3 following should be made to not throw a silent ReflectiveOperationException in createFieldValue
+        {
+            Map<String, Integer> wsetMap = new LinkedHashMap<>();
+            wsetMap.put("foo", 1);
+            WeightedSet<StringFieldValue> ws = (WeightedSet<StringFieldValue>) DataType.getWeightedSet(DataType.STRING).createFieldValue(wsetMap);
+            assertEquals(ws.get(new StringFieldValue("foo")), new Integer(1));
+        }
+        {
+            List<String> arrayArray = new ArrayList<>();
+            arrayArray.add("foo");
+            Array<StringFieldValue> array = (Array<StringFieldValue>) DataType.getArray(DataType.STRING).createFieldValue(arrayArray);
+            assertEquals(array.get(0), new StringFieldValue("foo"));
+        }
+        {
+            Map<String, String> mapMap = new LinkedHashMap<>();
+            mapMap.put("foo", "bar");
+            MapFieldValue<StringFieldValue, StringFieldValue> map = (MapFieldValue<StringFieldValue, StringFieldValue>) DataType.getMap(DataType.STRING, DataType.STRING).createFieldValue(mapMap);
+            assertEquals(map.get(new StringFieldValue("foo")), new StringFieldValue("bar"));
+        }
+    }
+
+    @Test
+    public void testIllegalFieldPathsInArrayDataType() {
+        ArrayDataType adt = DataType.getArray(DataType.STRING);
+        try {
+            adt.buildFieldPath("[      ");
+            fail("Should have gotten exception for illegal field path.");
+        } catch (IllegalArgumentException iae) {
+            // ok!
+        }
+    }
+
+    @Test
+    public void testCloningArrayDataType() {
+        ArrayDataType adt = DataType.getArray(DataType.STRING);
+        ArrayDataType adtClone = adt.clone();
+
+        assertNotSame(adt, adtClone);
+        assertEquals(adt, adtClone);
+        assertEquals(adt.getNestedType(), adtClone.getNestedType());
+        //we should consider NOT cloning primitive types, but they are mutable and just ugly, so just clone them
+    }
+
+    @Test
+    @SuppressWarnings("unchecked")
+    public void testInstantiatingArray() {
+        ArrayDataType adt = DataType.getArray(DataType.STRING);
+        Array<StringFieldValue> val = adt.createFieldValue();
+        val.add(new StringFieldValue("foobar"));
+        assertEquals(1, val.size());
+    }
+
+    @Test
+    public void requireThatCompareToIsImplemented() {
+        assertEquals(0, DataType.INT.compareTo(DataType.INT));
+        assertTrue(DataType.INT.compareTo(DataType.STRING) < 0);
+        assertTrue(DataType.STRING.compareTo(DataType.INT) > 0);
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/DocInDocTestCase.java b/document/src/test/java/com/yahoo/document/DocInDocTestCase.java
new file mode 100644
index 00000000000..ec6e287d105
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/DocInDocTestCase.java
@@ -0,0 +1,52 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import com.yahoo.document.datatypes.Array;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.document.serialization.*;
+import com.yahoo.io.GrowableByteBuffer;
+import junit.framework.TestCase;
+import org.junit.Test;
+
+import static junit.framework.Assert.assertEquals;
+import static junit.framework.Assert.assertNotSame;
+import static junit.framework.Assert.assertTrue;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class DocInDocTestCase extends TestCase {
+
+    @Test
+    public void testDocInDoc() {
+        DocumentTypeManager manager = new DocumentTypeManager();
+        DocumentTypeManagerConfigurer.configure(manager, "file:src/test/java/com/yahoo/document/documentmanager.docindoc.cfg");
+
+        Document inner1 = new Document(manager.getDocumentType("docindoc"), "doc:inner:number:one");
+        inner1.setFieldValue("name", new StringFieldValue("Donald Duck"));
+        inner1.setFieldValue("content", new StringFieldValue("Lives in Duckburg"));
+        Document inner2 = new Document(manager.getDocumentType("docindoc"), "doc:inner:number:two");
+        inner2.setFieldValue("name", new StringFieldValue("Uncle Scrooge"));
+        inner2.setFieldValue("content", new StringFieldValue("Lives in Duckburg, too."));
+
+        Array<Document> innerArray = (Array<Document>) manager.getDocumentType("outerdoc").getField("innerdocuments").getDataType().createFieldValue();
+        innerArray.add(inner1);
+        innerArray.add(inner2);
+
+        Document outer = new Document(manager.getDocumentType("outerdoc"), "doc:outer:the:only:one");
+        outer.setFieldValue("innerdocuments", innerArray);
+
+        DocumentSerializer serializer = DocumentSerializerFactory.create42();
+        serializer.write(outer);
+
+        GrowableByteBuffer buf = serializer.getBuf();
+        buf.flip();
+
+        DocumentDeserializer deserializer = DocumentDeserializerFactory.create42(manager, buf);
+        Document outerDeserialized = new Document(deserializer);
+
+        assertEquals(outer, outerDeserialized);
+        assertNotSame(outer, outerDeserialized);
+
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/DocumentCalculatorTestCase.java b/document/src/test/java/com/yahoo/document/DocumentCalculatorTestCase.java
new file mode 100755
index 00000000000..0e7eb72ee07
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/DocumentCalculatorTestCase.java
@@ -0,0 +1,112 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import com.yahoo.document.datatypes.ByteFieldValue;
+import com.yahoo.document.datatypes.DoubleFieldValue;
+import com.yahoo.document.datatypes.IntegerFieldValue;
+import com.yahoo.document.datatypes.LongFieldValue;
+
+import java.util.HashMap;
+
+/**
+ * @author thomasg
+ */
+public class DocumentCalculatorTestCase extends junit.framework.TestCase {
+
+    DocumentType testDocType = null;
+    DocumentTypeManager docMan;
+    Document doc;
+
+    public void setUp() {
+        docMan = new DocumentTypeManager();
+        testDocType = new DocumentType("testdoc");
+
+        testDocType.addHeaderField("byteattr", DataType.BYTE);
+        testDocType.addHeaderField("intattr", DataType.INT);
+        testDocType.addHeaderField("longattr", DataType.LONG);
+        testDocType.addHeaderField("doubleattr", DataType.DOUBLE);
+        testDocType.addHeaderField("missingattr", DataType.INT);
+
+        docMan.registerDocumentType(testDocType);
+        doc = new Document(testDocType, new DocumentId("doc:testdoc:http://www.ntnu.no/"));
+        doc.setFieldValue(testDocType.getField("byteattr"), new ByteFieldValue((byte)32));
+        doc.setFieldValue(testDocType.getField("intattr"), new IntegerFieldValue(468));
+        doc.setFieldValue(testDocType.getField("longattr"), new LongFieldValue((long)327));
+        doc.setFieldValue(testDocType.getField("doubleattr"), new DoubleFieldValue(25.0));
+    }
+
+    public void testConstant() throws Exception {
+        DocumentCalculator calculator = new DocumentCalculator("4.0");
+        assertEquals(4.0, calculator.evaluate(doc, new HashMap<String, Object>()));
+    }
+
+    public void testSimple() throws Exception {
+        DocumentCalculator calculator = new DocumentCalculator("(3 + 5) / 2");
+        assertEquals(4.0, calculator.evaluate(doc, new HashMap<String, Object>()));
+    }
+
+    public void testDivideByZero() throws Exception {
+        DocumentCalculator calculator = new DocumentCalculator("(3 + 5) / 0");
+        try {
+            System.out.println(calculator.evaluate(doc, new HashMap<String, Object>()));
+            assertTrue(false);
+        } catch (IllegalArgumentException e) {
+            // ok
+        }
+    }
+
+    public void testModByZero() throws Exception {
+        DocumentCalculator calculator = new DocumentCalculator("(3 + 5) % 0");
+        try {
+            System.out.println(calculator.evaluate(doc, new HashMap<String, Object>()));
+            assertTrue(false);
+        } catch (IllegalArgumentException e) {
+            // ok
+        }
+    }
+
+    public void testFieldDivideByZero() throws Exception {
+        try {
+            DocumentCalculator calculator = new DocumentCalculator("(testdoc.byteattr + testdoc.intattr) / testdoc.doubleattr");
+            doc.setFieldValue("doubleattr", new DoubleFieldValue(0.0));
+            calculator.evaluate(doc, new HashMap<String, Object>());
+            assertTrue(false);
+        } catch (IllegalArgumentException e) {
+            // ok
+        }
+    }
+
+    public void testVariables() throws Exception {
+        HashMap<String, Object> vars = new HashMap<String, Object>();
+        vars.put("x", new Double(3.0));
+        vars.put("y", new Double(5.0));
+        DocumentCalculator calculator = new DocumentCalculator("($x + $y) / 2");
+        assertEquals(4.0, calculator.evaluate(doc, vars));
+    }
+
+    public void testFields() throws Exception {
+        DocumentCalculator calculator = new DocumentCalculator("(testdoc.byteattr + testdoc.intattr) / testdoc.doubleattr");
+        assertEquals(20.0, calculator.evaluate(doc, new HashMap<String, Object>()));
+    }
+
+    public void testMissingField() throws Exception {
+        try {
+            DocumentCalculator calculator = new DocumentCalculator("(testdoc.nosuchattribute + testdoc.intattr) / testdoc.doubleattr");
+            calculator.evaluate(doc, new HashMap<String, Object>());
+            assertTrue(false);
+        } catch (IllegalArgumentException e) {
+            // ok
+        }
+    }
+
+    public void testFieldNotSet() throws Exception {
+        try {
+            DocumentCalculator calculator = new DocumentCalculator("(testdoc.missingattr + testdoc.intattr) / testdoc.doubleattr");
+            calculator.evaluate(doc, new HashMap<String, Object>());
+            assertTrue(false);
+        } catch (IllegalArgumentException e) {
+            // ok
+        }
+    }
+
+}
diff --git a/document/src/test/java/com/yahoo/document/DocumentIdTestCase.java b/document/src/test/java/com/yahoo/document/DocumentIdTestCase.java
new file mode 100644
index 00000000000..defb86387ab
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/DocumentIdTestCase.java
@@ -0,0 +1,294 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import com.yahoo.document.*;
+import com.yahoo.document.idstring.*;
+
+import java.math.BigInteger;
+
+import java.io.*;
+import java.util.regex.Pattern;
+import java.util.Arrays;
+
+public class DocumentIdTestCase extends junit.framework.TestCase {
+    DocumentTypeManager manager = new DocumentTypeManager();
+
+    public DocumentIdTestCase(String name) {
+        super(name);
+    }
+
+    protected void setUp() {
+        DocumentType testDocType = new DocumentType("testdoc");
+
+        testDocType.addHeaderField("intattr", DataType.INT);
+        testDocType.addField("rawattr", DataType.RAW);
+        testDocType.addField("floatattr", DataType.FLOAT);
+        testDocType.addHeaderField("stringattr", DataType.STRING);
+        testDocType.addHeaderField("Minattr", DataType.INT);
+
+        manager.registerDocumentType(testDocType);
+    }
+
+    public void testCompareTo() {
+        DocumentId docId1 = new Document(manager.getDocumentType("testdoc"), new DocumentId("doc:testdoc:http://www.uio.no/")).getId();
+        DocumentId docId2 = new Document(manager.getDocumentType("testdoc"), new DocumentId("doc:testdoc:http://www.uio.no/")).getId();
+        DocumentId docId3 = new Document(manager.getDocumentType("testdoc"), new DocumentId("doc:testdoc:http://www.ntnu.no/")).getId();
+
+        assertTrue(docId1.equals(docId2));
+        assertTrue(!docId1.equals(docId3));
+        assertTrue(docId1.compareTo(docId3) > 0);
+        assertTrue(docId3.compareTo(docId1) < 0);
+
+        assertEquals(docId1.hashCode(), docId2.hashCode());
+
+    }
+
+    private void checkInvalidUri(String uri) {
+        try {
+            //invalid URI
+            new DocumentId(uri);
+            fail();
+        } catch (IllegalArgumentException iae) {
+        }
+    }
+
+    public void testValidInvalidUriSchemes() {
+        try {
+            //valid URIs
+            new DocumentId("doc:blabla:something");
+            new DocumentId("doc:doc:doc");
+            new DocumentId("userdoc:bla:2387:");
+            new DocumentId("userdoc:bar:0:");
+            new DocumentId("userdoc:bar:18446744073709551615:");
+            new DocumentId("userdoc:foo:15:bar");
+            new DocumentId("id:namespace:type:n=42:whatever");
+            new DocumentId("id:namespace:type::whatever");
+        } catch (IllegalArgumentException iae) {
+            fail(iae.getMessage());
+        }
+
+        checkInvalidUri("foobar:");
+        checkInvalidUri("ballooo:blabla/something/");
+        checkInvalidUri("doc:");
+        checkInvalidUri("doc::");
+        checkInvalidUri("doc:::");
+        checkInvalidUri("doc::/");
+        checkInvalidUri("doc");
+        checkInvalidUri("userdoc:");
+        checkInvalidUri("userdoc::");
+        checkInvalidUri("userdoc:::");
+        checkInvalidUri("userdoc:::/");
+        checkInvalidUri("userdoc");
+        checkInvalidUri("userdoc:-87987//");
+        checkInvalidUri("userdoc:18446744073709551620/bar/");
+        checkInvalidUri("id:namespace:type");
+        checkInvalidUri("id:namespace:type:key-values");
+        checkInvalidUri("id:namespace:type:n=0,n=1:foo");
+        checkInvalidUri("id:namespace:type:g=foo,g=bar:foo");
+        checkInvalidUri("id:namespace:type:n=0,g=foo:foo");
+    }
+
+
+    //Compares globalId with C++ implementation located in
+    // ~document-HEAD/document/src/tests/cpp-globalidbucketids.txt
+    public void testCalculateGlobalId() throws IOException{
+
+        String file = "src/tests/cpp-globalidbucketids.txt";
+        BufferedReader fr = new BufferedReader(new FileReader(file));
+        String line;
+        String[] split_line;
+        String[] split_gid;
+        byte[] b;
+
+        // reads from file
+        while ((line = fr.readLine()) != null) {
+            split_line = line.split(" - ");
+            DocumentId mydoc = new DocumentId(split_line[0]);
+            b = mydoc.getGlobalId();
+            split_gid = Pattern.compile("\\(|\\)").split(split_line[1]);
+            compareStringByte(split_gid[1],b);
+        }
+        fr.close();
+    }
+
+    private void compareStringByte(String s, byte[] b){
+        /*
+        System.out.println("-- "+s+" --");
+        System.out.print("++ 0x");
+        for (int i=0; i<b.length; ++i) {
+            int nr = b[i] & 0xFF;
+            System.out.print(Integer.toHexString(nr / 16) + Integer.toHexString(nr % 16));
+        }
+        System.out.println(" ++");
+        */
+        s = s.substring(2);
+        assertEquals(s.length()/2, b.length);
+        for(int i=0; i<b.length;i++){
+            String ss = s.substring(2*i,2*i+2);
+            assertEquals(Integer.valueOf(ss, 16).intValue(),(((int)b[i])+256)%256);
+        }	
+    }
+
+    //Compares bucketId with C++ implementation located in
+    // ~document-HEAD/document/src/tests/cpp-globalidbucketids.txt
+    public void testGetBucketId() throws IOException{
+        String file = "src/tests/cpp-globalidbucketids.txt";
+        BufferedReader fr = new BufferedReader(new FileReader(file));
+        String line;
+        String[] split_line;
+        BucketId bid;
+
+        // reads from file
+        while ((line = fr.readLine()) != null) {
+            split_line = line.split(" - ");
+            DocumentId mydoc = new DocumentId(split_line[0]);
+            BucketIdFactory factory = new BucketIdFactory(32, 26, 6);
+            bid = new BucketId(factory.getBucketId(mydoc).getId());
+            assertEquals(split_line[2], bid.toString());
+        }
+        fr.close();
+    }
+
+    public void testGroupdoc() {
+        try {
+            //valid
+            new DocumentId("groupdoc:blabla:something:jkl");
+            new DocumentId("groupdoc:doc:doc:asd");
+            new DocumentId("groupdoc:bar:0:a");
+            new DocumentId("groupdoc:bar:18446744073709551615:");
+            new DocumentId("groupdoc:foo:15:bar");
+        } catch (IllegalArgumentException iae) {
+            fail(iae.getMessage());
+        }
+    }
+
+    public void testInvalidGroupdoc() {
+        checkInvalidUri("grouppdoc:blabla:something");
+        checkInvalidUri("groupdoc:blablasomething");
+    }
+
+    public void testUriNamespace() {
+        DocumentId docId = new DocumentId("doc:bar:foo");
+        assertEquals("doc:bar:foo", docId.toString());
+        assertEquals("doc", docId.getScheme().getType().toString());
+        assertEquals("bar", docId.getScheme().getNamespace());
+        assertEquals("foo", docId.getScheme().getNamespaceSpecific());
+
+        docId = new DocumentId("userdoc:ns:90:boo");
+        assertEquals("userdoc:ns:90:boo", docId.toString());
+        assertEquals("userdoc", docId.getScheme().getType().toString());
+        assertEquals("ns", docId.getScheme().getNamespace());
+        assertEquals("boo", docId.getScheme().getNamespaceSpecific());
+        assertEquals(90l, ((UserDocIdString) docId.getScheme()).getUserId());
+
+        docId = new DocumentId("userdoc:ns:18446744073709551615:boo");
+        assertEquals("userdoc:ns:18446744073709551615:boo", docId.toString());
+        assertEquals("userdoc", docId.getScheme().getType().toString());
+        assertEquals("ns", docId.getScheme().getNamespace());
+        assertEquals("boo", docId.getScheme().getNamespaceSpecific());
+        assertEquals(new BigInteger("18446744073709551615").longValue(), ((UserDocIdString) docId.getScheme()).getUserId());
+
+        docId = new DocumentId("userdoc:ns:9223372036854775808:boo");
+        assertEquals("userdoc:ns:9223372036854775808:boo", docId.toString());
+        assertEquals("userdoc", docId.getScheme().getType().toString());
+        assertEquals("ns", docId.getScheme().getNamespace());
+        assertEquals("boo", docId.getScheme().getNamespaceSpecific());
+        assertEquals(new BigInteger("9223372036854775808").longValue(), ((UserDocIdString) docId.getScheme()).getUserId());
+
+        BigInteger negativeUserId = new BigInteger("F00DCAFEDEADBABE", 16);
+        assertEquals(0xF00DCAFEDEADBABEl, negativeUserId.longValue());
+        docId = new DocumentId("userdoc:ns:"+negativeUserId+":bar");
+        assertEquals("userdoc:ns:17297704939806374590:bar", docId.toString());
+        assertEquals(negativeUserId.longValue(), ((UserDocIdString) docId.getScheme()).getUserId());
+
+        docId = new DocumentId("orderdoc(31,19):ns2:1234:1268182861:foo");
+        assertEquals("orderdoc(31,19):ns2:1234:1268182861:foo", docId.toString());
+        assertEquals("orderdoc", docId.getScheme().getType().toString());
+        assertEquals("ns2", docId.getScheme().getNamespace());
+        assertEquals("foo", docId.getScheme().getNamespaceSpecific());
+        assertEquals(31, ((OrderDocIdString)docId.getScheme()).getWidthBits());
+        assertEquals(19, ((OrderDocIdString)docId.getScheme()).getDivisionBits());
+        assertEquals("1234", ((OrderDocIdString)docId.getScheme()).getGroup());
+        assertEquals(1234, ((OrderDocIdString)docId.getScheme()).getUserId());
+        assertEquals(1268182861, ((OrderDocIdString)docId.getScheme()).getOrdering());
+    }
+
+    public void testIdStrings() {
+        DocumentId docId;
+        docId = new DocumentId(new DocIdString("test", "baaaa"));
+        assertEquals("doc:test:baaaa", docId.toString());
+        assertFalse(docId.hasDocType());
+
+        docId = new DocumentId(new UserDocIdString("test", 54, "something"));
+        assertEquals("userdoc:test:54:something", docId.toString());
+        assertFalse(docId.hasDocType());
+
+        docId = new DocumentId(new UserDocIdString("test", 0xFFFFFFFFFFFFFFFFl, "something"));
+        assertEquals("userdoc:test:18446744073709551615:something", docId.toString());
+
+        //sign flipped
+        docId = new DocumentId(new UserDocIdString("test", -8193, "something"));
+        assertEquals("userdoc:test:18446744073709543423:something", docId.toString());
+
+        docId = new DocumentId(new IdIdString("namespace", "type", "g=group", "foobar"));
+        assertEquals("id:namespace:type:g=group:foobar", docId.toString());
+        assertTrue(docId.hasDocType());
+        assertEquals("type", docId.getDocType());
+    }
+
+    public void testIdStringFeatures() {
+        DocumentId none = new DocumentId("id:ns:type::foo");
+        assertFalse(none.getScheme().hasGroup());
+        assertFalse(none.getScheme().hasNumber());
+
+        none = new DocumentId("doc:ns:foo");
+        assertFalse(none.getScheme().hasGroup());
+        assertFalse(none.getScheme().hasNumber());
+
+        DocumentId user = new DocumentId("id:ns:type:n=42:foo");
+        assertFalse(user.getScheme().hasGroup());
+        assertTrue(user.getScheme().hasNumber());
+        assertEquals(42, user.getScheme().getNumber());
+
+        user = new DocumentId("userdoc:ns:42:foo");
+        assertFalse(user.getScheme().hasGroup());
+        assertTrue(user.getScheme().hasNumber());
+        assertEquals(42, user.getScheme().getNumber());
+
+        DocumentId group = new DocumentId("id:ns:type:g=mygroup:foo");
+        assertTrue(group.getScheme().hasGroup());
+        assertFalse(group.getScheme().hasNumber());
+        assertEquals("mygroup", group.getScheme().getGroup());
+
+        group = new DocumentId("groupdoc:ns:mygroup:foo");
+        assertTrue(group.getScheme().hasGroup());
+        assertFalse(group.getScheme().hasNumber());
+        assertEquals("mygroup", group.getScheme().getGroup());
+
+        DocumentId order = new DocumentId("orderdoc(5,2):ns:42:007:foo");
+        assertTrue(order.getScheme().hasGroup());
+        assertTrue(order.getScheme().hasNumber());
+        assertEquals("42", order.getScheme().getGroup());
+        assertEquals(42, order.getScheme().getNumber());
+    }
+
+    public void testHashCodeOfGids() {
+        DocumentId docId0 = new DocumentId("doc:blabla:0");
+        byte[] docId0Gid = docId0.getGlobalId();
+        DocumentId docId0Copy = new DocumentId("doc:blabla:0");
+        byte[] docId0CopyGid = docId0Copy.getGlobalId();
+
+
+        //GIDs should be the same
+        for (int i = 0; i < docId0Gid.length; i++) {
+            assertEquals(docId0Gid[i], docId0CopyGid[i]);
+        }
+
+        //straight hashCode() of byte arrays won't be the same
+        assertFalse(docId0Gid.hashCode() == docId0CopyGid.hashCode());
+
+        //Arrays.hashCode() works better...
+        assertEquals(Arrays.hashCode(docId0Gid), Arrays.hashCode(docId0CopyGid));
+    }
+
+}
diff --git a/document/src/test/java/com/yahoo/document/DocumentPathUpdateTestCase.java b/document/src/test/java/com/yahoo/document/DocumentPathUpdateTestCase.java
new file mode 100755
index 00000000000..c414bea2cfa
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/DocumentPathUpdateTestCase.java
@@ -0,0 +1,626 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import com.yahoo.document.datatypes.*;
+import com.yahoo.document.fieldpathupdate.AddFieldPathUpdate;
+import com.yahoo.document.fieldpathupdate.AssignFieldPathUpdate;
+import com.yahoo.document.fieldpathupdate.RemoveFieldPathUpdate;
+import com.yahoo.document.serialization.*;
+import com.yahoo.io.GrowableByteBuffer;
+
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Tests applying and serializing document updates.
+ *
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class DocumentPathUpdateTestCase extends junit.framework.TestCase {
+    DocumentTypeManager docMan;
+
+    DocumentType docType = null;
+    DocumentType docType2 = null;
+
+    public void setUp() {
+        docMan = new DocumentTypeManager();
+
+        docType = new DocumentType("foobar");
+        docType.addField(new Field("num", DataType.INT));
+        docType.addField(new Field("num2", DataType.DOUBLE));
+        docType.addField(new Field("strfoo", DataType.STRING));
+
+        DataType stringarray = DataType.getArray(DataType.STRING);
+        docType.addField(new Field("strarray", stringarray));
+
+        DataType stringwset = DataType.getWeightedSet(DataType.STRING);
+        docType.addField(new Field("strwset", stringwset));
+
+        StructDataType mystructType = new StructDataType("mystruct");
+        mystructType.addField(new Field("title", DataType.STRING));
+        mystructType.addField(new Field("rating", DataType.INT));
+
+        DataType structmap = new MapDataType(DataType.STRING, mystructType);
+        docType.addField(new Field("structmap", structmap));
+
+        DataType structarray = new ArrayDataType(mystructType);
+        docType.addField(new Field("structarray", structarray));
+
+        DataType strmap = new MapDataType(DataType.STRING, DataType.STRING);
+        docType.addField(new Field("strmap", strmap));
+
+        docType.addField(new Field("struct", mystructType));
+
+        docMan.register(docType);
+
+        docType2 = new DocumentType("otherdoctype");
+        docType2.addField(new Field("strinother", DataType.STRING));
+        docMan.register(docType2);
+    }
+
+    public void testRemoveField() throws Exception {
+        Document doc = new Document(docMan.getDocumentType("foobar"), new DocumentId("doc:something:foooo"));
+        assertNull(doc.getFieldValue("strfoo"));
+        doc.setFieldValue("strfoo", "cocacola");
+        assertEquals(new StringFieldValue("cocacola"), doc.getFieldValue("strfoo"));
+        DocumentUpdate docUp = new DocumentUpdate(docType, new DocumentId("doc:foo:bar"));
+        docUp.addFieldPathUpdate(new RemoveFieldPathUpdate(doc.getDataType(), "strfoo", null));
+        docUp.applyTo(doc);
+        assertNull(doc.getFieldValue("strfoo"));
+   }
+
+    public void testApplyRemoveMultiList() throws Exception {
+        Document doc = new Document(docMan.getDocumentType("foobar"), new DocumentId("doc:something:foooo"));
+        assertNull(doc.getFieldValue("strarray"));
+        Array<StringFieldValue> strArray = new Array<>(doc.getField("strarray").getDataType());
+        strArray.add(new StringFieldValue("crouching tiger, hidden value"));
+        strArray.add(new StringFieldValue("remove val 1"));
+        strArray.add(new StringFieldValue("hello hello"));
+        doc.setFieldValue("strarray", strArray);
+        assertNotNull(doc.getFieldValue("strarray"));
+        DocumentUpdate docUp = new DocumentUpdate(docType, new DocumentId("doc:foo:bar"));
+        docUp.addFieldPathUpdate(new RemoveFieldPathUpdate(doc.getDataType(), "strarray[$x]", "foobar.strarray[$x] == \"remove val 1\""));
+        docUp.applyTo(doc);
+        assertEquals(2, ((List) doc.getFieldValue("strarray")).size());
+        List docList = (List) doc.getFieldValue("strarray");
+        assertEquals(new StringFieldValue("crouching tiger, hidden value"), docList.get(0));
+        assertEquals(new StringFieldValue("hello hello"), docList.get(1));
+    }
+
+    public void testApplyRemoveEntireListField() throws Exception {
+        Document doc = new Document(docMan.getDocumentType("foobar"), new DocumentId("doc:something:foooo"));
+        assertNull(doc.getFieldValue("strarray"));
+        Array<StringFieldValue> strArray = new Array<>(doc.getField("strarray").getDataType());
+        strArray.add(new StringFieldValue("this list"));
+        strArray.add(new StringFieldValue("should be"));
+        strArray.add(new StringFieldValue("totally removed"));
+        doc.setFieldValue("strarray", strArray);
+        DocumentUpdate docUp = new DocumentUpdate(docType, new DocumentId("doc:toast:jam"));
+        docUp.addFieldPathUpdate(new RemoveFieldPathUpdate(doc.getDataType(), "strarray", null));
+        docUp.applyTo(doc);
+        assertNull(doc.getFieldValue("strarray"));
+    }
+
+    public void testApplyRemoveMultiWset() throws Exception {
+        Document doc = new Document(docMan.getDocumentType("foobar"), new DocumentId("doc:something:foooo"));
+        assertNull(doc.getFieldValue("strwset"));
+        WeightedSet<StringFieldValue> strwset = new WeightedSet<>(doc.getDataType().getField("strwset").getDataType());
+        strwset.put(new StringFieldValue("hello hello"), 10);
+        strwset.put(new StringFieldValue("remove val 1"), 20);
+        doc.setFieldValue("strwset", strwset);
+        assertNotNull(doc.getFieldValue("strwset"));
+        DocumentUpdate docUp = new DocumentUpdate(docType, new DocumentId("doc:foo:bar"));
+        docUp.addFieldPathUpdate(new RemoveFieldPathUpdate(doc.getDataType(), "strwset{remove val 1}", ""));
+        docUp.applyTo(doc);
+        assertEquals(1, ((WeightedSet) doc.getFieldValue("strwset")).size());
+        WeightedSet docWset = (WeightedSet) doc.getFieldValue("strwset");
+        assertEquals(new Integer(10), docWset.get(new StringFieldValue("hello hello")));
+    }
+
+    public void testApplyAssignSingle() throws Exception {
+        Document doc = new Document(docMan.getDocumentType("foobar"), new DocumentId("doc:something:foooo"));
+        assertNull(doc.getFieldValue("strfoo"));
+        DocumentUpdate docUp = new DocumentUpdate(docType, new DocumentId("doc:foo:bar"));
+        docUp.addFieldPathUpdate(new AssignFieldPathUpdate(doc.getDataType(), "strfoo", "", new StringFieldValue("something")));
+        docUp.applyTo(doc);
+        assertEquals(new StringFieldValue("something"), doc.getFieldValue("strfoo"));
+    }
+
+    public void testApplyAssignMath() throws Exception {
+        Document doc = new Document(docMan.getDocumentType("foobar"), new DocumentId("doc:something:foooo"));
+        doc.setFieldValue(doc.getField("num"), new IntegerFieldValue(34));
+        DocumentUpdate docUp = new DocumentUpdate(docType, new DocumentId("doc:foo:bar"));
+        docUp.addFieldPathUpdate(new AssignFieldPathUpdate(doc.getDataType(), "num", "", "($value * 2) / $value"));
+        docUp.applyTo(doc);
+        assertEquals(new IntegerFieldValue(2), doc.getFieldValue(doc.getField("num")));
+    }
+
+    public void testDivideByZero() throws Exception {
+        Document doc = new Document(docMan.getDocumentType("foobar"), new DocumentId("doc:something:foooo"));
+        doc.setFieldValue(doc.getField("num"), new IntegerFieldValue(10));
+        DocumentUpdate docUp = new DocumentUpdate(docType, new DocumentId("doc:foo:bar"));
+        docUp.addFieldPathUpdate(new AssignFieldPathUpdate(doc.getDataType(), "num", "", "100 / ($value - 10)"));
+        docUp.applyTo(doc);
+        assertEquals(new IntegerFieldValue(10), doc.getFieldValue(doc.getField("num")));
+    }
+
+    public void testAssignMathFieldNotSet() throws Exception {
+        Document doc = new Document(docMan.getDocumentType("foobar"), new DocumentId("doc:something:foooo"));
+        doc.setFieldValue(doc.getField("num"), new IntegerFieldValue(10));
+        DocumentUpdate docUp = new DocumentUpdate(docType, new DocumentId("doc:foo:bar"));
+        docUp.addFieldPathUpdate(new AssignFieldPathUpdate(doc.getDataType(), "num", "", "100 + foobar.num2"));
+        docUp.applyTo(doc);
+        assertEquals(new IntegerFieldValue(10), doc.getFieldValue(doc.getField("num")));
+    }
+
+    public void testAssignMathMissingField() throws Exception {
+        Document doc = new Document(docMan.getDocumentType("foobar"), new DocumentId("doc:something:foooo"));
+        doc.setFieldValue(doc.getField("num"), new IntegerFieldValue(10));
+        DocumentUpdate docUp = new DocumentUpdate(docType, new DocumentId("doc:foo:bar"));
+        docUp.addFieldPathUpdate(new AssignFieldPathUpdate(doc.getDataType(), "num", "", "100 + foobar.bogus"));
+        docUp.applyTo(doc);
+        assertEquals(new IntegerFieldValue(10), doc.getFieldValue(doc.getField("num")));
+    }
+
+    public void testAssignMathTargetFieldNotSet() throws Exception {
+        Document doc = new Document(docMan.getDocumentType("foobar"), new DocumentId("doc:something:foooo"));
+        DocumentUpdate docUp = new DocumentUpdate(docType, new DocumentId("doc:foo:bar"));
+        docUp.addFieldPathUpdate(new AssignFieldPathUpdate(doc.getDataType(), "num", "", "100"));
+        docUp.applyTo(doc);
+        assertEquals(new IntegerFieldValue(100), doc.getFieldValue(doc.getField("num")));
+    }
+
+    public void testAssignMathTargetFieldNotSetWithValue() throws Exception {
+        Document doc = new Document(docMan.getDocumentType("foobar"), new DocumentId("doc:something:foooo"));
+        DocumentUpdate docUp = new DocumentUpdate(docType, new DocumentId("doc:foo:bar"));
+        docUp.addFieldPathUpdate(new AssignFieldPathUpdate(doc.getDataType(), "num", "", "$value + 5"));
+        docUp.applyTo(doc);
+        assertEquals(new IntegerFieldValue(5), doc.getFieldValue(doc.getField("num")));
+    }
+
+    public void testApplyAssignMultiList() throws Exception {
+        Document doc = new Document(docMan.getDocumentType("foobar"), new DocumentId("doc:something:foooo"));
+        assertNull(doc.getFieldValue("strarray"));
+        Array<StringFieldValue> strArray = new Array<StringFieldValue>(doc.getField("strarray").getDataType());
+        strArray.add(new StringFieldValue("hello hello"));
+        strArray.add(new StringFieldValue("blah blah"));
+        doc.setFieldValue("strarray", strArray);
+        assertNotNull(doc.getFieldValue("strarray"));
+        Array<StringFieldValue> array = new Array<>(doc.getField("strarray").getDataType());
+        array.add(new StringFieldValue("assigned val 0"));
+        array.add(new StringFieldValue("assigned val 1"));
+        DocumentUpdate docUp = new DocumentUpdate(docType, new DocumentId("doc:foo:bar"));
+        docUp.addFieldPathUpdate(new AssignFieldPathUpdate(doc.getDataType(), "strarray", "", array));
+        docUp.applyTo(doc);
+        assertEquals(2, ((List) doc.getFieldValue("strarray")).size());
+        List docList = (List) doc.getFieldValue("strarray");
+        assertEquals(new StringFieldValue("assigned val 0"), docList.get(0));
+        assertEquals(new StringFieldValue("assigned val 1"), docList.get(1));
+    }
+
+    public void testApplyAssignMultiWlist() throws Exception {
+        Document doc = new Document(docMan.getDocumentType("foobar"), new DocumentId("doc:something:foooo"));
+        assertNull(doc.getFieldValue("strwset"));
+        WeightedSet<StringFieldValue> strwset = new WeightedSet<>(doc.getDataType().getField("strwset").getDataType());
+        strwset.put(new StringFieldValue("hello hello"), 164);
+        strwset.put(new StringFieldValue("blahdi blahdi"), 243);
+        doc.setFieldValue("strwset", strwset);
+        assertNotNull(doc.getFieldValue("strwset"));
+        WeightedSet<StringFieldValue> assignWset = new WeightedSet<>(docType.getField("strwset").getDataType());
+        assignWset.put(new StringFieldValue("assigned val 0"), 5);
+        assignWset.put(new StringFieldValue("assigned val 1"), 10);
+        DocumentUpdate docUp = new DocumentUpdate(docType, new DocumentId("doc:foo:bar"));
+        docUp.addFieldPathUpdate(new AssignFieldPathUpdate(doc.getDataType(), "strwset", "", assignWset));
+        docUp.applyTo(doc);
+        assertEquals(2, ((WeightedSet) doc.getFieldValue("strwset")).size());
+        WeightedSet docWset = (WeightedSet) doc.getFieldValue("strwset");
+        assertEquals(new Integer(5), docWset.get(new StringFieldValue("assigned val 0")));
+        assertEquals(new Integer(10), docWset.get(new StringFieldValue("assigned val 1")));
+    }
+
+    public void testAssignWsetRemoveIfZero() throws Exception {
+        Document doc = new Document(docMan.getDocumentType("foobar"), new DocumentId("doc:something:foooo"));
+        assertNull(doc.getFieldValue(doc.getField("strwset")));
+        WeightedSet<StringFieldValue> strwset = new WeightedSet<>(doc.getDataType().getField("strwset").getDataType());
+        strwset.put(new StringFieldValue("hello hello"), 164);
+        strwset.put(new StringFieldValue("blahdi blahdi"), 243);
+        doc.setFieldValue(doc.getField("strwset"), strwset);
+        assertNotNull(doc.getFieldValue(doc.getField("strwset")));
+        DocumentUpdate docUp = new DocumentUpdate(docType, new DocumentId("doc:foo:bar"));
+        AssignFieldPathUpdate upd = new AssignFieldPathUpdate(doc.getDataType(), "strwset{hello hello}", "", "$value - 164");
+        upd.setRemoveIfZero(true);
+        docUp.addFieldPathUpdate(upd);
+        docUp.applyTo(doc);
+        WeightedSet docWset = (WeightedSet) doc.getFieldValue(doc.getField("strwset"));
+        assertEquals(1, docWset.size());
+        assertEquals(new Integer(243), docWset.get(new StringFieldValue("blahdi blahdi")));
+    }
+
+    public void testApplyAddMultiList() throws Exception {
+        Document doc = new Document(docMan.getDocumentType("foobar"), new DocumentId("doc:something:foooo"));
+        assertNull(doc.getFieldValue("strarray"));
+
+        Array<StringFieldValue> addList = new Array<StringFieldValue>(doc.getField("strarray").getDataType());
+        addList.add(new StringFieldValue("bo"));
+        addList.add(new StringFieldValue("ba"));
+        addList.add(new StringFieldValue("by"));
+        DocumentUpdate docUp = new DocumentUpdate(docType, new DocumentId("doc:foo:bar"));
+        docUp.addFieldPathUpdate(new AddFieldPathUpdate(doc.getDataType(), "strarray", "", addList));
+        docUp.applyTo(doc);
+        List<StringFieldValue> values = new ArrayList<>();
+        values.add(new StringFieldValue("bo"));
+        values.add(new StringFieldValue("ba"));
+        values.add(new StringFieldValue("by"));
+        assertEquals(values, doc.getFieldValue("strarray"));
+    }
+
+    public void testAddAndAssignList() throws Exception {
+        Document doc = new Document(docMan.getDocumentType("foobar"), new DocumentId("doc:something:foooo"));
+        assertNull(doc.getFieldValue("strarray"));
+
+        Array strArray = new Array(doc.getField("strarray").getDataType());
+        strArray.add(new StringFieldValue("hello hello"));
+        strArray.add(new StringFieldValue("blah blah"));
+        doc.setFieldValue("strarray", strArray);
+        assertNotNull(doc.getFieldValue("strarray"));
+
+        DocumentUpdate docUp = new DocumentUpdate(docType, new DocumentId("doc:foo:bar"));
+        docUp.addFieldPathUpdate(new AssignFieldPathUpdate(doc.getDataType(), "strarray[1]", "", new StringFieldValue("assigned val 1")));
+
+        Array adds = new Array(doc.getField("strarray").getDataType());
+        adds.add(new StringFieldValue("new value"));
+
+        docUp.addFieldPathUpdate(new AddFieldPathUpdate(doc.getDataType(), "strarray", "", adds));
+
+        docUp.applyTo(doc);
+        List docList = (List) doc.getFieldValue("strarray");
+        assertEquals(3, docList.size());
+        assertEquals(new StringFieldValue("hello hello"), docList.get(0));
+        assertEquals(new StringFieldValue("assigned val 1"), docList.get(1));
+        assertEquals(new StringFieldValue("new value"), docList.get(2));
+    }
+
+    public void testAssignSimpleMapValueWithVariable()
+    {
+        Document doc = new Document(docMan.getDocumentType("foobar"), new DocumentId("doc:something:foooo"));
+        MapFieldValue mfv = new MapFieldValue((MapDataType)doc.getField("strmap").getDataType());
+
+        mfv.put(new StringFieldValue("foo"), new StringFieldValue("bar"));
+        mfv.put(new StringFieldValue("baz"), new StringFieldValue("bananas"));
+        doc.setFieldValue("strmap", mfv);
+
+        // Select on map value, not key
+        DocumentUpdate docUp = new DocumentUpdate(docType, new DocumentId("doc:hargl:bargl"));
+        docUp.addFieldPathUpdate(new AssignFieldPathUpdate(doc.getDataType(), "strmap{$x}",
+                "foobar.strmap{$x} == \"bar\"", new StringFieldValue("shinyvalue")));
+        docUp.applyTo(doc);
+
+        MapFieldValue valueNow = (MapFieldValue)doc.getFieldValue("strmap");
+        assertEquals(2, valueNow.size());
+        assertEquals(new StringFieldValue("shinyvalue"), valueNow.get(new StringFieldValue("foo")));
+        assertEquals(new StringFieldValue("bananas"), valueNow.get(new StringFieldValue("baz")));
+    }
+
+    public void testKeyParsing() {
+        assertEquals(new FieldPathEntry.KeyParseResult("", 2), FieldPathEntry.parseKey("{}"));
+        assertEquals(new FieldPathEntry.KeyParseResult("abc", 5), FieldPathEntry.parseKey("{abc}"));
+        assertEquals(new FieldPathEntry.KeyParseResult("abc", 8), FieldPathEntry.parseKey("{   abc}"));
+        // TODO: post-skipping of spaces not supported for verbatim keys in C++. support here?
+        //assertEquals(new FieldPathEntry.KeyParseResult("abc", 8), FieldPathEntry.parseKey("{abc   }"));
+        assertEquals(new FieldPathEntry.KeyParseResult("hello", 9), FieldPathEntry.parseKey("{\"hello\"}"));
+        assertEquals(new FieldPathEntry.KeyParseResult("{abc}", 9), FieldPathEntry.parseKey("{\"{abc}\"}"));
+        assertEquals(new FieldPathEntry.KeyParseResult("abc", 5), FieldPathEntry.parseKey("{abc}stuff"));
+        assertEquals(new FieldPathEntry.KeyParseResult("abc", 10), FieldPathEntry.parseKey("{   \"abc\"}"));
+        assertEquals(new FieldPathEntry.KeyParseResult("abc", 10), FieldPathEntry.parseKey("{\"abc\"   }"));
+        assertEquals(new FieldPathEntry.KeyParseResult("abc", 13), FieldPathEntry.parseKey("{   \"abc\"   }"));
+        // Test quote escaping
+        assertEquals(new FieldPathEntry.KeyParseResult("\"doom house\"", 18), FieldPathEntry.parseKey("{\"\\\"doom house\\\"\"}"));
+        assertEquals(new FieldPathEntry.KeyParseResult("\"", 6), FieldPathEntry.parseKey("{\"\\\"\"}"));
+        assertEquals(new FieldPathEntry.KeyParseResult("a\"b\"c", 11), FieldPathEntry.parseKey("{\"a\\\"b\\\"c\"}"));
+        // Test failure conditions
+        try {
+            FieldPathEntry.parseKey("");
+            fail();
+        } catch (IllegalArgumentException e) {
+            assertEquals("Key '' does not start with '{'", e.getMessage());
+        }
+
+        try {
+            FieldPathEntry.parseKey("{");
+            fail();
+        } catch (IllegalArgumentException e) {
+            assertEquals("Key '{' is incomplete. No matching '}'", e.getMessage());
+        }
+
+        try {
+            FieldPathEntry.parseKey("{aaa");
+            fail();
+        } catch (IllegalArgumentException e) {
+            assertEquals("Key '{aaa' is incomplete. No matching '}'", e.getMessage());
+        }
+
+        try {
+            FieldPathEntry.parseKey("{\"things}");
+            fail();
+        } catch (IllegalArgumentException e) {
+            assertEquals("Escaped key '{\"things}' is incomplete. No matching '\"'", e.getMessage());
+        }
+
+        try {
+            FieldPathEntry.parseKey("{\"things\\}");
+            fail();
+        } catch (IllegalArgumentException e) {
+            assertEquals("Escaped key '{\"things\\}' has bad quote character escape sequence. Expected '\"'", e.getMessage());
+        }
+    }
+
+    public void testKeyWithEscapedChars()
+    {
+        Document doc = new Document(docMan.getDocumentType("foobar"), new DocumentId("doc:something:foooo"));
+        MapFieldValue mfv = new MapFieldValue((MapDataType)doc.getField("strmap").getDataType());
+
+        mfv.put(new StringFieldValue("here is a \"fancy\" :-} map key :-{"), new StringFieldValue("bar"));
+        mfv.put(new StringFieldValue("baz"), new StringFieldValue("bananas"));
+        doc.setFieldValue("strmap", mfv);
+
+        // Select on map value, not key
+        DocumentUpdate docUp = new DocumentUpdate(docType, new DocumentId("doc:hargl:bargl"));
+        docUp.addFieldPathUpdate(new AssignFieldPathUpdate(doc.getDataType(), "strmap{\"here is a \\\"fancy\\\" :-} map key :-{\"}",
+                "", new StringFieldValue("shinyvalue")));
+        docUp.applyTo(doc);
+
+        MapFieldValue valueNow = (MapFieldValue)doc.getFieldValue("strmap");
+        assertEquals(2, valueNow.size());
+        assertEquals(new StringFieldValue("shinyvalue"), valueNow.get(new StringFieldValue("here is a \"fancy\" :-} map key :-{")));
+        assertEquals(new StringFieldValue("bananas"), valueNow.get(new StringFieldValue("baz")));
+    }
+
+    public void testAssignMap() throws Exception {
+        Document doc = new Document(docMan.getDocumentType("foobar"), new DocumentId("doc:something:foooo"));
+        MapFieldValue mfv = new MapFieldValue((MapDataType)doc.getField("structmap").getDataType());
+        Struct fv1 = new Struct(mfv.getDataType().getValueType());
+        fv1.setFieldValue("title", new StringFieldValue("thomas"));
+        fv1.setFieldValue("rating", new IntegerFieldValue(32));
+
+        mfv.put(new StringFieldValue("foo"), fv1);
+
+        Struct fv2 = new Struct(mfv.getDataType().getValueType());
+        fv2.setFieldValue("title", new StringFieldValue("cyril"));
+        fv2.setFieldValue("rating", new IntegerFieldValue(16));
+
+        mfv.put(new StringFieldValue("bar"), fv2);
+
+        Struct fv3 = new Struct(mfv.getDataType().getValueType());
+        fv3.setFieldValue("title", new StringFieldValue("ulf"));
+        fv3.setFieldValue("rating", new IntegerFieldValue(8));
+
+        mfv.put(new StringFieldValue("zoo"), fv3);
+
+        doc.setFieldValue("structmap", mfv);
+
+        Struct fv4 = new Struct(mfv.getDataType().getValueType());
+        fv4.setFieldValue("title", new StringFieldValue("tor brede"));
+        fv4.setFieldValue("rating", new IntegerFieldValue(48));
+
+        DocumentUpdate docUp = new DocumentUpdate(docType, new DocumentId("doc:foo:bar"));
+        docUp.addFieldPathUpdate(new AssignFieldPathUpdate(doc.getDataType(), "structmap{bar}", "", fv4));
+        docUp.applyTo(doc);
+
+        MapFieldValue valueNow = (MapFieldValue)doc.getFieldValue("structmap");
+        assertEquals(fv1, valueNow.get(new StringFieldValue("foo")));
+        assertEquals(fv4, valueNow.get(new StringFieldValue("bar")));
+        assertEquals(fv3, valueNow.get(new StringFieldValue("zoo")));
+    }
+
+    public void testAssignMapStruct() throws Exception {
+        Document doc = new Document(docMan.getDocumentType("foobar"), new DocumentId("doc:something:foooo"));
+        MapFieldValue mfv = new MapFieldValue((MapDataType)doc.getField("structmap").getDataType());
+        Struct fv1 = new Struct(mfv.getDataType().getValueType());
+        fv1.setFieldValue("title", new StringFieldValue("thomas"));
+        fv1.setFieldValue("rating", new IntegerFieldValue(32));
+
+        mfv.put(new StringFieldValue("foo"), fv1);
+
+        Struct fv2 = new Struct(mfv.getDataType().getValueType());
+        fv2.setFieldValue("title", new StringFieldValue("cyril"));
+        fv2.setFieldValue("rating", new IntegerFieldValue(16));
+
+        mfv.put(new StringFieldValue("bar"), fv2);
+
+        Struct fv3 = new Struct(mfv.getDataType().getValueType());
+        fv3.setFieldValue("title", new StringFieldValue("ulf"));
+        fv3.setFieldValue("rating", new IntegerFieldValue(8));
+
+        mfv.put(new StringFieldValue("zoo"), fv3);
+
+        doc.setFieldValue("structmap", mfv);
+
+        Struct fv4 = new Struct(mfv.getDataType().getValueType());
+        fv4.setFieldValue("title", new StringFieldValue("cyril"));
+        fv4.setFieldValue("rating", new IntegerFieldValue(48));
+
+        DocumentUpdate docUp = new DocumentUpdate(docType, new DocumentId("doc:foo:bar"));
+        docUp.addFieldPathUpdate(new AssignFieldPathUpdate(doc.getDataType(), "structmap{bar}.rating", "", new IntegerFieldValue(48)));
+        docUp.applyTo(doc);
+
+        MapFieldValue valueNow = (MapFieldValue)doc.getFieldValue("structmap");
+        assertEquals(fv1, valueNow.get(new StringFieldValue("foo")));
+        assertEquals(fv4, valueNow.get(new StringFieldValue("bar")));
+        assertEquals(fv3, valueNow.get(new StringFieldValue("zoo")));
+    }
+
+    public void testAssignMapStructVariable() throws Exception {
+        Document doc = new Document(docMan.getDocumentType("foobar"), new DocumentId("doc:something:foooo"));
+        MapFieldValue mfv = new MapFieldValue((MapDataType)doc.getField("structmap").getDataType());
+        Struct fv1 = new Struct(mfv.getDataType().getValueType());
+        fv1.setFieldValue(fv1.getField("title"), new StringFieldValue("thomas"));
+        fv1.setFieldValue(fv1.getField("rating"), new IntegerFieldValue(32));
+
+        mfv.put(new StringFieldValue("foo"), fv1);
+
+        Struct fv2 = new Struct(mfv.getDataType().getValueType());
+        fv2.setFieldValue(fv2.getField("title"), new StringFieldValue("cyril"));
+        fv2.setFieldValue(fv2.getField("rating"), new IntegerFieldValue(16));
+
+        mfv.put(new StringFieldValue("bar"), fv2);
+
+        Struct fv3 = new Struct(mfv.getDataType().getValueType());
+        fv3.setFieldValue(fv3.getField("title"), new StringFieldValue("ulf"));
+        fv3.setFieldValue(fv3.getField("rating"), new IntegerFieldValue(8));
+
+        mfv.put(new StringFieldValue("zoo"), fv3);
+
+        doc.setFieldValue(doc.getField("structmap"), mfv);
+
+        Struct fv4 = new Struct(mfv.getDataType().getValueType());
+        fv4.setFieldValue(fv4.getField("title"), new StringFieldValue("cyril"));
+        fv4.setFieldValue(fv4.getField("rating"), new IntegerFieldValue(48));
+
+        DocumentUpdate docUp = new DocumentUpdate(docType, new DocumentId("doc:foo:bar"));
+        docUp.addFieldPathUpdate(new AssignFieldPathUpdate(doc.getDataType(), "structmap{$x}.rating", "foobar.structmap{$x}.title == \"cyril\"", new IntegerFieldValue(48)));
+        docUp.applyTo(doc);
+
+        MapFieldValue valueNow = (MapFieldValue)doc.getFieldValue("structmap");
+        assertEquals(fv1, valueNow.get(new StringFieldValue("foo")));
+        assertEquals(fv4, valueNow.get(new StringFieldValue("bar")));
+        assertEquals(fv3, valueNow.get(new StringFieldValue("zoo")));
+    }
+
+    public void testAssignMapNoexist() throws Exception {
+        Document doc = new Document(docMan.getDocumentType("foobar"), new DocumentId("doc:something:foooo"));
+        MapFieldValue mfv = new MapFieldValue((MapDataType)doc.getField("structmap").getDataType());
+
+        Struct fv1 = new Struct(mfv.getDataType().getValueType());
+        fv1.setFieldValue("title", new StringFieldValue("thomas"));
+        fv1.setFieldValue("rating", new IntegerFieldValue(32));
+
+        DocumentUpdate docUp = new DocumentUpdate(docType, new DocumentId("doc:foo:bar"));
+        docUp.addFieldPathUpdate(new AssignFieldPathUpdate(doc.getDataType(), "structmap{foo}", "", fv1));
+        docUp.applyTo(doc);
+
+        MapFieldValue valueNow = (MapFieldValue)doc.getFieldValue("structmap");
+        assertEquals(fv1, valueNow.get(new StringFieldValue("foo")));
+    }
+
+    public void testAssignMapNoexistNocreate() throws Exception {
+        Document doc = new Document(docMan.getDocumentType("foobar"), new DocumentId("doc:something:foooo"));
+        MapFieldValue mfv = new MapFieldValue((MapDataType)doc.getField("structmap").getDataType());
+
+        Struct fv1 = new Struct(mfv.getDataType().getValueType());
+        fv1.setFieldValue("title", new StringFieldValue("thomas"));
+        fv1.setFieldValue("rating", new IntegerFieldValue(32));
+
+        DocumentUpdate docUp = new DocumentUpdate(docType, new DocumentId("doc:foo:bar"));
+        AssignFieldPathUpdate ass = new AssignFieldPathUpdate(doc.getDataType(), "structmap{foo}", "", fv1);
+        ass.setCreateMissingPath(false);
+        docUp.addFieldPathUpdate(ass);
+        docUp.applyTo(doc);
+
+        MapFieldValue valueNow = (MapFieldValue)doc.getFieldValue("structmap");
+        assertNull(valueNow);
+    }
+
+    public void testAssignSerialization() throws Exception {
+        DocumentUpdate docUp = new DocumentUpdate(docType, new DocumentId("doc:foo:bar"));
+        AssignFieldPathUpdate ass = new AssignFieldPathUpdate(docType, "num", "", "3");
+        ass.setCreateMissingPath(false);
+        docUp.addFieldPathUpdate(ass);
+
+        GrowableByteBuffer buffer = new GrowableByteBuffer();
+        docUp.serialize(DocumentSerializerFactory.createHead(buffer));
+        buffer.flip();
+        DocumentUpdate docUp2 = new DocumentUpdate(DocumentDeserializerFactory.createHead(docMan, buffer));
+
+        assertEquals(docUp, docUp2);
+    }
+
+    public void testAddSerialization() throws Exception {
+        DocumentUpdate docUp = new DocumentUpdate(docType, new DocumentId("doc:foo:bar"));
+        Array strArray = new Array(docType.getField("strarray").getDataType());
+        strArray.add(new StringFieldValue("hello hello"));
+        strArray.add(new StringFieldValue("blah blah"));
+
+        AddFieldPathUpdate add = new AddFieldPathUpdate(docType, "strarray", "", strArray);
+        docUp.addFieldPathUpdate(add);
+
+        GrowableByteBuffer buffer = new GrowableByteBuffer();
+        docUp.serialize(DocumentSerializerFactory.createHead(buffer));
+        buffer.flip();
+        DocumentUpdate docUp2 = new DocumentUpdate(DocumentDeserializerFactory.createHead(docMan, buffer));
+
+        assertEquals(docUp, docUp2);
+    }
+
+    public void testRemoveSerialization() throws Exception {
+        DocumentUpdate docUp = new DocumentUpdate(docType, new DocumentId("doc:foo:bar"));
+        RemoveFieldPathUpdate remove = new RemoveFieldPathUpdate(docType, "num", "foobar.num > 0");
+        docUp.addFieldPathUpdate(remove);
+
+        GrowableByteBuffer buffer = new GrowableByteBuffer();
+        docUp.serialize(DocumentSerializerFactory.createHead(buffer));
+        buffer.flip();
+        DocumentUpdate docUp2 = new DocumentUpdate(DocumentDeserializerFactory.createHead(docMan, buffer));
+
+        assertEquals(docUp, docUp2);
+    }
+
+    public void testStartsWith() throws Exception {
+        FieldPath fp1 = docType.buildFieldPath("struct");
+        FieldPath fp2 = docType.buildFieldPath("struct.title");
+        assertTrue(fp2.startsWith(fp1));
+        assertTrue(fp2.startsWith(fp2));
+        assertFalse(fp1.startsWith(fp2));
+    }
+
+    private DocumentUpdate createDocumentUpdateForSerialization() {
+        docMan = DocumentTestCase.setUpCppDocType();
+        docType = docMan.getDocumentType("serializetest");
+
+        DocumentUpdate docUp = new DocumentUpdate(docType, new DocumentId("doc:serialization:xlanguage"));
+
+        AssignFieldPathUpdate ass = new AssignFieldPathUpdate(docType, "intfield", "", "3");
+        ass.setCreateMissingPath(false);
+        ass.setRemoveIfZero(true);
+        docUp.addFieldPathUpdate(ass);
+
+        Array fArray = new Array(docType.getField("arrayoffloatfield").getDataType());
+        fArray.add(new FloatFieldValue(12.0f));
+        fArray.add(new FloatFieldValue(5.0f));
+
+        AddFieldPathUpdate add = new AddFieldPathUpdate(docType, "arrayoffloatfield", "", fArray);
+        docUp.addFieldPathUpdate(add);
+
+        RemoveFieldPathUpdate remove = new RemoveFieldPathUpdate(docType, "intfield", "serializetest.intfield > 0");
+        docUp.addFieldPathUpdate(remove);
+
+        return docUp;
+    }
+
+    public void testGenerateSerializedFile() throws IOException {
+        DocumentUpdate docUp = createDocumentUpdateForSerialization();
+
+        GrowableByteBuffer buffer = new GrowableByteBuffer();
+        docUp.serialize(DocumentSerializerFactory.createHead(buffer));
+
+        int size = buffer.position();
+        buffer.position(0);
+
+        FileOutputStream fos = new FileOutputStream("src/tests/data/serialize-fieldpathupdate-java.dat");
+        fos.write(buffer.array(), 0, size);
+        fos.close();
+    }
+
+    public void testReadSerializedFile() throws IOException {
+        docMan = DocumentTestCase.setUpCppDocType();
+        byte[] data = DocumentTestCase.readFile("src/tests/data/serialize-fieldpathupdate-cpp.dat");
+        DocumentDeserializer buf = DocumentDeserializerFactory.createHead(docMan, GrowableByteBuffer.wrap(data));
+
+        DocumentUpdate upd = new DocumentUpdate(buf);
+
+        DocumentUpdate compare = createDocumentUpdateForSerialization();
+
+        assertEquals(compare, upd);
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/DocumentRemoveTestCase.java b/document/src/test/java/com/yahoo/document/DocumentRemoveTestCase.java
new file mode 100644
index 00000000000..62d65400d7b
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/DocumentRemoveTestCase.java
@@ -0,0 +1,44 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import org.junit.Test;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.CoreMatchers.not;
+import static org.junit.Assert.assertThat;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ * @since 5.1.10
+ */
+public class DocumentRemoveTestCase {
+
+    @Test
+    public void requireThatToStringWorks() {
+        DocumentId docId = new DocumentId("doc:this:is:a:test");
+        DocumentRemove r = new DocumentRemove(docId);
+        assertThat(r.toString().contains(docId.toString()), is(true));
+    }
+
+    @Test
+    public void requireThatEqualsAndHashCodeWorks() {
+        DocumentRemove r1 = new DocumentRemove(new DocumentId("doc:this:is:a:test"));
+        DocumentRemove r2 = new DocumentRemove(new DocumentId("doc:this:is:a:test"));
+        DocumentRemove r3 = new DocumentRemove(new DocumentId("doc:this:is:nonequal"));
+
+        assertThat(r1, equalTo(r1));
+        assertThat(r1, equalTo(r2));
+        assertThat(r2, equalTo(r1));
+        assertThat(r1.hashCode(), equalTo(r2.hashCode()));
+
+        assertThat(r1, not(equalTo(r3)));
+        assertThat(r3, not(equalTo(r1)));
+        assertThat(r2, not(equalTo(r3)));
+        assertThat(r3, not(equalTo(r2)));
+        assertThat(r1.hashCode(), not(equalTo(r3.hashCode())));
+
+        assertThat(r1, not(equalTo(new Object())));
+        assertThat(r1.equals("banana"), is(false));
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/DocumentSerializationTestCase.java b/document/src/test/java/com/yahoo/document/DocumentSerializationTestCase.java
new file mode 100644
index 00000000000..d57d29db81b
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/DocumentSerializationTestCase.java
@@ -0,0 +1,227 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import com.yahoo.compress.CompressionType;
+import com.yahoo.document.annotation.AbstractTypesTest;
+import com.yahoo.document.datatypes.Array;
+import com.yahoo.document.datatypes.ByteFieldValue;
+import com.yahoo.document.datatypes.DoubleFieldValue;
+import com.yahoo.document.datatypes.FloatFieldValue;
+import com.yahoo.document.datatypes.IntegerFieldValue;
+import com.yahoo.document.datatypes.LongFieldValue;
+import com.yahoo.document.datatypes.Raw;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.document.datatypes.WeightedSet;
+import com.yahoo.document.serialization.*;
+import com.yahoo.io.GrowableByteBuffer;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Tests serialization of all versions.
+ * <p/>
+ * This test tests serialization and deserialization of documents of all
+ * supported types.
+ * <p/>
+ * Serialization is only supported in newest format. Deserialization should work
+ * for all formats supported, but only the part that makes sense in the new
+ * format. Thus, if new format deprecates a datatype, that datatype, when
+ * serializing old versions, must either just be dropped or converted.
+ * <p/>
+ * Thus, we create document type programmatically, because all old versions need
+ * to make sense with current config.
+ * <p/>
+ * When we create a document programmatically. This is serialized into current
+ * version files. When altering the format, after the alteration, copy the
+ * current version files to a specific version file and add those to list of
+ * files this test checks.
+ * <p/>
+ * When adding new fields to the documents, use the version tagged with each
+ * file to ignore these field for old types.
+ *
+ * @author arnej27959
+ */
+public class DocumentSerializationTestCase extends AbstractTypesTest {
+
+    @Test
+    public void testSerializationAllVersions() throws IOException {
+
+        DocumentType docInDocType = new DocumentType("docindoc");
+        docInDocType.addField(new Field("stringindocfield", DataType.STRING, false));
+
+        DocumentType docType = new DocumentType("serializetest");
+        docType.addField(new Field("floatfield", DataType.FLOAT, true));
+        docType.addField(new Field("stringfield", DataType.STRING, true));
+        docType.addField(new Field("longfield", DataType.LONG, true));
+        docType.addField(new Field("urifield", DataType.URI, true));
+        docType.addField(new Field("intfield", DataType.INT, false));
+        docType.addField(new Field("rawfield", DataType.RAW, false));
+        docType.addField(new Field("doublefield", DataType.DOUBLE, false));
+        docType.addField(new Field("bytefield", DataType.BYTE, false));
+        DataType arrayOfFloatDataType = new ArrayDataType(DataType.FLOAT);
+        docType.addField(new Field("arrayoffloatfield", arrayOfFloatDataType, false));
+        DataType arrayOfArrayOfFloatDataType = new ArrayDataType(arrayOfFloatDataType);
+        docType.addField(new Field("arrayofarrayoffloatfield", arrayOfArrayOfFloatDataType, false));
+        docType.addField(new Field("docfield", DataType.DOCUMENT, false));
+        DataType weightedSetDataType = DataType.getWeightedSet(DataType.STRING, false, false);
+        docType.addField(new Field("wsfield", weightedSetDataType, false));
+
+        DocumentTypeManager docMan = new DocumentTypeManager();
+        docMan.register(docInDocType);
+        docMan.register(docType);
+
+        String path = "src/test/serializeddocuments/";
+
+        {
+            Document doc = new Document(docType, "doc:serializetest:http://test.doc.id/");
+            doc.setFieldValue("intfield", 5);
+            doc.setFieldValue("floatfield", -9.23);
+            doc.setFieldValue("stringfield", "This is a string.");
+            doc.setFieldValue("longfield", new LongFieldValue(398420092938472983l));
+            doc.setFieldValue("doublefield", new DoubleFieldValue(98374532.398820));
+            doc.setFieldValue("bytefield", new ByteFieldValue(254));
+            byte[] rawData = "RAW DATA".getBytes();
+            assertEquals(8, rawData.length);
+            doc.setFieldValue(docType.getField("rawfield"),new Raw(ByteBuffer.wrap("RAW DATA".getBytes())));
+            Document docInDoc = new Document(docInDocType, "doc:serializetest:http://doc.in.doc/");
+            docInDoc.setFieldValue("stringindocfield", "Elvis is dead");
+            doc.setFieldValue(docType.getField("docfield"), docInDoc);
+            Array<FloatFieldValue> floatArray = new Array<>(arrayOfFloatDataType);
+            floatArray.add(new FloatFieldValue(1.0f));
+            floatArray.add(new FloatFieldValue(2.0f));
+            doc.setFieldValue("arrayoffloatfield", floatArray);
+            WeightedSet<StringFieldValue> weightedSet = new WeightedSet<>(weightedSetDataType);
+            weightedSet.put(new StringFieldValue("Weighted 0"), 50);
+            weightedSet.put(new StringFieldValue("Weighted 1"), 199);
+            doc.setFieldValue("wsfield", weightedSet);
+
+            CompressionConfig noncomp = new CompressionConfig();
+            CompressionConfig lz4comp = new CompressionConfig(CompressionType.LZ4);
+            {
+                doc.getDataType().getHeaderType().setCompressionConfig(noncomp);
+                doc.getDataType().getBodyType().setCompressionConfig(noncomp);
+                FileOutputStream fout = new FileOutputStream(path + "document-java-currentversion-uncompressed.dat", false);
+                doc.serialize(fout);
+                fout.close();
+            }
+            {
+                doc.getDataType().getHeaderType().setCompressionConfig(lz4comp);
+                doc.getDataType().getBodyType().setCompressionConfig(lz4comp);
+                FileOutputStream fout = new FileOutputStream(path + "document-java-currentversion-lz4-9.dat", false);
+                doc.serialize(fout);
+                doc.getDataType().getHeaderType().setCompressionConfig(noncomp);
+                doc.getDataType().getBodyType().setCompressionConfig(noncomp);
+                fout.close();
+            }
+        }
+
+        class TestDoc {
+
+            String testFile;
+            int version;
+
+            TestDoc(String testFile, int version) {
+                this.testFile = testFile;
+                this.version = version;
+            }
+        }
+
+        String cpppath = "src/tests/data/";
+
+        List<TestDoc> tests = new ArrayList<>();
+        tests.add(new TestDoc(path + "document-java-currentversion-uncompressed.dat",
+                              Document.SERIALIZED_VERSION));
+        tests.add(new TestDoc(path + "document-java-currentversion-lz4-9.dat",
+                              Document.SERIALIZED_VERSION));
+        tests.add(new TestDoc(path + "document-java-v8-uncompressed.dat", 8));
+        tests.add(new TestDoc(cpppath + "document-cpp-currentversion-uncompressed.dat", 7));
+        tests.add(new TestDoc(cpppath + "document-cpp-currentversion-lz4-9.dat", 7));
+        tests.add(new TestDoc(cpppath + "document-cpp-v8-uncompressed.dat", 7));
+        tests.add(new TestDoc(cpppath + "document-cpp-v7-uncompressed.dat", 7));
+        tests.add(new TestDoc(cpppath + "serializev6.dat", 6));
+        for (TestDoc test : tests) {
+            File f = new File(test.testFile);
+            FileInputStream fin = new FileInputStream(f);
+            byte[] buffer = new byte[(int)f.length()];
+            int pos = 0;
+            int remaining = buffer.length;
+            while (remaining > 0) {
+                int read = fin.read(buffer, pos, remaining);
+                assertFalse(read == -1);
+                pos += read;
+                remaining -= read;
+            }
+            System.err.println("Checking doc from file " + test.testFile);
+
+            Document doc = new Document(DocumentDeserializerFactory.create42(docMan, GrowableByteBuffer.wrap(buffer)));
+
+            System.err.println("Id: " + doc.getId());
+
+            assertEquals(new IntegerFieldValue(5), doc.getFieldValue("intfield"));
+            assertEquals(-9.23, ((FloatFieldValue)doc.getFieldValue("floatfield")).getFloat(), 1E-6);
+            assertEquals(new StringFieldValue("This is a string."), doc.getFieldValue("stringfield"));
+            assertEquals(new LongFieldValue(398420092938472983l), doc.getFieldValue("longfield"));
+            assertEquals(98374532.398820, ((DoubleFieldValue)doc.getFieldValue("doublefield")).getDouble(), 1E-6);
+            assertEquals(new ByteFieldValue((byte)254),
+                         doc.getFieldValue("bytefield"));
+            ByteBuffer bbuffer = ((Raw)doc.getFieldValue("rawfield")).getByteBuffer();
+            if (!Arrays.equals("RAW DATA".getBytes(), bbuffer.array())) {
+                System.err.println("Expected 'RAW DATA' but got '"
+                                   + new String(bbuffer.array()) + "'.");
+                assertTrue(false);
+            }
+            if (test.version > 6) {
+                Document docInDoc = (Document)doc.getFieldValue("docfield");
+                assertTrue(docInDoc != null);
+                assertEquals(new StringFieldValue("Elvis is dead"),
+                             docInDoc.getFieldValue("stringindocfield"));
+            }
+            Array array = (Array)doc.getFieldValue("arrayoffloatfield");
+            assertTrue(array != null);
+            assertEquals(1.0f, ((FloatFieldValue)array.get(0)).getFloat(), 1E-6);
+            assertEquals(2.0f, ((FloatFieldValue)array.get(1)).getFloat(), 1E-6);
+            WeightedSet wset = (WeightedSet)doc.getFieldValue("wsfield");
+            assertTrue(wset != null);
+            assertEquals(Integer.valueOf(50), wset.get(new StringFieldValue("Weighted 0")));
+            assertEquals(Integer.valueOf(199), wset.get(new StringFieldValue("Weighted 1")));
+        }
+    }
+
+    @Test
+    public void testSerializeDeserializeWithAnnotations() throws IOException {
+        Document doc = new Document(docType, "doc:foo:bar");
+
+        doc.setFieldValue("age", (byte)123);
+        doc.setFieldValue("story", getAnnotatedString());
+        doc.setFieldValue("date", 13829297);
+        doc.setFieldValue("friend", 2384L);
+
+        GrowableByteBuffer buffer = new GrowableByteBuffer(1024);
+        DocumentSerializer serializer = DocumentSerializerFactory.create42(buffer);
+        serializer.write(doc);
+        buffer.flip();
+
+        FileOutputStream fos = new FileOutputStream("src/tests/data/serializejavawithannotations.dat");
+        fos.write(buffer.array(), 0, buffer.limit());
+        fos.close();
+
+        DocumentDeserializer deserializer = DocumentDeserializerFactory.create42(man, buffer);
+        Document doc2 = new Document(deserializer);
+
+        assertEquals(doc, doc2);
+        assertNotSame(doc, doc2);
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/DocumentTestCase.java b/document/src/test/java/com/yahoo/document/DocumentTestCase.java
new file mode 100644
index 00000000000..9a9e7042745
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/DocumentTestCase.java
@@ -0,0 +1,1407 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import com.yahoo.compress.CompressionType;
+import com.yahoo.document.datatypes.Array;
+import com.yahoo.document.datatypes.ByteFieldValue;
+import com.yahoo.document.datatypes.DoubleFieldValue;
+import com.yahoo.document.datatypes.FieldPathIteratorHandler;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.FloatFieldValue;
+import com.yahoo.document.datatypes.IntegerFieldValue;
+import com.yahoo.document.datatypes.LongFieldValue;
+import com.yahoo.document.datatypes.MapFieldValue;
+import com.yahoo.document.datatypes.Raw;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.document.datatypes.Struct;
+import com.yahoo.document.datatypes.WeightedSet;
+import com.yahoo.document.serialization.*;
+import com.yahoo.io.GrowableByteBuffer;
+import com.yahoo.vespa.objects.BufferSerializer;
+import org.junit.Test;
+
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/**
+ * Test for Document and all its features, including (de)serialization.
+ *
+ * @author <a href="thomasg@yahoo-inc.com>Thomas Gundersen</a>
+ * @author bratseth
+ */
+@SuppressWarnings("deprecation")
+public class DocumentTestCase extends DocumentTestCaseBase {
+
+    private static final String SERTEST_DOC_AS_XML_HEAD =
+            "<document documenttype=\"sertest\" documentid=\"doc:sertest:foobar\">\n" +
+            " <mailid>emailfromalicetobob&amp;someone</mailid>\n" +
+            " <date>-2013512400</date>\n" +
+            " <attachmentcount>2</attachmentcount>\n" +
+            " <rawfield binaryencoding=\"base64\">AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gISIjJCUmJygpKissLS4vMDEyMzQ1Njc4OTo7PD0+P0BBQkNERUZHSElKS0xNTk9QUVJTVFVWV1hZWltcXV5fYGFiYw==</rawfield>\n";
+
+    private static final String SERTEST_DOC_AS_XML_WEIGHT1 =
+            " <weightedfield>\n" +
+            "  <item weight=\"10\">this is another test, blah blah</item>\n" +
+            "  <item weight=\"5\">this is a test</item>\n" +
+            " </weightedfield>\n";
+
+    private static final String SERTEST_DOC_AS_XML_WEIGHT2 =
+            " <weightedfield>\n" +
+            "  <item weight=\"5\">this is a test</item>\n" +
+            "  <item weight=\"10\">this is another test, blah blah</item>\n" +
+            " </weightedfield>\n";
+
+    private static final String SERTEST_DOC_AS_XML_SUNNYVALE =
+            " <myposfield>N37.374821;W122.057174</myposfield>\n";
+
+    private static final String SERTEST_DOC_AS_XML_FOOT =
+            " <docindoc documenttype=\"docindoc\" documentid=\"doc:sertest:inserted\">\n" +
+            "  <tull>ball</tull>\n" +
+            " </docindoc>\n" +
+            " <mapfield>\n" +
+            "  <item>\n" +
+            "   <key>foo2</key>\n" +
+            "   <value>bar2</value>\n" +
+            "  </item>\n" +
+            "  <item>\n" +
+            "   <key>foo1</key>\n" +
+            "   <value>bar1</value>\n" +
+            "  </item>\n" +
+            " </mapfield>\n" +
+            SERTEST_DOC_AS_XML_SUNNYVALE +
+            "</document>\n";
+
+    static DocumentTypeManager setUpCppDocType() {
+        return setUpDocType("file:src/tests/data/crossplatform-java-cpp-document.cfg");
+    }
+
+    static DocumentTypeManager setUpDocType(String filename) {
+        DocumentTypeManager dcMan = new DocumentTypeManager();
+        DocumentTypeManagerConfigurer.configure(dcMan, filename);
+        return dcMan;
+    }
+
+    public void setUpSertestDocType() {
+        docMan = new DocumentTypeManager();
+
+        DocumentType docInDocType = new DocumentType("docindoc");
+        docInDocType.addField(new Field("tull", 2, docMan.getDataType(2), true));
+
+        docMan.registerDocumentType(docInDocType);
+
+        DocumentType sertestDocType = new DocumentType("sertest");
+        sertestDocType.addField(new Field("mailid", 2, docMan.getDataType(2), true));
+        sertestDocType.addField(new Field("date", 3, docMan.getDataType(0), true));
+        sertestDocType.addField(new Field("from", 4, docMan.getDataType(2), true));
+        sertestDocType.addField(new Field("to", 6, docMan.getDataType(2), true));
+        sertestDocType.addField(new Field("subject", 9, docMan.getDataType(2), true));
+        sertestDocType.addField(new Field("body", 10, docMan.getDataType(2), false));
+        sertestDocType.addField(new Field("attachmentcount", 11, docMan.getDataType(0), false));
+        sertestDocType.addField(new Field("attachments", 1081629685, DataType.getArray(docMan.getDataType(2)), false));
+        sertestDocType.addField(new Field("rawfield", 879, DataType.RAW, false));
+        sertestDocType.addField(new Field("weightedfield", 880, DataType.getWeightedSet(DataType.STRING), false));
+        sertestDocType.addField(new Field("weightedfieldCreate", 881, DataType.getWeightedSet(DataType.STRING, true, false), false));
+        sertestDocType.addField(new Field("docindoc", 882, docInDocType, false));
+        sertestDocType.addField(new Field("mapfield", 883, new MapDataType(DataType.STRING, DataType.STRING), false));
+        sertestDocType.addField(new Field("myposfield", 884, PositionDataType.INSTANCE, false));
+
+        docMan.registerDocumentType(sertestDocType);
+    }
+
+    static byte[] readFile(String filename) throws IOException {
+        FileInputStream fis = new FileInputStream(filename);
+        byte[] data = new byte[1000];
+        int tot = fis.read(data);
+        if (tot == -1) {
+            throw new IOException("Could not read from file " + filename);
+        }
+
+        return data;
+    }
+
+    private Document getSertestDocument() {
+        Document doc = new Document(docMan.getDocumentType("sertest"), new DocumentId("doc:sertest:foobar"));
+        doc.setFieldValue("mailid", "emailfromalicetobob");
+        doc.setFieldValue("date", -2013512400); // 03/13/06 11:00:00
+        doc.setFieldValue("attachmentcount", 2);
+
+        byte[] rawBytes = new byte[100];
+        for (int i = 0; i < rawBytes.length; i++) {
+            rawBytes[i] = (byte)i;
+        }
+
+        doc.setFieldValue("rawfield", new Raw(ByteBuffer.wrap(rawBytes)));
+
+        Document docInDoc = new Document(docMan.getDocumentType("docindoc"), new DocumentId("doc:sertest:inserted"));
+        docInDoc.setFieldValue("tull", "ball");
+        doc.setFieldValue("docindoc", docInDoc);
+
+        WeightedSet<StringFieldValue> wset = new WeightedSet<>(DataType.getWeightedSet(DataType.STRING));
+        wset.put(new StringFieldValue("this is a test"), 5);
+        wset.put(new StringFieldValue("this is another test, blah blah"), 10);
+        doc.setFieldValue("weightedfield", wset);
+
+        MapFieldValue<StringFieldValue, StringFieldValue> map = new MapFieldValue<>(new MapDataType(DataType.STRING, DataType.STRING));
+        map.put(new StringFieldValue("foo1"), new StringFieldValue("bar1"));
+        map.put(new StringFieldValue("foo2"), new StringFieldValue("bar2"));
+        doc.setFieldValue("mapfield", map);
+
+        return doc;
+    }
+
+    @Test
+    public void testTypeChecking() {
+        DocumentType type = new DocumentType("test");
+        type.addField(new Field("double", DataType.DOUBLE));
+        type.addField(new Field("float", DataType.FLOAT));
+        type.addField(new Field("int", DataType.INT));
+        type.addField(new Field("long", DataType.LONG));
+        type.addField(new Field("string", DataType.STRING));
+
+        Document doc = new Document(type, "doc:scheme:");
+        FieldValue stringVal = new StringFieldValue("69");
+        FieldValue doubleVal = new DoubleFieldValue(6.9);
+        FieldValue floatVal = new FloatFieldValue(6.9f);
+        FieldValue intVal = new IntegerFieldValue(69);
+        FieldValue longVal = new LongFieldValue(69L);
+
+        doc.setFieldValue("string", stringVal);
+        doc.setFieldValue("string", doubleVal);
+        doc.setFieldValue("string", floatVal);
+        doc.setFieldValue("string", intVal);
+        doc.setFieldValue("string", longVal);
+
+        doc.setFieldValue("double", stringVal);
+        doc.setFieldValue("double", doubleVal);
+        doc.setFieldValue("double", floatVal);
+        doc.setFieldValue("double", intVal);
+        doc.setFieldValue("double", longVal);
+
+        doc.setFieldValue("float", stringVal);
+        doc.setFieldValue("float", doubleVal);
+        doc.setFieldValue("float", floatVal);
+        doc.setFieldValue("float", intVal);
+        doc.setFieldValue("float", longVal);
+
+        doc.setFieldValue("int", stringVal);
+        doc.setFieldValue("int", doubleVal);
+        doc.setFieldValue("int", floatVal);
+        doc.setFieldValue("int", intVal);
+        doc.setFieldValue("int", longVal);
+
+        doc.setFieldValue("long", stringVal);
+        doc.setFieldValue("long", doubleVal);
+        doc.setFieldValue("long", floatVal);
+        doc.setFieldValue("long", intVal);
+        doc.setFieldValue("long", longVal);
+    }
+
+    class VariableIteratorHandler extends FieldPathIteratorHandler {
+
+        public String retVal = "";
+
+        @Override
+        public void onPrimitive(FieldValue fv) {
+
+            for (Map.Entry<String, IndexValue> entry : getVariables().entrySet()) {
+                retVal += entry.getKey() + ": " + entry.getValue() + ",";
+            }
+            retVal += " - " + fv + "\n";
+        }
+    }
+
+    @Test
+    public void testVariables() {
+        ArrayDataType iarr = new ArrayDataType(DataType.INT);
+        ArrayDataType iiarr = new ArrayDataType(iarr);
+        ArrayDataType iiiarr = new ArrayDataType(iiarr);
+
+        DocumentType type = new DocumentType("test");
+        type.addField(new Field("iiiarray", iiiarr));
+
+        Array<Array<Array<IntegerFieldValue>>> iiiaV = new Array<>(iiiarr);
+        for (int i = 1; i < 4; i++) {
+            Array<Array<IntegerFieldValue>> iiaV = new Array<>(iiarr);
+            for (int j = 1; j < 4; j++) {
+                Array<IntegerFieldValue> iaV = new Array<>(iarr);
+                for (int k = 1; k < 4; k++) {
+                    iaV.add(new IntegerFieldValue(i * j * k));
+                }
+                iiaV.add(iaV);
+            }
+            iiiaV.add(iiaV);
+        }
+
+        Document doc = new Document(type, new DocumentId("doc:foo:testdoc"));
+        doc.setFieldValue("iiiarray", iiiaV);
+
+        {
+            VariableIteratorHandler handler = new VariableIteratorHandler();
+            FieldPath path = type.buildFieldPath("iiiarray[$x][$y][$z]");
+            doc.iterateNested(path, 0, handler);
+
+            String fasit =
+                    "x: 0,y: 0,z: 0, - 1\n" +
+                    "x: 0,y: 0,z: 1, - 2\n" +
+                    "x: 0,y: 0,z: 2, - 3\n" +
+                    "x: 0,y: 1,z: 0, - 2\n" +
+                    "x: 0,y: 1,z: 1, - 4\n" +
+                    "x: 0,y: 1,z: 2, - 6\n" +
+                    "x: 0,y: 2,z: 0, - 3\n" +
+                    "x: 0,y: 2,z: 1, - 6\n" +
+                    "x: 0,y: 2,z: 2, - 9\n" +
+                    "x: 1,y: 0,z: 0, - 2\n" +
+                    "x: 1,y: 0,z: 1, - 4\n" +
+                    "x: 1,y: 0,z: 2, - 6\n" +
+                    "x: 1,y: 1,z: 0, - 4\n" +
+                    "x: 1,y: 1,z: 1, - 8\n" +
+                    "x: 1,y: 1,z: 2, - 12\n" +
+                    "x: 1,y: 2,z: 0, - 6\n" +
+                    "x: 1,y: 2,z: 1, - 12\n" +
+                    "x: 1,y: 2,z: 2, - 18\n" +
+                    "x: 2,y: 0,z: 0, - 3\n" +
+                    "x: 2,y: 0,z: 1, - 6\n" +
+                    "x: 2,y: 0,z: 2, - 9\n" +
+                    "x: 2,y: 1,z: 0, - 6\n" +
+                    "x: 2,y: 1,z: 1, - 12\n" +
+                    "x: 2,y: 1,z: 2, - 18\n" +
+                    "x: 2,y: 2,z: 0, - 9\n" +
+                    "x: 2,y: 2,z: 1, - 18\n" +
+                    "x: 2,y: 2,z: 2, - 27\n";
+
+            assertEquals(fasit, handler.retVal);
+        }
+    }
+
+    @Test
+    public void testGetRecursiveValue() {
+        Document doc = new Document(testDocType, new DocumentId("doc:ns:testdoc"));
+        doc.setFieldValue("primitive1", 1);
+
+        Struct l1s1 = new Struct(doc.getField("l1s1").getDataType());
+        l1s1.setFieldValue("primitive1", 2);
+
+        Struct l2s1 = new Struct(doc.getField("struct2").getDataType());
+        l2s1.setFieldValue("primitive1", 3);
+        l2s1.setFieldValue("primitive2", 4);
+
+        Array<IntegerFieldValue> iarr1 = new Array<>(l2s1.getField("iarray").getDataType());
+        iarr1.add(new IntegerFieldValue(11));
+        iarr1.add(new IntegerFieldValue(12));
+        iarr1.add(new IntegerFieldValue(13));
+        l2s1.setFieldValue("iarray", iarr1);
+
+        ArrayDataType dt = (ArrayDataType)l2s1.getField("sarray").getDataType();
+        Array<Struct> sarr1 = new Array<>(dt);
+        {
+            Struct l3s1 = new Struct(dt.getNestedType());
+            l3s1.setFieldValue("primitive1", 1);
+            l3s1.setFieldValue("primitive2", 2);
+            sarr1.add(l3s1);
+        }
+        {
+            Struct l3s1 = new Struct(dt.getNestedType());
+            l3s1.setFieldValue("primitive1", 1);
+            l3s1.setFieldValue("primitive2", 2);
+            sarr1.add(l3s1);
+        }
+        l2s1.setFieldValue("sarray", sarr1);
+
+        MapFieldValue<StringFieldValue, StringFieldValue> smap1 = new MapFieldValue<>((MapDataType)l2s1.getField("smap").getDataType());
+        smap1.put(new StringFieldValue("leonardo"), new StringFieldValue("dicaprio"));
+        smap1.put(new StringFieldValue("ellen"), new StringFieldValue("page"));
+        smap1.put(new StringFieldValue("joseph"), new StringFieldValue("gordon-levitt"));
+        l2s1.setFieldValue("smap", smap1);
+
+        l1s1.setFieldValue("ss", l2s1.clone());
+
+        MapFieldValue<StringFieldValue, Struct> structmap1 = new MapFieldValue<>((MapDataType)l1s1.getField("structmap").getDataType());
+        structmap1.put(new StringFieldValue("test"), l2s1.clone());
+        l1s1.setFieldValue("structmap", structmap1);
+
+        WeightedSet<StringFieldValue> wset1 = new WeightedSet<>(l1s1.getField("wset").getDataType());
+        wset1.add(new StringFieldValue("foo"));
+        wset1.add(new StringFieldValue("bar"));
+        wset1.add(new StringFieldValue("zoo"));
+        l1s1.setFieldValue("wset", wset1);
+
+        Struct l2s2 = new Struct(doc.getField("struct2").getDataType());
+        l2s2.setFieldValue("primitive1", 5);
+        l2s2.setFieldValue("primitive2", 6);
+
+        WeightedSet<Struct> wset2 = new WeightedSet<>(l1s1.getField("structwset").getDataType());
+        wset2.add(l2s1.clone());
+        wset2.add(l2s2.clone());
+        l1s1.setFieldValue("structwset", wset2);
+
+        doc.setFieldValue("l1s1", l1s1.clone());
+
+        {
+            FieldValue fv = doc.getRecursiveValue("l1s1");
+            assertEquals(l1s1, fv);
+        }
+
+        {
+            FieldValue fv = doc.getRecursiveValue("l1s1.primitive1");
+            assertEquals(new IntegerFieldValue(2), fv);
+        }
+
+        {
+            FieldValue fv = doc.getRecursiveValue("l1s1.ss");
+            assertEquals(l2s1, fv);
+        }
+
+        {
+            FieldValue fv = doc.getRecursiveValue("l1s1.ss.iarray");
+            assertEquals(iarr1, fv);
+        }
+
+        {
+            FieldValue fv = doc.getRecursiveValue("l1s1.ss.iarray[2]");
+            assertEquals(new IntegerFieldValue(13), fv);
+        }
+
+        {
+            FieldValue fv = doc.getRecursiveValue("l1s1.ss.iarray[3]");
+            assertNull(fv);
+        }
+
+        {
+            FieldValue fv = doc.getRecursiveValue("l1s1.ss.sarray[0].primitive1");
+            assertEquals(new IntegerFieldValue(1), fv);
+        }
+
+        {
+            FieldValue fv = doc.getRecursiveValue("l1s1.ss.smap{joseph}");
+            assertEquals(new StringFieldValue("gordon-levitt"), fv);
+        }
+
+        {
+            FieldValue fv = doc.getRecursiveValue("l1s1.ss.smap.key");
+            assertEquals(3, ((Array)fv).size());
+        }
+
+        {
+            FieldValue fv = doc.getRecursiveValue("l1s1.structmap{test}.primitive1");
+            assertEquals(new IntegerFieldValue(3), fv);
+        }
+
+        {
+            FieldValue fv = doc.getRecursiveValue("l1s1.structmap.value.primitive1");
+            assertEquals(new IntegerFieldValue(3), fv);
+        }
+
+        {
+            FieldValue fv = doc.getRecursiveValue("l1s1.wset{foo}");
+            assertEquals(new IntegerFieldValue(1), fv);
+        }
+
+        {
+            FieldValue fv = doc.getRecursiveValue("l1s1.wset.key");
+            assertEquals(3, ((Array)fv).size());
+        }
+
+        {
+            FieldValue fv = doc.getRecursiveValue("l1s1.structwset.key.primitive1");
+            assertEquals(DataType.INT, (((ArrayDataType)fv.getDataType()).getNestedType()));
+            assertEquals(2, ((Array)fv).size());
+        }
+    }
+
+    class ModifyIteratorHandler extends FieldPathIteratorHandler {
+
+        public ModificationStatus doModify(FieldValue fv) {
+            if (fv instanceof StringFieldValue) {
+                fv.assign("newvalue");
+                return ModificationStatus.MODIFIED;
+            }
+
+            return ModificationStatus.NOT_MODIFIED;
+        }
+
+        public boolean onComplex(FieldValue fv) {
+            return false;
+        }
+    }
+
+    class AddIteratorHandler extends FieldPathIteratorHandler {
+
+        @SuppressWarnings("unchecked")
+        public ModificationStatus doModify(FieldValue fv) {
+            if (fv instanceof Array) {
+                ((Array)fv).add(new IntegerFieldValue(32));
+                return ModificationStatus.MODIFIED;
+            }
+
+            return ModificationStatus.NOT_MODIFIED;
+        }
+
+        public boolean onComplex(FieldValue fv) {
+            return false;
+        }
+    }
+
+    class RemoveIteratorHandler extends FieldPathIteratorHandler {
+
+        public ModificationStatus doModify(FieldValue fv) {
+            return ModificationStatus.REMOVED;
+        }
+
+        public boolean onComplex(FieldValue fv) {
+            return false;
+        }
+    }
+
+    @Test
+    public void testModifyDocument() {
+        Document doc = new Document(testDocType, new DocumentId("doc:ns:testdoc"));
+        doc.setFieldValue("primitive1", 1);
+
+        Struct l1s1 = new Struct(doc.getField("l1s1").getDataType());
+        l1s1.setFieldValue("primitive1", 2);
+
+        Struct l2s1 = new Struct(doc.getField("struct2").getDataType());
+        l2s1.setFieldValue("primitive1", 3);
+        l2s1.setFieldValue("primitive2", 4);
+
+        Array<IntegerFieldValue> iarr1 = new Array<>(l2s1.getField("iarray").getDataType());
+        iarr1.add(new IntegerFieldValue(11));
+        iarr1.add(new IntegerFieldValue(12));
+        iarr1.add(new IntegerFieldValue(13));
+        l2s1.setFieldValue("iarray", iarr1);
+
+        ArrayDataType dt = (ArrayDataType)l2s1.getField("sarray").getDataType();
+        Array<Struct> sarr1 = new Array<>(dt);
+        {
+            Struct l3s1 = new Struct(dt.getNestedType());
+            l3s1.setFieldValue("primitive1", 1);
+            l3s1.setFieldValue("primitive2", 2);
+            sarr1.add(l3s1);
+        }
+        {
+            Struct l3s1 = new Struct(dt.getNestedType());
+            l3s1.setFieldValue("primitive1", 1);
+            l3s1.setFieldValue("primitive2", 2);
+            sarr1.add(l3s1);
+        }
+        l2s1.setFieldValue("sarray", sarr1);
+
+        MapFieldValue<StringFieldValue, StringFieldValue> smap1 = new MapFieldValue<>((MapDataType)l2s1.getField("smap").getDataType());
+        smap1.put(new StringFieldValue("leonardo"), new StringFieldValue("dicaprio"));
+        smap1.put(new StringFieldValue("ellen"), new StringFieldValue("page"));
+        smap1.put(new StringFieldValue("joseph"), new StringFieldValue("gordon-levitt"));
+        l2s1.setFieldValue("smap", smap1);
+
+        l1s1.setFieldValue("ss", l2s1.clone());
+
+        MapFieldValue<StringFieldValue, Struct> structmap1 = new MapFieldValue<>((MapDataType)l1s1.getField("structmap").getDataType());
+        structmap1.put(new StringFieldValue("test"), l2s1.clone());
+        l1s1.setFieldValue("structmap", structmap1);
+
+        WeightedSet<StringFieldValue> wset1 = new WeightedSet<>(l1s1.getField("wset").getDataType());
+        wset1.add(new StringFieldValue("foo"));
+        wset1.add(new StringFieldValue("bar"));
+        wset1.add(new StringFieldValue("zoo"));
+        l1s1.setFieldValue("wset", wset1);
+
+        Struct l2s2 = new Struct(doc.getField("struct2").getDataType());
+        l2s2.setFieldValue("primitive1", 5);
+        l2s2.setFieldValue("primitive2", 6);
+
+        WeightedSet<Struct> wset2 = new WeightedSet<>(l1s1.getField("structwset").getDataType());
+        wset2.add(l2s1.clone());
+        wset2.add(l2s2.clone());
+        l1s1.setFieldValue("structwset", wset2);
+
+        doc.setFieldValue("l1s1", l1s1.clone());
+
+        {
+            ModifyIteratorHandler handler = new ModifyIteratorHandler();
+
+            FieldPath path = doc.getDataType().buildFieldPath("l1s1.structmap.value.smap{leonardo}");
+            doc.iterateNested(path, 0, handler);
+
+            FieldValue fv = doc.getRecursiveValue("l1s1.structmap.value.smap{leonardo}");
+            assertEquals(new StringFieldValue("newvalue"), fv);
+        }
+
+        {
+            AddIteratorHandler handler = new AddIteratorHandler();
+            FieldPath path = doc.getDataType().buildFieldPath("l1s1.ss.iarray");
+            doc.iterateNested(path, 0, handler);
+
+            FieldValue fv = doc.getRecursiveValue("l1s1.ss.iarray");
+            assertTrue(((Array)fv).contains(new IntegerFieldValue(32)));
+            assertEquals(4, ((Array)fv).size());
+        }
+
+        {
+            RemoveIteratorHandler handler = new RemoveIteratorHandler();
+            FieldPath path = doc.getDataType().buildFieldPath("l1s1.ss.iarray[1]");
+            doc.iterateNested(path, 0, handler);
+
+            FieldValue fv = doc.getRecursiveValue("l1s1.ss.iarray");
+
+            assertFalse(((Array)fv).contains(new Integer(12)));
+            assertEquals(3, ((Array)fv).size());
+        }
+
+        {
+            RemoveIteratorHandler handler = new RemoveIteratorHandler();
+            FieldPath path = doc.getDataType().buildFieldPath("l1s1.ss.iarray[$x]");
+            doc.iterateNested(path, 0, handler);
+
+            FieldValue fv = doc.getRecursiveValue("l1s1.ss.iarray");
+            assertEquals(0, ((Array)fv).size());
+        }
+
+        {
+            RemoveIteratorHandler handler = new RemoveIteratorHandler();
+            FieldPath path = doc.getDataType().buildFieldPath("l1s1.structmap.value.smap{leonardo}");
+            doc.iterateNested(path, 0, handler);
+
+            FieldValue fv = doc.getRecursiveValue("l1s1.structmap.value.smap");
+            assertFalse(((MapFieldValue)fv).contains(new StringFieldValue("leonardo")));
+        }
+
+        {
+            RemoveIteratorHandler handler = new RemoveIteratorHandler();
+            FieldPath path = doc.getDataType().buildFieldPath("l1s1.wset{foo}");
+            doc.iterateNested(path, 0, handler);
+
+            FieldValue fv = doc.getRecursiveValue("l1s1.wset");
+            assertFalse(((WeightedSet)fv).contains(new StringFieldValue("foo")));
+        }
+    }
+
+    @Test
+    public void testNoType() {
+        try {
+            new Document(null, new DocumentId("doc:null:URI"));
+            fail("Should have gotten an Exception");
+        } catch (NullPointerException | IllegalArgumentException e) {
+            // Success
+        }
+    }
+
+    @Test
+    public void testURI() {
+        String uri = "doc:testdoc:http://www.ntnu.no/";
+
+        DocumentType documentType = docMan.getDocumentType("testdoc");
+        assertNotNull(documentType);
+        Document doc = new Document(docMan.getDocumentType("testdoc"), new DocumentId(uri));
+        assertEquals(doc.getId().toString(), uri);
+    }
+
+    @Test
+    public void testSetGet() {
+        Document doc = new Document(docMan.getDocumentType("testdoc"), new DocumentId("doc:testdoc:test"));
+        Object val = doc.getFieldValue(minField.getName());
+        assertNull(val);
+        doc.setFieldValue(minField.getName(), 500);
+        val = doc.getFieldValue(minField.getName());
+        assertEquals(new IntegerFieldValue(500), val);
+        val = doc.getFieldValue(minField.getName());
+        assertEquals(new IntegerFieldValue(500), val);
+        doc.removeFieldValue(minField);
+        assertNull(doc.getFieldValue(minField.getName()));
+        assertNull(doc.getFieldValue("doesntexist"));
+    }
+
+    @Test
+    public void testGetField() {
+        Document doc = getTestDocument();
+
+        assertNull(doc.getFieldValue("doesntexist"));
+        assertNull(doc.getFieldValue("notintype"));
+    }
+
+    @Test
+    public void testCppDocCompressed() throws IOException {
+        docMan = setUpCppDocType();
+        byte[] data = readFile("src/test/document/serializecpp-lz4-level9.dat");
+        ByteBuffer buf = ByteBuffer.wrap(data);
+
+        Document doc = docMan.createDocument(new GrowableByteBuffer(buf));
+
+        validateCppDoc(doc);
+    }
+
+    @Test
+    public void testCppDoc() throws IOException {
+        docMan = setUpCppDocType();
+        byte[] data = readFile("src/test/document/serializecpp.dat");
+        ByteBuffer buf = ByteBuffer.wrap(data);
+
+        Document doc = docMan.createDocument(new GrowableByteBuffer(buf));
+        validateCppDoc(doc);
+    }
+
+    @Test
+    public void testV6Doc() throws IOException {
+        docMan = setUpCppDocType();
+        byte[] data = readFile("src/tests/data/serializev6.dat");
+        ByteBuffer buf = ByteBuffer.wrap(data);
+
+        Document doc = docMan.createDocument(new GrowableByteBuffer(buf));
+        validateCppDocNotMap(doc);
+    }
+
+    public void validateCppDoc(Document doc) throws IOException {
+        validateCppDocNotMap(doc);
+        MapFieldValue map = (MapFieldValue)doc.getFieldValue("mapfield");
+        assertEquals(map.get(new StringFieldValue("foo1")), new StringFieldValue("bar1"));
+        assertEquals(map.get(new StringFieldValue("foo2")), new StringFieldValue("bar2"));
+    }
+
+    @SuppressWarnings("unchecked")
+    public void validateCppDocNotMap(Document doc) throws IOException {
+        // in practice to validate v6 serialization
+        assertEquals("doc:serializetest:http://test.doc.id/", doc.getId().toString());
+        assertEquals(new IntegerFieldValue(5), doc.getFieldValue("intfield"));
+        assertEquals(new FloatFieldValue((float)-9.23), doc.getFieldValue("floatfield"));
+        assertEquals(new StringFieldValue("This is a string."), doc.getFieldValue("stringfield"));
+        assertEquals(new LongFieldValue(398420092938472983L), doc.getFieldValue("longfield"));
+        assertEquals(new DoubleFieldValue(98374532.398820d), doc.getFieldValue("doublefield"));
+        assertEquals(new StringFieldValue("http://this.is.a.test/"), doc.getFieldValue("urifield"));
+        //NOTE: The value really is unsigned 254, which becomes signed -2:
+        assertEquals(new ByteFieldValue(-2), doc.getFieldValue("bytefield"));
+        ByteBuffer raw = ByteBuffer.wrap("RAW DATA".getBytes());
+        assertEquals(new Raw(raw), doc.getFieldValue("rawfield"));
+
+        Document docindoc = (Document)doc.getFieldValue("docfield");
+        assertEquals(docMan.getDocumentType("docindoc"), docindoc.getDataType());
+        assertEquals(new DocumentId("doc:docindoc:http://embedded"), docindoc.getId());
+
+        Array<FloatFieldValue> array = (Array<FloatFieldValue>)doc.getFieldValue("arrayoffloatfield");
+        assertEquals(new FloatFieldValue(1.0f), array.get(0));
+        assertEquals(new FloatFieldValue(2.0f), array.get(1));
+
+        WeightedSet<StringFieldValue> wset = (WeightedSet<StringFieldValue>)doc.getFieldValue("wsfield");
+        assertEquals(new Integer(50), wset.get(new StringFieldValue("Weighted 0")));
+        assertEquals(new Integer(199), wset.get(new StringFieldValue("Weighted 1")));
+    }
+
+    @Test
+    public void testCppDocSplit() throws IOException {
+        docMan = setUpCppDocType();
+        byte[] headerData = readFile("src/test/document/serializecppsplit_header.dat");
+        byte[] bodyData = readFile("src/test/document/serializecppsplit_body.dat");
+
+        DocumentDeserializer header = DocumentDeserializerFactory.create42(docMan, GrowableByteBuffer.wrap(headerData),
+                GrowableByteBuffer.wrap(bodyData));
+
+        Document doc = new Document(header);
+
+        assertEquals("doc:serializetest:http://test.doc.id/", doc.getId().toString());
+        assertEquals(new IntegerFieldValue(5), doc.getFieldValue("intfield"));
+        assertEquals(new FloatFieldValue((float)-9.23), doc.getFieldValue("floatfield"));
+        assertEquals(new StringFieldValue("This is a string."), doc.getFieldValue("stringfield"));
+        assertEquals(new LongFieldValue(398420092938472983L), doc.getFieldValue("longfield"));
+        assertEquals(new DoubleFieldValue(98374532.398820d), doc.getFieldValue("doublefield"));
+        assertEquals(new StringFieldValue("http://this.is.a.test/"), doc.getFieldValue("urifield"));
+        //NOTE: The value really is unsigned 254, which becomes signed -2:
+        assertEquals(new ByteFieldValue((byte)-2), doc.getFieldValue("bytefield"));
+        ByteBuffer raw = ByteBuffer.wrap("RAW DATA".getBytes());
+        assertEquals(new Raw(raw), doc.getFieldValue("rawfield"));
+
+        Document docindoc = (Document)doc.getFieldValue("docfield");
+        assertEquals(docMan.getDocumentType("docindoc"), docindoc.getDataType());
+        assertEquals(new DocumentId("doc:docindoc:http://embedded"), docindoc.getId());
+
+        WeightedSet wset = (WeightedSet)doc.getFieldValue("wsfield");
+        assertEquals(new Integer(50), wset.get(new StringFieldValue("Weighted 0")));
+        assertEquals(new Integer(199), wset.get(new StringFieldValue("Weighted 1")));
+    }
+
+    @Test
+    public void testCppDocSplitNoBody() throws IOException {
+        docMan = setUpCppDocType();
+        byte[] headerData = readFile("src/test/document/serializecppsplit_header.dat");
+
+        DocumentDeserializer header = DocumentDeserializerFactory.create42(docMan, GrowableByteBuffer.wrap(headerData));
+
+        Document doc = new Document(header);
+
+        assertEquals("doc:serializetest:http://test.doc.id/", doc.getId().toString());
+        assertEquals(new FloatFieldValue((float)-9.23), doc.getFieldValue("floatfield"));
+        assertEquals(new StringFieldValue("This is a string."), doc.getFieldValue("stringfield"));
+        assertEquals(new LongFieldValue(398420092938472983L), doc.getFieldValue("longfield"));
+        assertEquals(new StringFieldValue("http://this.is.a.test/"), doc.getFieldValue("urifield"));
+    }
+
+    @Test
+    public void testGenerateSerializedFile() throws IOException {
+
+        docMan = setUpCppDocType();
+        Document doc = new Document(docMan.getDocumentType("serializetest"),
+                                    new DocumentId("doc:serializetest:http://test.doc.id/"));
+
+        Document docindoc = new Document(docMan.getDocumentType("docindoc"),
+                                         new DocumentId("doc:serializetest:http://doc.in.doc/"));
+        docindoc.setFieldValue("stringindocfield", "Elvis is dead");
+        doc.setFieldValue("docfield", docindoc);
+
+        Array<FloatFieldValue> l = new Array<>(doc.getField("arrayoffloatfield").getDataType());
+        l.add(new FloatFieldValue((float)1.0));
+        l.add(new FloatFieldValue((float)2.0));
+        doc.setFieldValue("arrayoffloatfield", l);
+
+        WeightedSet<StringFieldValue>
+                wset = new WeightedSet<>(doc.getDataType().getField("wsfield").getDataType());
+        wset.put(new StringFieldValue("Weighted 0"), 50);
+        wset.put(new StringFieldValue("Weighted 1"), 199);
+        doc.setFieldValue("wsfield", wset);
+
+        MapFieldValue<StringFieldValue, StringFieldValue> map =
+                new MapFieldValue<>(
+                        (MapDataType)doc.getDataType().getField("mapfield").getDataType());
+        map.put(new StringFieldValue("foo1"), new StringFieldValue("bar1"));
+        map.put(new StringFieldValue("foo2"), new StringFieldValue("bar2"));
+        doc.setFieldValue("mapfield", map);
+
+        doc.setFieldValue("bytefield", new ByteFieldValue((byte)254));
+        doc.setFieldValue("rawfield", new Raw(ByteBuffer.wrap("RAW DATA".getBytes())));
+        doc.setFieldValue("intfield", new IntegerFieldValue(5));
+        doc.setFieldValue("floatfield", new FloatFieldValue(-9.23f));
+        doc.setFieldValue("stringfield", new StringFieldValue("This is a string."));
+        doc.setFieldValue("longfield", new LongFieldValue(398420092938472983L));
+        doc.setFieldValue("doublefield", new DoubleFieldValue(98374532.398820d));
+        doc.setFieldValue("urifield", new StringFieldValue("http://this.is.a.test/"));
+
+        int size = doc.getSerializedSize();
+        GrowableByteBuffer buf = new GrowableByteBuffer(size, 2.0f);
+
+        doc.serialize(buf);
+        assertEquals(size, buf.position());
+
+        buf.position(0);
+
+        FileOutputStream fos = new FileOutputStream("src/tests/data/serializejava.dat");
+        fos.write(buf.array(), 0, size);
+        fos.close();
+
+        CompressionConfig noncomp = new CompressionConfig();
+        CompressionConfig lz4comp = new CompressionConfig(CompressionType.LZ4);
+
+        doc.getDataType().getHeaderType().setCompressionConfig(lz4comp);
+        doc.getDataType().getBodyType().setCompressionConfig(lz4comp);
+        buf = new GrowableByteBuffer(size, 2.0f);
+
+        doc.serialize(buf);
+        doc.getDataType().getHeaderType().setCompressionConfig(noncomp);
+        doc.getDataType().getBodyType().setCompressionConfig(noncomp);
+        fos = new FileOutputStream("src/tests/data/serializejava-compressed.dat");
+        fos.write(buf.array(), 0, buf.position());
+        fos.close();
+    }
+
+    @Test
+    public void testSerializeDeserialize() {
+        setUpSertestDocType();
+        Document doc = getSertestDocument();
+
+        GrowableByteBuffer data = new GrowableByteBuffer();
+        doc.serialize(data);
+        int size = doc.getSerializedSize();
+
+        assertEquals(size, data.position());
+
+        data.flip();
+
+        try {
+            FileOutputStream fos = new FileOutputStream("src/test/files/testser.dat");
+            fos.write(data.array(), 0, data.remaining());
+            fos.close();
+        } catch (Exception e) {
+        }
+
+        Document doc2 = docMan.createDocument(data);
+
+        assertEquals(doc.getFieldValue("mailid"), doc2.getFieldValue("mailid"));
+        assertEquals(doc.getFieldValue("date"), doc2.getFieldValue("date"));
+        assertEquals(doc.getFieldValue("from"), doc2.getFieldValue("from"));
+        assertEquals(doc.getFieldValue("to"), doc2.getFieldValue("to"));
+        assertEquals(doc.getFieldValue("subject"), doc2.getFieldValue("subject"));
+        assertEquals(doc.getFieldValue("body"), doc2.getFieldValue("body"));
+        assertEquals(doc.getFieldValue("attachmentcount"), doc2.getFieldValue("attachmentcount"));
+        assertEquals(doc.getFieldValue("attachments"), doc2.getFieldValue("attachments"));
+        byte[] docRawBytes = ((Raw)doc.getFieldValue("rawfield")).getByteBuffer().array();
+        byte[] doc2RawBytes = ((Raw)doc2.getFieldValue("rawfield")).getByteBuffer().array();
+        assertEquals(docRawBytes.length, doc2RawBytes.length);
+        for (int i = 0; i < docRawBytes.length; i++) {
+            assertEquals(docRawBytes[i], doc2RawBytes[i]);
+        }
+        assertEquals(doc.getFieldValue("weightedfield"), doc2.getFieldValue("weightedfield"));
+        assertEquals(doc.getFieldValue("mapfield"), doc2.getFieldValue("mapfield"));
+        // Do the same thing, splitting document in two
+        DocumentSerializer header = DocumentSerializerFactory.create42(new GrowableByteBuffer(), true);
+        DocumentSerializer body = DocumentSerializerFactory.create42(new GrowableByteBuffer());
+        doc.serializeHeader(header);
+        doc.serializeBody(body);
+        header.getBuf().flip();
+        body.getBuf().flip();
+
+        try {
+            FileOutputStream fos = new FileOutputStream("src/test/files/testser-split.header.dat");
+            fos.write(header.getBuf().array(), 0, header.getBuf().remaining());
+            fos.close();
+            fos = new FileOutputStream("src/test/files/testser-split.body.dat");
+            fos.write(body.getBuf().array(), 0, body.getBuf().remaining());
+            fos.close();
+        } catch (Exception e) {
+        }
+
+        DocumentDeserializer deser = DocumentDeserializerFactory.create42(docMan, header.getBuf(), body.getBuf());
+
+        doc2 = new Document(deser);
+
+        assertEquals(doc.getFieldValue("mailid"), doc2.getFieldValue("mailid"));
+        assertEquals(doc.getFieldValue("date"), doc2.getFieldValue("date"));
+        assertEquals(doc.getFieldValue("from"), doc2.getFieldValue("from"));
+        assertEquals(doc.getFieldValue("to"), doc2.getFieldValue("to"));
+        assertEquals(doc.getFieldValue("subject"), doc2.getFieldValue("subject"));
+        assertEquals(doc.getFieldValue("body"), doc2.getFieldValue("body"));
+        assertEquals(doc.getFieldValue("attachmentcount"), doc2.getFieldValue("attachmentcount"));
+        assertEquals(doc.getFieldValue("attachments"), doc2.getFieldValue("attachments"));
+        docRawBytes = ((Raw)doc.getFieldValue("rawfield")).getByteBuffer().array();
+        doc2RawBytes = ((Raw)doc2.getFieldValue("rawfield")).getByteBuffer().array();
+        assertEquals(docRawBytes.length, doc2RawBytes.length);
+        for (int i = 0; i < docRawBytes.length; i++) {
+            assertEquals(docRawBytes[i], doc2RawBytes[i]);
+        }
+        assertEquals(doc.getFieldValue("weightedfield"), doc2.getFieldValue("weightedfield"));
+        assertEquals(doc.getFieldValue("mapfield"), doc2.getFieldValue("mapfield"));
+
+        Document docInDoc = (Document)doc.getFieldValue("docindoc");
+        assert (docInDoc != null);
+        assertEquals(new StringFieldValue("ball"), docInDoc.getFieldValue("tull"));
+    }
+
+    @Test
+    public void testSerializeDeserializeCompressed() {
+        setUpSertestDocType();
+        Document doc = getSertestDocument();
+
+        CompressionConfig noncomp = new CompressionConfig();
+        CompressionConfig lz4comp = new CompressionConfig(CompressionType.LZ4);
+
+        doc.getDataType().getHeaderType().setCompressionConfig(lz4comp);
+        doc.getDataType().getBodyType().setCompressionConfig(lz4comp);
+
+        GrowableByteBuffer data = new GrowableByteBuffer();
+        doc.serialize(data);
+        int size = doc.getSerializedSize();
+        doc.getDataType().getHeaderType().setCompressionConfig(noncomp);
+        doc.getDataType().getBodyType().setCompressionConfig(noncomp);
+
+        assertEquals(size, data.position());
+
+        data.flip();
+
+        try {
+            FileOutputStream fos = new FileOutputStream("src/test/files/testser.dat");
+            fos.write(data.array(), 0, data.remaining());
+            fos.close();
+        } catch (Exception e) {
+        }
+
+        Document doc2 = docMan.createDocument(data);
+
+        assertEquals(doc.getFieldValue("mailid"), doc2.getFieldValue("mailid"));
+        assertEquals(doc.getFieldValue("date"), doc2.getFieldValue("date"));
+        assertEquals(doc.getFieldValue("from"), doc2.getFieldValue("from"));
+        assertEquals(doc.getFieldValue("to"), doc2.getFieldValue("to"));
+        assertEquals(doc.getFieldValue("subject"), doc2.getFieldValue("subject"));
+        assertEquals(doc.getFieldValue("body"), doc2.getFieldValue("body"));
+        assertEquals(doc.getFieldValue("attachmentcount"), doc2.getFieldValue("attachmentcount"));
+        assertEquals(doc.getFieldValue("attachments"), doc2.getFieldValue("attachments"));
+        byte[] docRawBytes = ((Raw)doc.getFieldValue("rawfield")).getByteBuffer().array();
+        byte[] doc2RawBytes = ((Raw)doc2.getFieldValue("rawfield")).getByteBuffer().array();
+        assertEquals(docRawBytes.length, doc2RawBytes.length);
+        for (int i = 0; i < docRawBytes.length; i++) {
+            assertEquals(docRawBytes[i], doc2RawBytes[i]);
+        }
+        assertEquals(doc.getFieldValue("weightedfield"), doc2.getFieldValue("weightedfield"));
+        assertEquals(doc.getFieldValue("mapfield"), doc2.getFieldValue("mapfield"));
+
+        // Do the same thing, splitting document in two
+        BufferSerializer header = new BufferSerializer(new GrowableByteBuffer());
+        BufferSerializer body = new BufferSerializer(new GrowableByteBuffer());
+        doc.serializeHeader(header);
+        doc.serializeBody(body);
+        header.getBuf().flip();
+        body.getBuf().flip();
+
+        try {
+            FileOutputStream fos = new FileOutputStream("src/test/files/testser-split.header.dat");
+            fos.write(header.getBuf().array(), 0, header.getBuf().remaining());
+            fos.close();
+            fos = new FileOutputStream("src/test/files/testser-split.body.dat");
+            fos.write(body.getBuf().array(), 0, body.getBuf().remaining());
+            fos.close();
+        } catch (Exception e) {
+        }
+
+        DocumentDeserializer deser = DocumentDeserializerFactory.create42(docMan, header.getBuf(), body.getBuf());
+
+        doc2 = new Document(deser);
+
+        assertEquals(doc.getFieldValue("mailid"), doc2.getFieldValue("mailid"));
+        assertEquals(doc.getFieldValue("date"), doc2.getFieldValue("date"));
+        assertEquals(doc.getFieldValue("from"), doc2.getFieldValue("from"));
+        assertEquals(doc.getFieldValue("to"), doc2.getFieldValue("to"));
+        assertEquals(doc.getFieldValue("subject"), doc2.getFieldValue("subject"));
+        assertEquals(doc.getFieldValue("body"), doc2.getFieldValue("body"));
+        assertEquals(doc.getFieldValue("attachmentcount"), doc2.getFieldValue("attachmentcount"));
+        assertEquals(doc.getFieldValue("attachments"), doc2.getFieldValue("attachments"));
+        docRawBytes = ((Raw)doc.getFieldValue("rawfield")).getByteBuffer().array();
+        doc2RawBytes = ((Raw)doc2.getFieldValue("rawfield")).getByteBuffer().array();
+        assertEquals(docRawBytes.length, doc2RawBytes.length);
+        for (int i = 0; i < docRawBytes.length; i++) {
+            assertEquals(docRawBytes[i], doc2RawBytes[i]);
+        }
+        assertEquals(doc.getFieldValue("weightedfield"), doc2.getFieldValue("weightedfield"));
+        assertEquals(doc.getFieldValue("mapfield"), doc2.getFieldValue("mapfield"));
+    }
+
+    @Test
+    public void testDeserialize() {
+        setUpSertestDocType();
+
+        BufferSerializer buf = new BufferSerializer();
+        try {
+            new Document(DocumentDeserializerFactory.create42(docMan, buf.getBuf()));
+            assertTrue(false);
+        } catch (Exception e) {
+            assertTrue(true);
+        }
+
+        buf = BufferSerializer.wrap("Hello world".getBytes());
+        try {
+            new Document(DocumentDeserializerFactory.create42(docMan, buf.getBuf()));
+            assertTrue(false);
+        } catch (Exception e) {
+            assertTrue(true);
+        }
+    }
+
+    @Test
+    public void testInheritance() {
+        // Create types that inherit each other.. And test that it works..
+        DocumentType parentType = new DocumentType("parent");
+        parentType.addField(new Field("parentbodyint", DataType.INT, false));
+        parentType.addField(new Field("parentheaderint", DataType.INT, true));
+        parentType.addField(new Field("overwritten", DataType.INT, true));
+        DocumentType childType = new DocumentType("child");
+        childType.addField(new Field("childbodyint", DataType.INT, false));
+        childType.addField(new Field("childheaderint", DataType.INT, true));
+        childType.addField(new Field("overwritten", DataType.INT, true));
+        childType.inherit(parentType);
+
+        DocumentTypeManager manager = new DocumentTypeManager();
+        manager.register(childType);
+
+        Document child = new Document(childType, new DocumentId("doc:what:test"));
+        child.setFieldValue(childType.getField("parentbodyint"), new IntegerFieldValue(4));
+        child.setFieldValue("parentheaderint", 6);
+        child.setFieldValue("overwritten", 7);
+        child.setFieldValue("childbodyint", 14);
+
+        GrowableByteBuffer buffer = new GrowableByteBuffer(1024, 2f);
+        child.serialize(buffer);
+        buffer.flip();
+        Document childCopy = manager.createDocument(buffer);
+
+        // Test various ways of retrieving values
+        assertEquals(new IntegerFieldValue(4), childCopy.getFieldValue(childType.getField("parentbodyint")));
+        assertEquals(new IntegerFieldValue(6), childCopy.getFieldValue("parentheaderint"));
+        assertEquals(new IntegerFieldValue(7), childCopy.getFieldValue("overwritten"));
+
+        assertEquals(child, childCopy);
+    }
+
+    @Test
+    public void testInheritanceTypeMismatch() {
+        DocumentType parentType = new DocumentType("parent");
+        parentType.addField(new Field("parentbodyint", DataType.INT, false));
+        parentType.addField(new Field("parentheaderint", DataType.INT, true));
+        parentType.addField(new Field("overwritten", DataType.STRING, true));
+        DocumentType childType = new DocumentType("child");
+        childType.addField(new Field("childbodyint", DataType.INT, false));
+        childType.addField(new Field("childheaderint", DataType.INT, true));
+        childType.addField(new Field("overwritten", DataType.INT, true));
+        try {
+            childType.inherit(parentType);
+            fail("Inheritance with conflicting types worked.");
+        } catch (IllegalArgumentException e) {
+            assertEquals(e.getMessage(),
+                         "Inheritance type mismatch: field \"overwritten\" in datatype \"child\" must have same datatype as in parent document type \"parent\"");
+        }
+    }
+
+    @Test
+    public void testFieldValueImplementations() {
+        docMan = new DocumentTypeManager();
+        DocumentType docType = new DocumentType("impl");
+        docType.addField(new Field("something", DataType.getArray(DataType.STRING), false));
+        docMan.register(docType);
+
+        //just checks that isAssignableFrom() in Document.setFieldValue() goes the right way
+
+        Document doc = new Document(docMan.getDocumentType("impl"), new DocumentId("doc:doctest:fooooobardoc"));
+        Array<StringFieldValue> testlist = new Array<>(doc.getField("something").getDataType());
+        doc.setFieldValue("something", testlist);
+    }
+
+    @Test
+    public void testCompressionConfigured() {
+
+        int size_uncompressed;
+        {
+            DocumentTypeManager docMan = new DocumentTypeManager();
+            docMan.configure("file:src/tests/data/cppdocument.cfg");
+
+            Document doc = new Document(docMan.getDocumentType("serializetest"), new DocumentId("doc:test:test"));
+
+            doc.setFieldValue("stringfield",
+                              "compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me ");
+
+            GrowableByteBuffer data = new GrowableByteBuffer();
+            doc.serialize(data);
+            size_uncompressed = data.position();
+        }
+
+        DocumentTypeManager docMan = new DocumentTypeManager();
+        docMan.configure("file:src/tests/data/compressed.cfg");
+
+        Document doc = new Document(docMan.getDocumentType("serializetest"), new DocumentId("doc:test:test"));
+
+        doc.setFieldValue("stringfield",
+                          "compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me ");
+
+        GrowableByteBuffer data = new GrowableByteBuffer();
+        doc.serialize(data);
+        int size_compressed = data.position();
+
+        assertTrue(size_compressed + " < " + size_uncompressed, size_compressed < size_uncompressed);
+    }
+
+    @Test
+    public void testDocumentDataType() {
+        //use documenttypemanagerconfigurer to read config
+        docMan = new DocumentTypeManager();
+        DocumentTypeManagerConfigurer.configure(docMan, "file:src/test/document/docindoc.cfg");
+
+        //get a document type
+        docMan.getDocumentType("outerdoc");
+
+        //create document and necessary structures
+        Document outerdoc = new Document(docMan.getDocumentType("outerdoc"), new DocumentId("doc:recursion:outerdoc"));
+        Document innerdoc = new Document(docMan.getDocumentType("innerdoc"), new DocumentId("doc:recursion:innerdoc"));
+
+        innerdoc.setFieldValue("intfield", 55);
+
+        outerdoc.setFieldValue("stringfield", "boooo");
+        outerdoc.setFieldValue("docfield", innerdoc);
+
+        //serialize document
+        int size = outerdoc.getSerializedSize();
+        GrowableByteBuffer buf = new GrowableByteBuffer(size, 2.0f);
+        outerdoc.serialize(buf);
+
+        assertEquals(size, buf.position());
+
+        //deserialize document
+        buf.position(0);
+        Document outerdoc2 = docMan.createDocument(buf);
+
+        //compare values
+        assertEquals(outerdoc, outerdoc2);
+    }
+
+    @Test
+    public void testTimestamp() {
+        Document doc = new Document(docMan.getDocumentType("testdoc"), new DocumentId("doc:testdoc:timetest"));
+        assertNull(doc.getLastModified());
+        doc.setLastModified(4350129845023985L);
+        assertEquals(new Long(4350129845023985L), doc.getLastModified());
+        doc.setLastModified(null);
+        assertNull(doc.getLastModified());
+
+        Long timestamp = System.currentTimeMillis();
+        doc.setLastModified(timestamp);
+        assertEquals(timestamp, doc.getLastModified());
+
+        GrowableByteBuffer buf = new GrowableByteBuffer();
+        doc.getSerializedSize();
+        doc.serialize(buf);
+        buf.position(0);
+
+        Document doc2 = docMan.createDocument(buf);
+        assertNull(doc2.getLastModified());
+    }
+
+    @Test
+    public void testToXml() {
+        setUpSertestDocType();
+        Document doc = getSertestDocument();
+        doc.setFieldValue("mailid", "emailfromalicetobob&someone");
+        // Pizza Hut Sunnyvale: x="-122057174" y="37374821" latlong="N37.374821;W122.057174"
+        doc.setFieldValue("myposfield", PositionDataType.valueOf(-122057174, 37374821));
+        String xml = doc.toXML(" ");
+        System.out.println(xml);
+        assertTrue(xml.contains(SERTEST_DOC_AS_XML_HEAD));
+        assertTrue(xml.contains(SERTEST_DOC_AS_XML_FOOT));
+        assertTrue(xml.contains(SERTEST_DOC_AS_XML_WEIGHT1) || xml.contains(SERTEST_DOC_AS_XML_WEIGHT2));
+        assertTrue(xml.contains(SERTEST_DOC_AS_XML_SUNNYVALE));
+    }
+
+    @Test
+    public void testXmlSerializer() {
+        setUpSertestDocType();
+        Document doc = getSertestDocument();
+        doc.setFieldValue("mailid", "emailfromalicetobob&someone");
+        doc.setFieldValue("myposfield", PositionDataType.valueOf(-122057174, 37374821));
+        String xml = doc.toXML(" ");
+        XmlDocumentWriter w = XmlDocumentWriter.createWriter(" ");
+        w.write(doc);
+        String otherXml = doc.toXML(" ");
+        assertEquals(xml, otherXml);
+    }
+
+    @Test
+    public void testSingleFieldToXml() {
+        Document doc = new Document(docMan.getDocumentType("testdoc"), new DocumentId("doc:testdoc:xmltest"));
+        doc.setFieldValue("stringattr", new StringFieldValue("hello world"));
+        assertEquals("<value>hello world</value>\n", doc.getFieldValue("stringattr").toXml());
+    }
+
+    @Test
+    public void testDelegatedDocumentToXml() {
+        Document doc = new Document(docMan.getDocumentType("testdoc"), new DocumentId("doc:testdoc:xmltest"));
+        doc.setFieldValue("stringattr", new StringFieldValue("hello universe"));
+        // Should just delegate to toXML
+        assertEquals(
+                "<document documenttype=\"testdoc\" documentid=\"doc:testdoc:xmltest\">\n" +
+                "  <stringattr>hello universe</stringattr>\n" +
+                "</document>\n",
+                doc.toXml());
+    }
+
+    @Test
+    public void testEmptyStringsSerialization() {
+        docMan = new DocumentTypeManager();
+        DocumentType docType = new DocumentType("emptystrings");
+        docType.addField(new Field("emptystring", DataType.STRING));
+        docType.addField(new Field("nullstring", DataType.STRING));
+        docType.addField(new Field("spacestring", DataType.STRING));
+        docType.addField(new Field("astring", DataType.STRING));
+        docMan.registerDocumentType(docType);
+
+        GrowableByteBuffer grbuf = new GrowableByteBuffer();
+        {
+            Document doc = new Document(docType, new DocumentId("doc:a:b:emptystrings"));
+
+            doc.setFieldValue("emptystring", "");
+            doc.removeFieldValue("nullstring");
+            doc.setFieldValue("spacestring", " ");
+            doc.setFieldValue("astring", "a");
+            assertEquals(new StringFieldValue(""), doc.getFieldValue("emptystring"));
+            assertNull(doc.getFieldValue("nullstring"));
+            assertEquals(new StringFieldValue(" "), doc.getFieldValue("spacestring"));
+            assertEquals(new StringFieldValue("a"), doc.getFieldValue("astring"));
+
+            doc.getSerializedSize();
+            doc.serialize(grbuf);
+
+            grbuf.flip();
+        }
+        {
+            Document doc2 = docMan.createDocument(grbuf);
+
+            assertEquals(new StringFieldValue(""), doc2.getFieldValue("emptystring"));
+            assertNull(doc2.getFieldValue("nullstring"));
+            assertEquals(new StringFieldValue(" "), doc2.getFieldValue("spacestring"));
+            assertEquals(new StringFieldValue("a"), doc2.getFieldValue("astring"));
+
+        }
+    }
+
+    @Test
+    public void testBug2354045() {
+        DocumentTypeManager docMan = new DocumentTypeManager();
+        DocumentType docType = new DocumentType("bug2354045");
+        docType.addField(new Field("string", DataType.STRING));
+        docMan.register(docType);
+
+        GrowableByteBuffer grbuf = new GrowableByteBuffer();
+
+        Document doc = new Document(docType, new DocumentId("doc:a:b:strings"));
+
+        doc.removeFieldValue("string");
+        assertNull(doc.getFieldValue("string"));
+
+        doc.getSerializedSize();
+        doc.serialize(grbuf);
+
+        grbuf.flip();
+
+        Document doc2 = docMan.createDocument(grbuf);
+        assertNull(doc2.getFieldValue("string"));
+        assertEquals(doc, doc2);
+    }
+
+    @Test
+    public void testUnknownFieldsDeserialization() {
+        DocumentTypeManager docTypeManasjer = new DocumentTypeManager();
+
+        GrowableByteBuffer buf = new GrowableByteBuffer();
+
+        {
+            DocumentType typeWithDinner = new DocumentType("elvis");
+            typeWithDinner.addField("breakfast", DataType.STRING);
+            typeWithDinner.addField("lunch", DataType.INT);
+            typeWithDinner.addField("dinner", DataType.DOUBLE);
+            docTypeManasjer.registerDocumentType(typeWithDinner);
+
+            Document docWithDinner = new Document(typeWithDinner, "doc:elvis:has:left:the:building");
+            docWithDinner.setFieldValue("breakfast", "peanut butter");
+            docWithDinner.setFieldValue("lunch", 14);
+            docWithDinner.setFieldValue("dinner", 5.43d);
+
+            docWithDinner.serialize(buf);
+            buf.flip();
+
+            docTypeManasjer.clear();
+        }
+
+        {
+            DocumentType typeWithoutDinner = new DocumentType("elvis");
+            typeWithoutDinner.addField("breakfast", DataType.STRING);
+            typeWithoutDinner.addField("lunch", DataType.INT);
+            //no dinner
+            docTypeManasjer.registerDocumentType(typeWithoutDinner);
+
+            Document docWithoutDinner = docTypeManasjer.createDocument(buf);
+
+            assertEquals(new StringFieldValue("peanut butter"), docWithoutDinner.getFieldValue("breakfast"));
+            assertEquals(new IntegerFieldValue(14), docWithoutDinner.getFieldValue("lunch"));
+            assertNull(docWithoutDinner.getFieldValue("dinner"));
+        }
+    }
+
+    @Test
+    public void testBug3233988() {
+        DocumentType type = new DocumentType("foo");
+        Field field = new Field("productdesc", DataType.STRING);
+        type.addField(field);
+
+        Document doc;
+
+        doc = new Document(type, "doc:foo:bar:bar");
+        doc.removeFieldValue("productdesc");
+        assertNull(doc.getFieldValue("productdesc"));
+
+        doc = new Document(type, "doc:foo:bar:bar");
+        assertNull(doc.getFieldValue("productdesc"));
+    }
+
+    @Test
+    public void testRequireThatDocumentWithIdSchemaIdChecksType() {
+        DocumentType docType = new DocumentType("mytype");
+        try {
+            new Document(docType, "id:namespace:mytype::foo");
+        } catch (Exception e) {
+            fail();
+        }
+
+        try {
+            new Document(docType, "id:namespace:wrong-type::foo");
+            fail();
+        } catch (IllegalArgumentException e) {
+        }
+    }
+
+    private class MyDocumentReader implements DocumentReader {
+
+        @Override
+        public void read(Document document) {
+        }
+
+        @Override
+        public DocumentId readDocumentId() {
+            return null;
+        }
+
+        @Override
+        public DocumentType readDocumentType() {
+            return null;
+        }
+
+    }
+
+    @Test
+    public void testRequireThatChangingDocumentTypeChecksId() {
+        MyDocumentReader reader = new MyDocumentReader();
+        Document doc = new Document(reader);
+        doc.setId(new DocumentId("id:namespace:mytype::foo"));
+        DocumentType docType = new DocumentType("mytype");
+        try {
+            doc.setDataType(docType);
+        } catch (Exception e) {
+            fail();
+        }
+        doc = new Document(reader);
+        doc.setId(new DocumentId("id:namespace:mytype::foo"));
+        DocumentType wrongType = new DocumentType("wrongtype");
+        try {
+            doc.setDataType(wrongType);
+            fail();
+        } catch (IllegalArgumentException e) {
+        }
+    }
+
+    @Test
+    public void testDocumentComparisonDoesNotCorruptStateBug6394548() {
+        DocumentTypeManager docMan = new DocumentTypeManager();
+        DocumentType docType = new DocumentType("bug2354045");
+        docType.addField(new Field("string", 2, DataType.STRING, true));
+        docType.addField(new Field("int", 1, DataType.INT, true));
+        docType.addField(new Field("float", 0, DataType.FLOAT, true));
+        docMan.register(docType);
+
+        Document doc1 = new Document(docType, new DocumentId("doc:a:b:bug6394548"));
+        doc1.setFieldValue("string", new StringFieldValue("hello world"));
+        doc1.setFieldValue("int", new IntegerFieldValue(1234));
+        doc1.setFieldValue("float", new FloatFieldValue(5.5f));
+        String doc1Before = doc1.toXml();
+
+        Document doc2 = new Document(docType, new DocumentId("doc:a:b:bug6394548"));
+        doc2.setFieldValue("string", new StringFieldValue("aardvark"));
+        doc2.setFieldValue("int", new IntegerFieldValue(90909));
+        doc2.setFieldValue("float", new FloatFieldValue(777.15f));
+        String doc2Before = doc2.toXml();
+
+        doc1.compareTo(doc2);
+
+        String doc1After = doc1.toXml();
+        String doc2After = doc2.toXml();
+
+        assertEquals(doc1Before, doc1After);
+        assertEquals(doc2Before, doc2After);
+    }
+
+}
diff --git a/document/src/test/java/com/yahoo/document/DocumentTestCaseBase.java b/document/src/test/java/com/yahoo/document/DocumentTestCaseBase.java
new file mode 100644
index 00000000000..11f2103f238
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/DocumentTestCaseBase.java
@@ -0,0 +1,85 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import com.yahoo.document.datatypes.FloatFieldValue;
+import com.yahoo.document.datatypes.Raw;
+
+import java.nio.ByteBuffer;
+
+import static org.junit.Assert.assertNotNull;
+
+public class DocumentTestCaseBase {
+
+    protected Field byteField = null;
+    protected Field intField = null;
+    protected Field rawField = null;
+    protected Field floatField = null;
+    protected Field stringField = null;
+    protected Field minField = null;
+    protected DocumentType testDocType = null;
+    protected DocumentTypeManager docMan;
+
+    public DocumentTestCaseBase() {
+        docMan = new DocumentTypeManager();
+        testDocType = new DocumentType("testdoc");
+
+        testDocType.addHeaderField("byteattr", DataType.BYTE);
+        testDocType.addHeaderField("intattr", DataType.INT);
+        testDocType.addField("rawattr", DataType.RAW);
+        testDocType.addField("floatattr", DataType.FLOAT);
+        testDocType.addHeaderField("stringattr", DataType.STRING);
+        testDocType.addHeaderField("Minattr", DataType.INT);
+        testDocType.addHeaderField("Minattr2", DataType.INT);
+        testDocType.addHeaderField("primitive1", DataType.INT);
+
+        StructDataType sdt = new StructDataType("struct1");
+        sdt.addField(new Field("primitive1", DataType.INT));
+        sdt.addField(new Field("primitive2", DataType.INT));
+
+        StructDataType sdt2 = new StructDataType("struct2");
+        sdt2.addField(new Field("primitive1", DataType.INT));
+        sdt2.addField(new Field("primitive2", DataType.INT));
+        sdt2.addField(new Field("iarray", new ArrayDataType(DataType.INT)));
+        sdt2.addField(new Field("sarray", new ArrayDataType(sdt)));
+        sdt2.addField(new Field("smap", new MapDataType(DataType.STRING, DataType.STRING)));
+
+        testDocType.addField(new Field("struct2", sdt2));
+
+        StructDataType sdt3 = new StructDataType("struct3");
+        sdt3.addField(new Field("primitive1", DataType.INT));
+        sdt3.addField(new Field("ss", sdt2));
+        sdt3.addField(new Field("structmap", new MapDataType(DataType.STRING, sdt2)));
+        sdt3.addField(new Field("wset", new WeightedSetDataType(DataType.STRING, false, false)));
+        sdt3.addField(new Field("structwset", new WeightedSetDataType(sdt2, false, false)));
+
+        testDocType.addField(new Field("l1s1", sdt3));
+
+        byteField = testDocType.getField("byteattr");
+        intField = testDocType.getField("intattr");
+        rawField = testDocType.getField("rawattr");
+        floatField = testDocType.getField("floatattr");
+        stringField = testDocType.getField("stringattr");
+        minField = testDocType.getField("Minattr");
+
+        docMan.registerDocumentType(testDocType);
+    }
+
+    public Document getTestDocument() {
+        Document doc = new Document(docMan.getDocumentType("testdoc"), new DocumentId("doc:testdoc:http://www.ntnu.no/"));
+        doc.setFieldValue(byteField.getName(), (byte) 30);
+        doc.setFieldValue(byteField.getName(), (byte)30);
+        doc.setFieldValue(intField.getName(), 50);
+
+        ByteBuffer buf = ByteBuffer.allocate(7).put("hei der".getBytes());
+        buf.flip();
+
+        doc.setFieldValue(rawField, new Raw(buf));
+        doc.setFieldValue(floatField, new FloatFieldValue((float) 3.56));
+        doc.setFieldValue("stringattr", "tjohei");
+
+        assertNotNull(doc.getId());
+        assertNotNull(doc.getDataType());
+
+        return doc;
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/DocumentTypeIdTestCase.java b/document/src/test/java/com/yahoo/document/DocumentTypeIdTestCase.java
new file mode 100644
index 00000000000..e086aa63d09
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/DocumentTypeIdTestCase.java
@@ -0,0 +1,49 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import org.junit.Test;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.CoreMatchers.not;
+import static org.junit.Assert.assertThat;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ * @since 5.1.10
+ */
+public class DocumentTypeIdTestCase {
+
+    @Test
+    public void requireThatToStringWorks() {
+        DocumentTypeId r = new DocumentTypeId(123);
+        assertThat(r.toString().contains("123"), is(true));
+    }
+
+    @Test
+    public void requireThatEqualsAndHashCodeWorks() {
+        DocumentTypeId r1 = new DocumentTypeId(123);
+        DocumentTypeId r2 = new DocumentTypeId(123);
+        DocumentTypeId r3 = new DocumentTypeId(456);
+
+        assertThat(r1, equalTo(r1));
+        assertThat(r1, equalTo(r2));
+        assertThat(r2, equalTo(r1));
+        assertThat(r1.hashCode(), equalTo(r2.hashCode()));
+
+        assertThat(r1, not(equalTo(r3)));
+        assertThat(r3, not(equalTo(r1)));
+        assertThat(r2, not(equalTo(r3)));
+        assertThat(r3, not(equalTo(r2)));
+        assertThat(r1.hashCode(), not(equalTo(r3.hashCode())));
+
+        assertThat(r1, not(equalTo(new Object())));
+        assertThat(r1.equals("foobar"), is(false));
+    }
+
+    @Test
+    public void requireThatAccessorsWork() {
+        DocumentTypeId r1 = new DocumentTypeId(123);
+        assertThat(r1.getId(), equalTo(123));
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/DocumentTypeManagerTestCase.java b/document/src/test/java/com/yahoo/document/DocumentTypeManagerTestCase.java
new file mode 100644
index 00000000000..000c2adbb2f
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/DocumentTypeManagerTestCase.java
@@ -0,0 +1,497 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import com.yahoo.document.annotation.*;
+import com.yahoo.document.datatypes.Array;
+import com.yahoo.document.datatypes.FloatFieldValue;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.document.datatypes.StructuredFieldValue;
+
+import java.util.Iterator;
+
+/**
+ * @author <a href=TODO>Thomas Gundersen</a>
+ */
+public class DocumentTypeManagerTestCase extends junit.framework.TestCase {
+    public DocumentTypeManagerTestCase(String name) {
+        super(name);
+    }
+
+    // Verify that we can register and retrieve fields.
+    public void testRegisterAndGet() {
+        DocumentTypeManager manager = new DocumentTypeManager();
+
+        DocumentType newDocType = new DocumentType("testdoc");
+        newDocType.addField("Fjomp", DataType.INT);
+        newDocType.addHeaderField("Fjols", DataType.STRING);
+
+        manager.registerDocumentType(newDocType);
+
+        DocumentType fetchedDocType = manager.getDocumentType(new DataTypeName("testdoc"));
+
+        Field fetched4 = fetchedDocType.getField("Fjomp");
+
+        assertEquals("Fjomp", fetched4.getName());
+        assertEquals(fetched4.getDataType(), DataType.INT);
+        assertEquals(fetched4.isHeader(), false);
+    }
+
+    public void testBasicTypes() {
+        DocumentTypeManager dtm = new DocumentTypeManager();
+        DataType intType = dtm.getDataType("int");
+        assertSame(DataType.INT, intType);
+
+        DataType stringType = dtm.getDataType("string");
+        assertSame(DataType.STRING, stringType);
+
+        DataType longType = dtm.getDataType("long");
+        assertSame(DataType.LONG, longType);
+
+        DataType doubleType = dtm.getDataType("double");
+        assertSame(DataType.DOUBLE, doubleType);
+    }
+
+    public void testRecursiveRegister() {
+        StructDataType struct = new StructDataType("mystruct");
+        DataType wset1 = DataType.getWeightedSet(DataType.getArray(DataType.INT));
+        DataType wset2 = DataType.getWeightedSet(DataType.getArray(DataType.TAG));
+        struct.addField(new Field("foo", wset1, true));
+        struct.addField(new Field("bar", wset2, false));
+        DataType array = DataType.getArray(struct);
+        DocumentType docType = new DocumentType("mydoc");
+        docType.addField("hmm", array);
+        DocumentType docType2 = new DocumentType("myotherdoc");
+        docType2.addField("myint", DataType.INT);
+        docType2.inherit(docType);
+
+        DocumentTypeManager manager = new DocumentTypeManager();
+        manager.register(docType2);
+
+        assertEquals(struct, manager.getDataType(struct.getId()));
+        assertEquals(struct, manager.getDataType("mystruct"));
+        assertEquals(wset1, manager.getDataType(wset1.getId()));
+        assertEquals(wset2, manager.getDataType(wset2.getId()));
+        assertEquals(array, manager.getDataType(array.getId()));
+        assertEquals(docType, manager.getDataType("mydoc"));
+        assertEquals(docType2, manager.getDataType("myotherdoc"));
+        assertEquals(docType, manager.getDocumentType(new DataTypeName("mydoc")));
+        assertEquals(docType2, manager.getDocumentType(new DataTypeName("myotherdoc")));
+    }
+
+    public void testMultipleDocuments() {
+        DocumentType docType1 = new DocumentType("foo0");
+        docType1.addField("bar", DataType.INT);
+
+        DocumentType docType2 = new DocumentType("foo1");
+        docType2.addField("bar", DataType.STRING);
+
+        DocumentType docType3 = new DocumentType("foo2");
+        docType3.addField("bar", DataType.FLOAT);
+
+        DocumentType docType4 = new DocumentType("foo3");
+        docType4.addField("bar", DataType.RAW);
+
+        DocumentTypeManager manager = new DocumentTypeManager();
+        manager.clear();
+        manager.register(docType1);
+        manager.register(docType2);
+        manager.register(docType3);
+        manager.register(docType4);
+
+        assertSame(docType1, manager.getDocumentType(new DataTypeName("foo0")));
+        assertSame(docType2, manager.getDocumentType(new DataTypeName("foo1")));
+        assertSame(docType3, manager.getDocumentType(new DataTypeName("foo2")));
+        assertSame(docType4, manager.getDocumentType(new DataTypeName("foo3")));
+        
+        assertEquals(manager.getDocumentTypes().size(), 5);
+        assertNotNull(manager.getDocumentTypes().get(new DataTypeName("document")));
+        assertEquals(manager.getDocumentTypes().get(new DataTypeName("foo0")), docType1);
+        assertEquals(manager.getDocumentTypes().get(new DataTypeName("foo0")), new DocumentType("foo0"));
+        assertEquals(manager.getDocumentTypes().get(new DataTypeName("foo1")), docType2);
+    }
+
+    public void testReverseMapOrder() {
+        DocumentTypeManager manager = new DocumentTypeManager();
+
+        DocumentTypeManagerConfigurer.configure(manager, "file:src/test/document/documentmanager.map.cfg");
+
+        assertNotNull(manager.getDataType(1000));
+        assertNotNull(manager.getDataType(1001));
+    }
+
+    @SuppressWarnings("deprecation")
+    public void testConfigure() {
+        DocumentTypeManager manager = new DocumentTypeManager();
+
+        DocumentTypeManagerConfigurer.configure(manager, "file:src/test/document/documentmanager.cfg");
+
+        Iterator typeIt = manager.documentTypeIterator();
+        DocumentType type = null;
+
+        while (typeIt.hasNext()) {
+            type = (DocumentType) typeIt.next();
+            if (type.getName().equals("foobar")) {
+                break;
+            }
+        }
+        assertNotNull(type);
+
+        assertTrue(type.hasField("foobarfield0"));
+        assertTrue(type.hasField("foobarfield1"));
+
+        Field foobarfield0 = type.getField("foobarfield0");
+        assertTrue(!foobarfield0.isHeader());
+        assertTrue(foobarfield0.getDataType().getCode() == 2);
+
+        Field foobarfield1 = type.getField("foobarfield1");
+        assertTrue(foobarfield1.isHeader());
+        assertTrue(foobarfield1.getDataType().getCode() == 4);
+
+
+        typeIt = manager.documentTypeIterator();
+        while (typeIt.hasNext()) {
+            type = (DocumentType) typeIt.next();
+            if (type.getName().equals("banana")) {
+                break;
+            }
+        }
+
+        assertTrue(type.hasField("bananafield0"));
+
+        Field bananafield0 = type.getField("bananafield0");
+        assertTrue(bananafield0.isHeader());
+        assertTrue(bananafield0.getDataType().getCode() == 16);
+
+        //inheritance:
+        Iterator inhIt = type.getInheritedTypes().iterator();
+
+        DocumentType inhType = (DocumentType) inhIt.next();
+        assertTrue(inhType.getName().equals("foobar"));
+
+        assertFalse(inhIt.hasNext());
+
+        typeIt = manager.documentTypeIterator();
+        while (typeIt.hasNext()) {
+            type = (DocumentType) typeIt.next();
+            if (type.getName().equals("customtypes")) {
+                break;
+            }
+        }
+
+        assertTrue(type.hasField("arrayfloat"));
+        assertTrue(type.hasField("arrayarrayfloat"));
+
+        Field arrayfloat = type.getField("arrayfloat");
+        assertTrue(!arrayfloat.isHeader());
+        ArrayDataType dataType = (ArrayDataType) arrayfloat.getDataType();
+        assertTrue(dataType.getCode() == 99);
+        assertTrue(dataType.getValueClass().equals(Array.class));
+        assertTrue(dataType.getNestedType().getCode() == 1);
+        assertTrue(dataType.getNestedType().getValueClass().equals(FloatFieldValue.class));
+
+
+        Field arrayarrayfloat = type.getField("arrayarrayfloat");
+        ArrayDataType subType = (ArrayDataType) arrayarrayfloat.getDataType();
+        assertTrue(!arrayarrayfloat.isHeader());
+        assertTrue(subType.getCode() == 4003);
+        assertTrue(subType.getValueClass().equals(Array.class));
+        assertTrue(subType.getNestedType().getCode() == 99);
+        assertTrue(subType.getNestedType().getValueClass().equals(Array.class));
+        ArrayDataType subSubType = (ArrayDataType) subType.getNestedType();
+        assertTrue(subSubType.getNestedType().getCode() == 1);
+        assertTrue(subSubType.getNestedType().getValueClass().equals(FloatFieldValue.class));
+    }
+
+    public void testConfigureUpdate() {
+        DocumentTypeManager manager = new DocumentTypeManager();
+
+        DocumentTypeManagerConfigurer.configure(manager, "file:src/test/document/documentmanager.cfg");
+
+        DocumentType banana = manager.getDocumentType(new DataTypeName("banana"));
+        DocumentType customtypes = manager.getDocumentType(new DataTypeName("customtypes"));
+
+        assertNull(banana.getField("newfield"));
+        assertEquals(new Field("arrayfloat", 9489, new ArrayDataType(DataType.FLOAT, 99), false), customtypes.getField("arrayfloat"));
+
+        DocumentTypeManagerConfigurer.configure(manager, "file:src/test/document/documentmanager.updated.cfg");
+
+        banana = manager.getDocumentType(new DataTypeName("banana"));
+        customtypes = manager.getDocumentType(new DataTypeName("customtypes"));
+
+        assertEquals(new Field("newfield", 12345, DataType.STRING, true), banana.getField("newfield"));
+        assertNull(customtypes.getField("arrayfloat"));
+    }
+
+    public void testConfigureWithAnnotations() {
+        DocumentTypeManager manager = new DocumentTypeManager();
+
+        DocumentTypeManagerConfigurer.configure(manager, "file:src/test/document/documentmanager.annotationtypes1.cfg");
+
+        /*
+  annotation banana {
+    field brand type string { }
+  }
+
+  annotation food {
+    field what type annotationreference<banana> { }
+  }
+
+  annotation cyclic {
+    field blah type annotationreference<cyclic> { }
+  }
+
+  annotation a {
+    field foo type annotationreference<b> { }
+  }
+
+  annotation b {
+  }
+         */
+
+        AnnotationTypeRegistry atr = manager.getAnnotationTypeRegistry();
+        assertEquals(AnnotationTypes.ALL_TYPES.size() + 5, atr.getTypes().size());
+
+        AnnotationType banana = atr.getType("banana");
+        assertTrue(banana.getDataType() instanceof StructDataType);
+        assertEquals("annotation.banana", banana.getDataType().getName());
+        assertEquals(1, ((StructDataType) banana.getDataType()).getFields().size());
+        assertEquals(DataType.STRING, ((StructDataType) banana.getDataType()).getField("brand").getDataType());
+
+        AnnotationType food = atr.getType("food");
+        assertTrue(food.getDataType() instanceof StructDataType);
+        assertEquals("annotation.food", food.getDataType().getName());
+        assertEquals(1, ((StructDataType) food.getDataType()).getFields().size());
+        AnnotationReferenceDataType what = (AnnotationReferenceDataType) ((StructDataType) food.getDataType()).getField("what").getDataType();
+        assertSame(banana, what.getAnnotationType());
+
+        AnnotationType cyclic = atr.getType("cyclic");
+        assertTrue(cyclic.getDataType() instanceof StructDataType);
+        assertEquals("annotation.cyclic", cyclic.getDataType().getName());
+        assertEquals(1, ((StructDataType) cyclic.getDataType()).getFields().size());
+        AnnotationReferenceDataType blah = (AnnotationReferenceDataType) ((StructDataType) cyclic.getDataType()).getField("blah").getDataType();
+        assertSame(cyclic, blah.getAnnotationType());
+
+        AnnotationType b = atr.getType("b");
+        assertNull(b.getDataType());
+
+        AnnotationType a = atr.getType("a");
+        assertTrue(a.getDataType() instanceof StructDataType);
+        assertEquals("annotation.a", a.getDataType().getName());
+        assertEquals(1, ((StructDataType) a.getDataType()).getFields().size());
+        AnnotationReferenceDataType foo = (AnnotationReferenceDataType) ((StructDataType) a.getDataType()).getField("foo").getDataType();
+        assertSame(b, foo.getAnnotationType());
+
+    }
+
+    public void testConfigureWithAnnotationsWithInheritance() {
+        DocumentTypeManager manager = new DocumentTypeManager();
+
+        DocumentTypeManagerConfigurer.configure(manager, "file:src/test/document/documentmanager.annotationtypes2.cfg");
+
+
+        /*
+  annotation fruit {
+  }
+
+  annotation banana inherits fruit {
+    field brand type string { }
+  }
+
+  annotation vehicle {
+    field numwheels type int { }
+  }
+
+  annotation car inherits vehicle {
+    field color type string { }
+  }
+
+  annotation intern inherits employee {
+    field enddate type long { }
+  }
+
+  annotation employee inherits worker {
+    field employeeid type int { }
+  }
+
+  annotation worker inherits person {
+  }
+
+  annotation person {
+    field name type string { }
+  }
+         */
+
+        AnnotationTypeRegistry atr = manager.getAnnotationTypeRegistry();
+        assertEquals(AnnotationTypes.ALL_TYPES.size() + 8, atr.getTypes().size());
+
+        AnnotationType fruit = atr.getType("fruit");
+        assertNull(fruit.getDataType());
+
+        AnnotationType banana = atr.getType("banana");
+        assertTrue(banana.getDataType() instanceof StructDataType);
+        assertEquals("annotation.banana", banana.getDataType().getName());
+        assertEquals(1, ((StructDataType) banana.getDataType()).getFields().size());
+        assertEquals(DataType.STRING, ((StructDataType) banana.getDataType()).getField("brand").getDataType());
+        assertEquals(0, ((StructDataType) banana.getDataType()).getInheritedTypes().size());
+
+        AnnotationType vehicle = atr.getType("vehicle");
+        assertTrue(vehicle.getDataType() instanceof StructDataType);
+        assertEquals("annotation.vehicle", vehicle.getDataType().getName());
+        assertEquals(1, ((StructDataType) vehicle.getDataType()).getFields().size());
+        assertEquals(DataType.INT, ((StructDataType) vehicle.getDataType()).getField("numwheels").getDataType());
+        assertEquals(0, ((StructDataType) vehicle.getDataType()).getInheritedTypes().size());
+
+        AnnotationType car = atr.getType("car");
+        assertTrue(car.getDataType() instanceof StructDataType);
+        assertEquals("annotation.car", car.getDataType().getName());
+        assertEquals(2, ((StructDataType) car.getDataType()).getFields().size());
+        assertEquals(DataType.INT, ((StructDataType) car.getDataType()).getField("numwheels").getDataType());
+        assertEquals(DataType.STRING, ((StructDataType) car.getDataType()).getField("color").getDataType());
+        assertEquals(1, ((StructDataType) car.getDataType()).getInheritedTypes().size());
+        assertSame(vehicle.getDataType(), ((StructDataType) car.getDataType()).getInheritedTypes().toArray()[0]);
+
+        AnnotationType person = atr.getType("person");
+        assertTrue(person.getDataType() instanceof StructDataType);
+        assertEquals("annotation.person", person.getDataType().getName());
+        assertEquals(1, ((StructDataType) person.getDataType()).getFields().size());
+        assertEquals(DataType.STRING, ((StructDataType) person.getDataType()).getField("name").getDataType());
+        assertEquals(0, ((StructDataType) person.getDataType()).getInheritedTypes().size());
+
+        AnnotationType worker = atr.getType("worker");
+        assertTrue(worker.getDataType() instanceof StructDataType);
+        assertEquals("annotation.worker", worker.getDataType().getName());
+        assertEquals(1, ((StructDataType) worker.getDataType()).getFields().size());
+        assertEquals(DataType.STRING, ((StructDataType) worker.getDataType()).getField("name").getDataType());
+        assertEquals(1, ((StructDataType) worker.getDataType()).getInheritedTypes().size());
+        assertSame(person.getDataType(), ((StructDataType) worker.getDataType()).getInheritedTypes().toArray()[0]);
+
+        AnnotationType employee = atr.getType("employee");
+        assertTrue(employee.getDataType() instanceof StructDataType);
+        assertEquals("annotation.employee", employee.getDataType().getName());
+        assertEquals(2, ((StructDataType) employee.getDataType()).getFields().size());
+        assertEquals(DataType.STRING, ((StructDataType) employee.getDataType()).getField("name").getDataType());
+        assertEquals(DataType.INT, ((StructDataType) employee.getDataType()).getField("employeeid").getDataType());
+        assertEquals(1, ((StructDataType) employee.getDataType()).getInheritedTypes().size());
+        assertSame(worker.getDataType(), ((StructDataType) employee.getDataType()).getInheritedTypes().toArray()[0]);
+
+        AnnotationType intern = atr.getType("intern");
+        assertTrue(intern.getDataType() instanceof StructDataType);
+        assertEquals("annotation.intern", intern.getDataType().getName());
+        assertEquals(3, ((StructDataType) intern.getDataType()).getFields().size());
+        assertEquals(DataType.STRING, ((StructDataType) intern.getDataType()).getField("name").getDataType());
+        assertEquals(DataType.INT, ((StructDataType) intern.getDataType()).getField("employeeid").getDataType());
+        assertEquals(DataType.LONG, ((StructDataType) intern.getDataType()).getField("enddate").getDataType());
+        assertEquals(1, ((StructDataType) intern.getDataType()).getInheritedTypes().size());
+        assertSame(employee.getDataType(), ((StructDataType) intern.getDataType()).getInheritedTypes().toArray()[0]);
+
+    }
+
+    public void testStructsAnyOrder() {
+        /*
+search annotationsimplicitstruct {
+
+  document annotationsimplicitstruct {
+      field structfield type sct {
+      }
+
+      field structarrayfield type array<sct> {
+      }
+  }
+
+  struct sct {
+    field s1 type string {}
+    field s2 type string {}
+    field s3 type sct {}
+    field s4 type foo {}
+  }
+
+  struct foo {
+    field s1 type int {}
+  }
+}
+         */
+
+        DocumentTypeManager manager = new DocumentTypeManager();
+
+        DocumentTypeManagerConfigurer.configure(manager, "file:src/test/document/documentmanager.structsanyorder.cfg");
+
+        StructDataType foo = (StructDataType) manager.getDataType("foo");
+        assertNotNull(foo);
+        assertEquals(1, foo.getFields().size());
+        Field foos1 = foo.getField("s1");
+        assertSame(DataType.INT, foos1.getDataType());
+
+        StructDataType sct = (StructDataType) manager.getDataType("sct");
+        assertNotNull(sct);
+        assertEquals(4, sct.getFields().size());
+        Field s1 = sct.getField("s1");
+        assertSame(DataType.STRING, s1.getDataType());
+        Field s2 = sct.getField("s2");
+        assertSame(DataType.STRING, s2.getDataType());
+        Field s3 = sct.getField("s3");
+        assertSame(sct, s3.getDataType());
+        Field s4 = sct.getField("s4");
+        assertSame(foo, s4.getDataType());
+    }
+
+    public void testSombrero1() {
+        DocumentTypeManager manager = new DocumentTypeManager();
+        DocumentTypeManagerConfigurer.configure(manager, "file:src/test/document/documentmanager.sombrero1.cfg");
+
+        {
+            StringFieldValue sfv = new StringFieldValue("ballooooo");
+            String text = sfv.getString();
+            AnnotationType type = manager.getAnnotationTypeRegistry().getType("base");
+            StructuredFieldValue value = (StructuredFieldValue) type.getDataType().createFieldValue();
+            value.setFieldValue("x", 10);
+            SpanNode span = new Span(0, text.length());
+            SpanTree tree = new SpanTree("span", span);
+            tree.annotate(span, new Annotation(type, value));
+            sfv.setSpanTree(tree);
+        }
+
+        {
+            StringFieldValue sfv = new StringFieldValue("ballooooo");
+            String text = sfv.getString();
+            AnnotationType type = manager.getAnnotationTypeRegistry().getType("derived");
+            StructuredFieldValue value = (StructuredFieldValue) type.getDataType().createFieldValue();
+            value.setFieldValue("x", 10);
+            SpanNode span = new Span(0, text.length());
+            SpanTree tree = new SpanTree("span", span);
+            tree.annotate(span, new Annotation(type, value));
+            sfv.setSpanTree(tree);
+        }
+    }
+
+    public void testPolymorphy() {
+        /*
+  annotation super {
+  }
+  annotation sub inherits super {
+  }
+  annotation blah {
+    field a type annotationreference<super> {}
+  }
+         */
+
+
+        DocumentTypeManager manager = new DocumentTypeManager();
+
+        DocumentTypeManagerConfigurer.configure(manager, "file:src/test/document/documentmanager.annotationspolymorphy.cfg");
+
+        AnnotationType suuper = manager.getAnnotationTypeRegistry().getType("super");
+        AnnotationType sub = manager.getAnnotationTypeRegistry().getType("sub");
+
+
+        //reference type for super annotation type
+        AnnotationReferenceDataType refType = (AnnotationReferenceDataType) ((StructDataType) manager.getAnnotationTypeRegistry().getType("blah").getDataType()).getField("a").getDataType();
+
+        Annotation superAnnotation = new Annotation(suuper);
+        Annotation subAnnotation = new Annotation(sub);
+
+        new AnnotationReference(refType, superAnnotation);
+        //this would fail without polymorphy support:
+        new AnnotationReference(refType, subAnnotation);
+
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/DocumentTypeTestCase.java b/document/src/test/java/com/yahoo/document/DocumentTypeTestCase.java
new file mode 100644
index 00000000000..7712cc06c41
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/DocumentTypeTestCase.java
@@ -0,0 +1,107 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import java.util.Iterator;
+
+/**
+ * TODO: Document purpose
+ *
+ * @author <a href="thomasg@yahoo-inc.com>Thomas Gundersen</a>
+ * @author <a href="bratseth@yahoo-inc.com>Jon S Bratseth</a>
+ */
+public class DocumentTypeTestCase extends junit.framework.TestCase {
+
+    public DocumentTypeTestCase(String name) {
+        super(name);
+    }
+
+    // Verify that we can register and retrieve fields.
+    public void testSetGet() {
+        DocumentType docType = new DocumentType("testdoc");
+        docType.addField("Bongle", DataType.STRING);
+        docType.addField("nalle", DataType.INT);
+
+        assertEquals(docType.getField("Bongle").getName(), "Bongle");
+        assertNull(docType.getField("bongle"));
+
+    }
+
+    public void testInheritance() {
+        DocumentTypeManager typeManager = new DocumentTypeManager();
+
+        DocumentType child = new DocumentType("child");
+        //typeManager.register(child);
+
+        /*
+        ListIterator inherited = child.inheritedIterator();
+        assertTrue(inherited.hasNext());
+        assertEquals(new DataTypeName("document", 0), inherited.next());
+        assertTrue(!inherited.hasNext());
+        */
+        Iterator inherited;
+
+        child.addField("childfield", DataType.INT);
+        child.addField("overridden", DataType.STRING);
+
+        DocumentType parent1 = new DocumentType("parent1");
+        parent1.addField("overridden", DataType.STRING);
+        parent1.addField("parent1field", DataType.STRING);
+        child.inherit(parent1);
+
+        DocumentType parent2 = new DocumentType("parent2");
+        parent2.addField("parent2field", DataType.STRING);
+        child.inherit(parent2);
+
+        DocumentType root = new DocumentType("root");
+        root.addField("rootfield", DataType.STRING);
+        parent1.inherit(root);
+        parent2.inherit(root);
+
+        typeManager.register(root);
+        typeManager.register(parent1);
+        typeManager.register(parent2);
+        typeManager.register(child);
+
+        inherited = child.getInheritedTypes().iterator();
+        assertEquals(parent1, inherited.next());
+        assertEquals(parent2, inherited.next());
+        assertTrue(!inherited.hasNext());
+
+        inherited = parent1.getInheritedTypes().iterator();
+        assertEquals(root, inherited.next());
+        assertTrue(!inherited.hasNext());
+
+        inherited = parent2.getInheritedTypes().iterator();
+        assertEquals(root, inherited.next());
+        assertTrue(!inherited.hasNext());
+
+        inherited = root.getInheritedTypes().iterator();
+        assertEquals(DataType.DOCUMENT, inherited.next());
+        assertTrue(!inherited.hasNext());
+
+        Iterator fields = child.fieldSet().iterator();
+        Field field;
+
+        field = (Field) fields.next();
+        assertEquals("rootfield", field.getName());
+
+        field = (Field) fields.next();
+        assertEquals("overridden", field.getName());
+        assertEquals(DataType.STRING, field.getDataType());
+
+        field = (Field) fields.next();
+        assertEquals("parent1field", field.getName());
+
+        field = (Field) fields.next();
+        assertEquals("parent2field", field.getName());
+
+        field = (Field) fields.next();
+        assertEquals("childfield", field.getName());
+
+        assertFalse(fields.hasNext());
+
+        assert(child.getField("rootfield") != null);
+
+        // TODO: Test uninheriting
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/DocumentUpdateTestCase.java b/document/src/test/java/com/yahoo/document/DocumentUpdateTestCase.java
new file mode 100644
index 00000000000..0e7313991f8
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/DocumentUpdateTestCase.java
@@ -0,0 +1,577 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import com.yahoo.document.datatypes.*;
+import com.yahoo.document.fieldpathupdate.FieldPathUpdate;
+import com.yahoo.document.select.parser.ParseException;
+import com.yahoo.document.serialization.*;
+import com.yahoo.document.update.AssignValueUpdate;
+import com.yahoo.document.update.FieldUpdate;
+import com.yahoo.document.update.ValueUpdate;
+import com.yahoo.io.GrowableByteBuffer;
+
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+
+/**
+ * Tests applying and serializing document updates.
+ *
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class DocumentUpdateTestCase extends junit.framework.TestCase {
+    DocumentTypeManager docMan;
+
+    DocumentType docType = null;
+    DocumentType docType2 = null;
+    DocumentUpdate docUp = null;
+
+    FieldUpdate assignSingle = null;
+    FieldUpdate assignMultiList = null;
+    FieldUpdate assignMultiWset = null;
+
+    FieldUpdate clearSingleField = null;
+    FieldUpdate removeMultiList = null;
+    FieldUpdate removeMultiWset = null;
+
+    FieldUpdate addSingle = null;
+    FieldUpdate addMultiList = null;
+    FieldUpdate addMultiWset = null;
+
+    public void setUp() {
+        docMan = new DocumentTypeManager();
+
+        docType = new DocumentType("foobar");
+        docType.addField(new Field("strfoo", DataType.STRING));
+
+        DataType stringarray = DataType.getArray(DataType.STRING);
+        docType.addField(new Field("strarray", stringarray));
+
+        DataType stringwset = DataType.getWeightedSet(DataType.STRING);
+        docType.addField(new Field("strwset", stringwset));
+        docMan.register(docType);
+
+        docType2 = new DocumentType("otherdoctype");
+        docType2.addField(new Field("strinother", DataType.STRING));
+        docMan.register(docType2);
+
+        docUp = new DocumentUpdate(docType, new DocumentId("doc:foo:bar"));
+
+        assignSingle = FieldUpdate.createAssign(docType.getField("strfoo"), new StringFieldValue("something"));
+
+        Array<StringFieldValue> assignList = new Array<StringFieldValue>(docType.getField("strarray").getDataType());
+        assignList.add(new StringFieldValue("assigned val 0"));
+        assignList.add(new StringFieldValue("assigned val 1"));
+        assignMultiList = FieldUpdate.createAssign(docType.getField("strarray"), assignList);
+
+        WeightedSet<StringFieldValue> assignWset = new WeightedSet<>(docType.getField("strwset").getDataType());
+        assignWset.put(new StringFieldValue("assigned val 0"), 5);
+        assignWset.put(new StringFieldValue("assigned val 1"), 10);
+        assignMultiWset = FieldUpdate.createAssign(docType.getField("strwset"), assignWset);
+
+        clearSingleField = FieldUpdate.createClearField(docType.getField("strfoo"));
+
+        removeMultiList = FieldUpdate.createRemove(docType.getField("strarray"), new StringFieldValue("remove val 1"));
+
+        removeMultiWset = FieldUpdate.createRemove(docType.getField("strwset"), new StringFieldValue("remove val 1"));
+
+        try {
+            addSingle = FieldUpdate.createAdd(docType.getField("strfoo"), new StringFieldValue("add val"));
+            fail("Shouldn't be able to add to a single-value field");
+        } catch (UnsupportedOperationException uoe) {
+            //success
+        }
+
+        List<StringFieldValue> addList = new ArrayList<>();
+        addList.add(new StringFieldValue("bo"));
+        addList.add(new StringFieldValue("ba"));
+        addList.add(new StringFieldValue("by"));
+        addMultiList = FieldUpdate.createAddAll(docType.getField("strarray"), addList);
+
+        WeightedSet<StringFieldValue> addSet = new WeightedSet<StringFieldValue>(docType.getField("strwset").getDataType());
+        addSet.put(new StringFieldValue("banana"), 137);
+        addSet.put(new StringFieldValue("is"), 143);
+        addSet.put(new StringFieldValue("a"), 157);
+        addSet.put(new StringFieldValue("great"), 163);
+        addSet.put(new StringFieldValue("fruit"), 189);
+        addMultiWset = FieldUpdate.createAddAll(docType.getField("strwset"), addSet);
+    }
+
+    public void testApplyRemoveSingle() {
+        Document doc = new Document(docMan.getDocumentType("foobar"), new DocumentId("doc:something:foooo"));
+        assertNull(doc.getFieldValue("strfoo"));
+        doc.setFieldValue("strfoo", new StringFieldValue("cocacola"));
+        assertEquals(new StringFieldValue("cocacola"), doc.getFieldValue("strfoo"));
+        docUp.addFieldUpdate(clearSingleField);
+        docUp.applyTo(doc);
+        assertNull(doc.getFieldValue("strfoo"));
+    }
+
+    public void testApplyRemoveMultiList() {
+        Document doc = new Document(docMan.getDocumentType("foobar"), new DocumentId("doc:something:foooo"));
+        assertNull(doc.getFieldValue("strarray"));
+        Array<StringFieldValue> strArray = new Array<>(DataType.getArray(DataType.STRING));
+        strArray.add(new StringFieldValue("hello hello"));
+        strArray.add(new StringFieldValue("remove val 1"));
+        doc.setFieldValue("strarray", strArray);
+        assertNotNull(doc.getFieldValue("strarray"));
+        docUp.addFieldUpdate(removeMultiList);
+        docUp.applyTo(doc);
+        assertEquals(1, ((List)doc.getFieldValue("strarray")).size());
+        List docList = (List)doc.getFieldValue("strarray");
+        assertEquals(new StringFieldValue("hello hello"), docList.get(0));
+    }
+
+    public void testApplyRemoveMultiWset() {
+        Document doc = new Document(docMan.getDocumentType("foobar"), new DocumentId("doc:something:foooo"));
+        assertNull(doc.getFieldValue("strwset"));
+        WeightedSet<StringFieldValue> strwset = new WeightedSet<>(doc.getDataType().getField("strwset").getDataType());
+        strwset.put(new StringFieldValue("hello hello"), 10);
+        strwset.put(new StringFieldValue("remove val 1"), 20);
+        doc.setFieldValue("strwset", strwset);
+        assertNotNull(doc.getFieldValue("strwset"));
+        docUp.addFieldUpdate(removeMultiWset);
+        docUp.applyTo(doc);
+        assertEquals(1, ((WeightedSet)doc.getFieldValue("strwset")).size());
+        WeightedSet docWset = (WeightedSet)doc.getFieldValue("strwset");
+        assertEquals(new Integer(10), docWset.get(new StringFieldValue("hello hello")));
+    }
+
+    public void testApplyAssignSingle() {
+        Document doc = new Document(docMan.getDocumentType("foobar"), new DocumentId("doc:something:foooo"));
+        assertNull(doc.getFieldValue("strfoo"));
+        docUp.addFieldUpdate(assignSingle);
+        docUp.applyTo(doc);
+        assertEquals(new StringFieldValue("something"), doc.getFieldValue("strfoo"));
+    }
+
+    public void testApplyAssignMultiList() {
+        Document doc = new Document(docMan.getDocumentType("foobar"), new DocumentId("doc:something:foooo"));
+        assertNull(doc.getFieldValue("strarray"));
+        Array<StringFieldValue> strArray = new Array<>(DataType.getArray(DataType.STRING));
+        strArray.add(new StringFieldValue("hello hello"));
+        strArray.add(new StringFieldValue("blah blah"));
+        doc.setFieldValue("strarray", strArray);
+        assertNotNull(doc.getFieldValue("strarray"));
+        docUp.addFieldUpdate(assignMultiList);
+        docUp.applyTo(doc);
+        assertEquals(2, ((List)doc.getFieldValue("strarray")).size());
+        List docList = (List)doc.getFieldValue("strarray");
+        assertEquals(new StringFieldValue("assigned val 0"), docList.get(0));
+        assertEquals(new StringFieldValue("assigned val 1"), docList.get(1));
+    }
+
+    public void testApplyAssignMultiWlist() {
+        Document doc = new Document(docMan.getDocumentType("foobar"), new DocumentId("doc:something:foooo"));
+        assertNull(doc.getFieldValue("strwset"));
+        WeightedSet<StringFieldValue> strwset = new WeightedSet<>(doc.getDataType().getField("strwset").getDataType());
+        strwset.put(new StringFieldValue("hello hello"), 164);
+        strwset.put(new StringFieldValue("blahdi blahdi"), 243);
+        doc.setFieldValue("strwset", strwset);
+        assertNotNull(doc.getFieldValue("strwset"));
+        docUp.addFieldUpdate(assignMultiWset);
+        docUp.applyTo(doc);
+        assertEquals(2, ((WeightedSet)doc.getFieldValue("strwset")).size());
+        WeightedSet docWset = (WeightedSet)doc.getFieldValue("strwset");
+        assertEquals(new Integer(5), docWset.get(new StringFieldValue("assigned val 0")));
+        assertEquals(new Integer(10), docWset.get(new StringFieldValue("assigned val 1")));
+    }
+
+    public void testApplyAddSingle() {
+        //Unneeded, we can't construct an add for single-value
+    }
+
+    public void testApplyAddMultiList() {
+        Document doc = new Document(docMan.getDocumentType("foobar"), new DocumentId("doc:something:foooo"));
+        assertNull(doc.getFieldValue("strarray"));
+
+        docUp.addFieldUpdate(addMultiList);
+        docUp.applyTo(doc);
+        List<StringFieldValue> values = new ArrayList<>();
+        values.add(new StringFieldValue("bo"));
+        values.add(new StringFieldValue("ba"));
+        values.add(new StringFieldValue("by"));
+        assertEquals(values, doc.getFieldValue("strarray"));
+    }
+
+    public void testApplyAddMultiWset() {
+        Document doc = new Document(docMan.getDocumentType("foobar"), new DocumentId("doc:something:foooo"));
+        assertNull(doc.getFieldValue("strwset"));
+
+        WeightedSet<StringFieldValue> wset = new WeightedSet<>(doc.getDataType().getField("strwset").getDataType());
+        wset.put(new StringFieldValue("this"), 5);
+        wset.put(new StringFieldValue("is"), 10);
+        wset.put(new StringFieldValue("a"), 15);
+        wset.put(new StringFieldValue("test"), 20);
+        doc.setFieldValue("strwset", wset);
+
+        docUp.addFieldUpdate(addMultiWset);
+        docUp.applyTo(doc);
+
+        WeightedSet<StringFieldValue> values = new WeightedSet<>(doc.getDataType().getField("strwset").getDataType());
+        values.put(new StringFieldValue("this"), 5);
+        values.put(new StringFieldValue("is"), 143);
+        values.put(new StringFieldValue("a"), 157);
+        values.put(new StringFieldValue("test"), 20);
+        values.put(new StringFieldValue("banana"), 137);
+        values.put(new StringFieldValue("great"), 163);
+        values.put(new StringFieldValue("fruit"), 189);
+        assertEquals(values, doc.getFieldValue("strwset"));
+    }
+
+    public void testUpdatesToTheSameFieldAreCombining() {
+        DocumentType docType = new DocumentType("my_type");
+        Field field = new Field("my_int", DataType.INT);
+        docType.addField(field);
+
+        DocumentUpdate update = new DocumentUpdate(docType, new DocumentId("doc:foo:"));
+        update.addFieldUpdate(FieldUpdate.createAssign(field, new IntegerFieldValue(1)));
+        update.addFieldUpdate(FieldUpdate.createAssign(field, new IntegerFieldValue(2)));
+
+        assertEquals(1, update.getFieldUpdates().size());
+        FieldUpdate fieldUpdate = update.getFieldUpdate(0);
+        assertNotNull(fieldUpdate);
+        assertEquals(field, fieldUpdate.getField());
+        assertEquals(2, fieldUpdate.getValueUpdates().size());
+        ValueUpdate valueUpdate = fieldUpdate.getValueUpdate(0);
+        assertNotNull(valueUpdate);
+        assertTrue(valueUpdate instanceof AssignValueUpdate);
+        assertEquals(new IntegerFieldValue(1), valueUpdate.getValue());
+        assertNotNull(valueUpdate = fieldUpdate.getValueUpdate(1));
+        assertTrue(valueUpdate instanceof AssignValueUpdate);
+        assertEquals(new IntegerFieldValue(2), valueUpdate.getValue());
+    }
+
+    public void testUpdateToWrongField() {
+        DocumentType docType = new DocumentType("my_type");
+        DocumentUpdate update = new DocumentUpdate(docType, new DocumentId("doc:foo:"));
+        try {
+            update.addFieldUpdate(FieldUpdate.createIncrement(new Field("my_int", DataType.INT), 1));
+            fail();
+        } catch (IllegalArgumentException e) {
+            System.out.println(e.getMessage());
+        }
+    }
+
+    public void testSerialize() {
+        docUp.addFieldUpdate(assignSingle);
+        docUp.addFieldUpdate(addMultiList);
+
+        DocumentDeserializer buf = DocumentDeserializerFactory.create42(docMan, new GrowableByteBuffer());
+        docUp.serialize((DocumentUpdateWriter)buf);
+        buf.getBuf().flip();
+
+        try {
+            FileOutputStream fos = new FileOutputStream("src/test/files/updateser.dat");
+            fos.write(buf.getBuf().array(), 0, buf.getBuf().remaining());
+            fos.close();
+        } catch (Exception e) {
+        }
+
+        assertEquals(2 // version
+                     + (11 + 1) //docid doc:foo:bar\0
+                     + 1 //contents
+                     + (6 + 1 + 2) //doctype foobar\0\0\0
+                     + 4 //num field updates
+
+                     //field update 1:
+                     + (4 //field id
+                        + 4 //num valueupdates
+
+                        + (4 //valueUpdateClassID
+                           + 1 //valuepresent
+                           + (1 + 1 + 9 + 1)) //value
+
+                        //field update 2
+                        + (4 //field id
+                           + 4 //num valueupdates
+
+                           + (4  //valueUpdateClassID
+                              + (4 + 4 + 4 + (1 + 1 + 2 + 1) + 4 + (1 + 1 + 2 + 1) + 4 + (1 + 1 + 2 + 1))))) //value
+                , buf.getBuf().remaining());
+
+        DocumentUpdate docUpDeser = new DocumentUpdate(buf);
+
+        assertEquals(docUp.getDocumentType(), docUpDeser.getDocumentType());
+        assertEquals(docUp, docUpDeser);
+    }
+
+    public void testCppDocUpd() throws IOException {
+        docMan = DocumentTestCase.setUpCppDocType();
+        byte[] data = DocumentTestCase.readFile("src/tests/data/serializeupdatecpp.dat");
+        DocumentDeserializer buf = DocumentDeserializerFactory.create42(docMan, GrowableByteBuffer.wrap(data));
+
+        DocumentType type = docMan.getDocumentType("serializetest");
+
+        DocumentUpdate upd = new DocumentUpdate(buf);
+
+        assertEquals(new DocumentId("doc:update:test"), upd.getId());
+        assertEquals(type, upd.getType());
+
+        FieldUpdate serAssignFU = upd.getFieldUpdate(0);
+        assertEquals(type.getField("intfield"), serAssignFU.getField());
+        ValueUpdate serAssign = serAssignFU.getValueUpdate(0);
+        assertEquals(ValueUpdate.ValueUpdateClassID.ASSIGN, serAssign.getValueUpdateClassID());
+        assertEquals(new IntegerFieldValue(4), serAssign.getValue());
+
+        FieldUpdate serAddFU = upd.getFieldUpdate(2);
+        assertEquals(type.getField("arrayoffloatfield"), serAddFU.getField());
+        ValueUpdate serAdd1 = serAddFU.getValueUpdate(0);
+        assertEquals(ValueUpdate.ValueUpdateClassID.ADD, serAdd1.getValueUpdateClassID());
+        FloatFieldValue addParam1 = (FloatFieldValue)serAdd1.getValue();
+        assertEquals(new FloatFieldValue(5.00f), addParam1);
+        ValueUpdate serAdd2 = serAddFU.getValueUpdate(1);
+        assertEquals(ValueUpdate.ValueUpdateClassID.ADD, serAdd2.getValueUpdateClassID());
+        FloatFieldValue addparam2 = (FloatFieldValue)serAdd2.getValue();
+        assertEquals(new FloatFieldValue(4.23f), addparam2);
+        ValueUpdate serAdd3 = serAddFU.getValueUpdate(2);
+        assertEquals(ValueUpdate.ValueUpdateClassID.ADD, serAdd3.getValueUpdateClassID());
+        FloatFieldValue addparam3 = (FloatFieldValue)serAdd3.getValue();
+        assertEquals(new FloatFieldValue(-1.00f), addparam3);
+
+        FieldUpdate arithFU = upd.getFieldUpdate(3);
+        assertEquals(type.getField("intfield"), serAssignFU.getField());
+        ValueUpdate serArith = arithFU.getValueUpdate(0);
+        assertEquals(ValueUpdate.ValueUpdateClassID.ARITHMETIC, serArith.getValueUpdateClassID());
+
+        FieldUpdate wsetFU = upd.getFieldUpdate(4);
+        assertEquals(type.getField("wsfield"), wsetFU.getField());
+        assertEquals(2, wsetFU.size());
+        ValueUpdate mapUpd = wsetFU.getValueUpdate(0);
+        assertEquals(ValueUpdate.ValueUpdateClassID.MAP, mapUpd.getValueUpdateClassID());
+        mapUpd = wsetFU.getValueUpdate(1);
+        assertEquals(ValueUpdate.ValueUpdateClassID.MAP, mapUpd.getValueUpdateClassID());
+    }
+
+    public void testGenerateSerializedFile() throws IOException {
+        docMan = DocumentTestCase.setUpCppDocType();
+
+        DocumentType type = docMan.getDocumentType("serializetest");
+        DocumentUpdate upd = new DocumentUpdate(type, new DocumentId("doc:update:test"));
+        FieldUpdate serAssign = FieldUpdate.createAssign(type.getField("intfield"), new IntegerFieldValue(4));
+        upd.addFieldUpdate(serAssign);
+        FieldUpdate serClearField = FieldUpdate.createClearField(type.getField("floatfield"));
+        upd.addFieldUpdate(serClearField);
+        List<FloatFieldValue> arrayOfFloat = new ArrayList<>();
+        arrayOfFloat.add(new FloatFieldValue(5.00f));
+        arrayOfFloat.add(new FloatFieldValue(4.23f));
+        arrayOfFloat.add(new FloatFieldValue(-1.00f));
+        FieldUpdate serAdd = FieldUpdate.createAddAll(type.getField("arrayoffloatfield"), arrayOfFloat);
+        upd.addFieldUpdate(serAdd);
+
+        GrowableByteBuffer buf = new GrowableByteBuffer(100, 2.0f);
+        upd.serialize(buf);
+
+        int size = buf.position();
+        buf.position(0);
+
+        FileOutputStream fos = new FileOutputStream("src/tests/data/serializeupdatejava.dat");
+        fos.write(buf.array(), 0, size);
+        fos.close();
+    }
+
+    public void testRequireThatAddAllCombinesFieldUpdates() {
+        DocumentType docType = new DocumentType("my_type");
+        Field field = new Field("my_int", DataType.INT);
+        docType.addField(field);
+
+        FieldUpdate fooField = FieldUpdate.createAssign(field, new IntegerFieldValue(1));
+        DocumentUpdate fooUpdate = new DocumentUpdate(docType, new DocumentId("doc:foo:"));
+        fooUpdate.addFieldUpdate(fooField);
+
+        FieldUpdate barField = FieldUpdate.createAssign(field, new IntegerFieldValue(2));
+        DocumentUpdate barUpdate = new DocumentUpdate(docType, new DocumentId("doc:foo:"));
+        barUpdate.addFieldUpdate(barField);
+
+        fooUpdate.addAll(barUpdate);
+        assertEquals(1, fooUpdate.getFieldUpdates().size());
+        FieldUpdate fieldUpdate = fooUpdate.getFieldUpdate(0);
+        assertNotNull(fieldUpdate);
+        assertEquals(field, fieldUpdate.getField());
+        assertEquals(2, fieldUpdate.getValueUpdates().size());
+        ValueUpdate valueUpdate = fieldUpdate.getValueUpdate(0);
+        assertNotNull(valueUpdate);
+        assertTrue(valueUpdate instanceof AssignValueUpdate);
+        assertEquals(new IntegerFieldValue(1), valueUpdate.getValue());
+        assertNotNull(valueUpdate = fieldUpdate.getValueUpdate(1));
+        assertTrue(valueUpdate instanceof AssignValueUpdate);
+        assertEquals(new IntegerFieldValue(2), valueUpdate.getValue());
+    }
+
+    public void testInstantiationAndEqualsHashCode() {
+        DocumentType type = new DocumentType("doo");
+        DocumentUpdate d1 = new DocumentUpdate(type, new DocumentId("doc:this:is:a:test"));
+        DocumentUpdate d2 = new DocumentUpdate(type, "doc:this:is:a:test");
+
+        assertEquals(d1, d2);
+        assertEquals(d1, d1);
+        assertEquals(d2, d1);
+        assertEquals(d2, d2);
+        assertFalse(d1.equals(new Object()));
+        assertEquals(d1.hashCode(), d2.hashCode());
+    }
+
+    public void testThatApplyingToWrongTypeFails() {
+        DocumentType t1 = new DocumentType("doo");
+        DocumentUpdate documentUpdate = new DocumentUpdate(t1, new DocumentId("doc:this:is:a:test"));
+
+        DocumentType t2 = new DocumentType("foo");
+        Document document = new Document(t2, "doc:this:is:another:test");
+
+        try {
+            documentUpdate.applyTo(document);
+            fail("Should have gotten exception here.");
+        } catch (IllegalArgumentException iae) {
+            //ok!
+        }
+    }
+
+    public void testFieldUpdatesInDocUp() throws ParseException {
+        DocumentType t1 = new DocumentType("doo");
+        Field f1 = new Field("field1", DataType.STRING);
+        Field f2 = new Field("field2", DataType.STRING);
+        t1.addField(f1);
+        t1.addField(f2);
+
+        DocumentUpdate documentUpdate = new DocumentUpdate(t1, new DocumentId("doc:this:is:a:test"));
+
+        assertEquals(0, documentUpdate.size());
+
+        FieldUpdate fu1 = FieldUpdate.createAssign(f1, new StringFieldValue("banana"));
+        FieldUpdate fu2 = FieldUpdate.createAssign(f2, new StringFieldValue("apple"));
+        documentUpdate.addFieldUpdate(fu1);
+
+        assertEquals(1, documentUpdate.size());
+
+        documentUpdate.clearFieldUpdates();
+
+        assertEquals(0, documentUpdate.size());
+
+        documentUpdate.addFieldUpdate(fu1);
+        documentUpdate.addFieldUpdate(fu2);
+
+        assertEquals(2, documentUpdate.size());
+
+
+        assertSame(fu1, documentUpdate.getFieldUpdate(f1));
+        assertSame(fu1, documentUpdate.getFieldUpdate(0));
+        assertSame(fu2, documentUpdate.getFieldUpdate(1));
+
+        documentUpdate.setFieldUpdate(0, fu2);
+        documentUpdate.setFieldUpdate(1, fu1);
+        assertEquals(2, documentUpdate.size());
+        assertSame(fu1, documentUpdate.getFieldUpdate(1));
+        assertSame(fu2, documentUpdate.getFieldUpdate(0));
+
+        try {
+            documentUpdate.setFieldUpdates(null);
+            fail("Should have gotten NullPointerException");
+        } catch (NullPointerException npe) {
+            //ok!
+        }
+
+        List<FieldUpdate> fus = new ArrayList<>();
+        fus.add(fu1);
+        fus.add(fu2);
+
+        documentUpdate.setFieldUpdates(fus);
+        assertEquals(2, documentUpdate.size());
+        assertSame(fu1, documentUpdate.getFieldUpdate(0));
+        assertSame(fu2, documentUpdate.getFieldUpdate(1));
+
+        documentUpdate.removeFieldUpdate(1);
+        assertEquals(1, documentUpdate.size());
+        assertSame(fu1, documentUpdate.getFieldUpdate(0));
+
+
+        documentUpdate.toString();
+
+        assertFalse(documentUpdate.isEmpty());
+
+        Iterator<FieldPathUpdate> fpUpdates = documentUpdate.iterator();
+        assertFalse(fpUpdates.hasNext());
+    }
+
+    public void testAddAll() {
+        DocumentType t1 = new DocumentType("doo");
+        DocumentType t2 = new DocumentType("foo");
+        Field f1 = new Field("field1", DataType.STRING);
+        Field f2 = new Field("field2", DataType.STRING);
+        t1.addField(f1);
+        t2.addField(f2);
+
+        DocumentUpdate du1 = new DocumentUpdate(t1, new DocumentId("doc:this:is:a:test"));
+        DocumentUpdate du2 = new DocumentUpdate(t2, "doc:this:is:another:test");
+
+        FieldUpdate fu1 = FieldUpdate.createAssign(f1, new StringFieldValue("banana"));
+        FieldUpdate fu2 = FieldUpdate.createAssign(f2, new StringFieldValue("apple"));
+        du1.addFieldUpdate(fu1);
+        du2.addFieldUpdate(fu2);
+
+        du1.addAll(null);
+
+        try {
+            du1.addAll(du2);
+            fail("Should have gotten exception");
+        } catch (IllegalArgumentException iae) {
+            //ok!
+        }
+
+        DocumentUpdate du3 = new DocumentUpdate(t2, new DocumentId("doc:this:is:a:test"));
+
+        try {
+            du1.addAll(du3);
+            fail("Should have gotten exception");
+        } catch (IllegalArgumentException iae) {
+            //ok!
+        }
+    }
+
+    private void assertDocumentUpdateFlag(boolean createIfNonExistent, int value) {
+        DocumentUpdateFlags f1 = new DocumentUpdateFlags();
+        f1.setCreateIfNonExistent(createIfNonExistent);
+        assertEquals(createIfNonExistent, f1.getCreateIfNonExistent());
+        int combined = f1.injectInto(value);
+        System.out.println("createIfNonExistent=" + createIfNonExistent + ", value=" + value + ", combined=" + combined);
+
+        DocumentUpdateFlags f2 = DocumentUpdateFlags.extractFlags(combined);
+        int extractedValue = DocumentUpdateFlags.extractValue(combined);
+        assertEquals(createIfNonExistent, f2.getCreateIfNonExistent());
+        assertEquals(value, extractedValue);
+    }
+
+    public void testRequireThatDocumentUpdateFlagsIsWorking() {
+        { // create-if-non-existent = true
+            assertDocumentUpdateFlag(true, 0);
+            assertDocumentUpdateFlag(true, 1);
+            assertDocumentUpdateFlag(true, 2);
+            assertDocumentUpdateFlag(true, 9999);
+            assertDocumentUpdateFlag(true, 0xFFFFFFE);
+            assertDocumentUpdateFlag(true, 0xFFFFFFF);
+        }
+        { // create-if-non-existent = false
+            assertDocumentUpdateFlag(false, 0);
+            assertDocumentUpdateFlag(false, 1);
+            assertDocumentUpdateFlag(false, 2);
+            assertDocumentUpdateFlag(false, 9999);
+            assertDocumentUpdateFlag(false, 0xFFFFFFE);
+            assertDocumentUpdateFlag(false, 0xFFFFFFF);
+        }
+    }
+
+    public void testRequireThatCreateIfNonExistentFlagIsSerializedAndDeserialized() {
+        docUp.setCreateIfNonExistent(true);
+
+        DocumentSerializer serializer = DocumentSerializerFactory.createHead(new GrowableByteBuffer());
+        docUp.serialize(serializer);
+        serializer.getBuf().flip();
+
+        DocumentDeserializer deserializer = DocumentDeserializerFactory.createHead(docMan, serializer.getBuf());
+        DocumentUpdate deserialized = new DocumentUpdate(deserializer);
+        assertEquals(docUp, deserialized);
+        assertTrue(deserialized.getCreateIfNonExistent());
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/DocumentUtilTestCase.java b/document/src/test/java/com/yahoo/document/DocumentUtilTestCase.java
new file mode 100644
index 00000000000..60d4e994bff
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/DocumentUtilTestCase.java
@@ -0,0 +1,47 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ * @since 5.1.10
+ */
+public class DocumentUtilTestCase extends junit.framework.TestCase {
+
+    public void testBasic() {
+/***
+        final long heapBytes = Runtime.getRuntime().maxMemory();
+        long maxPendingBytes = DocumentUtil.calculateMaxPendingSize(1.0, 1.0, 0);
+        almostEquals(heapBytes / 5, maxPendingBytes, 512*1024);
+    }
+
+    public void test2_2() {
+        final long heapBytes = Runtime.getRuntime().maxMemory();
+        long maxPendingBytes = DocumentUtil.calculateMaxPendingSize(2.0, 2.0, 0);
+        almostEquals(heapBytes / 5, maxPendingBytes, 512*1024);
+    }
+
+    public void test4_4() {
+        final long heapBytes = Runtime.getRuntime().maxMemory();
+        long maxPendingBytes = DocumentUtil.calculateMaxPendingSize(4.0, 4.0, 0);
+        almostEquals(heapBytes / 17, maxPendingBytes, 512*1024);
+    }
+
+    public void test8_8() {
+        final long heapBytes = Runtime.getRuntime().maxMemory();
+        long maxPendingBytes = DocumentUtil.calculateMaxPendingSize(8.0, 8.0, 0);
+        almostEquals(heapBytes / 65, maxPendingBytes, 512*1024);
+    }
+
+    public void test10000_10000() {
+        long maxPendingBytes = DocumentUtil.calculateMaxPendingSize(10000.0, 10000.0, 0);
+        almostEquals(1*1024*1024, maxPendingBytes, 512*1024);
+***/
+    }
+
+    private static void almostEquals(long expected, long actual, long off) {
+        System.err.println("Got actual " + (((double) actual) / 1024d / 1024d) + " MB, expected "
+                + (((double) expected) / 1024d / 1024d) + " MB, within +/- " + (((double) off) / 1024d / 1024d) + " MB");
+
+        assertTrue(actual > (expected - off) && actual < (expected + off));
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/FieldPathEntryTestCase.java b/document/src/test/java/com/yahoo/document/FieldPathEntryTestCase.java
new file mode 100644
index 00000000000..064cadc0db9
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/FieldPathEntryTestCase.java
@@ -0,0 +1,38 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import org.junit.Test;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.not;
+import static org.junit.Assert.assertThat;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ * @since 5.1.10
+ */
+public class FieldPathEntryTestCase {
+    @Test
+    public void testKeyParseResult() {
+        FieldPathEntry.KeyParseResult result1 = new FieldPathEntry.KeyParseResult("banana", 2);
+        FieldPathEntry.KeyParseResult result2 = new FieldPathEntry.KeyParseResult("banana", 2);
+        FieldPathEntry.KeyParseResult result3 = new FieldPathEntry.KeyParseResult("apple", 2);
+        FieldPathEntry.KeyParseResult result4 = new FieldPathEntry.KeyParseResult("banana", 3);
+
+
+        assertThat(result1, equalTo(result2));
+        assertThat(result2, equalTo(result1));
+        assertThat(result1.hashCode(), equalTo(result2.hashCode()));
+        assertThat(result1.toString(), equalTo(result2.toString()));
+
+        assertThat(result1, not(equalTo(result3)));
+        assertThat(result3, not(equalTo(result1)));
+        assertThat(result1.hashCode(), not(equalTo(result3.hashCode())));
+        assertThat(result1.toString(), not(equalTo(result3.toString())));
+
+        assertThat(result1, not(equalTo(result4)));
+        assertThat(result4, not(equalTo(result1)));
+        assertThat(result1.hashCode(), not(equalTo(result4.hashCode())));
+        assertThat(result1.toString(), not(equalTo(result4.toString())));
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/FieldTestCase.java b/document/src/test/java/com/yahoo/document/FieldTestCase.java
new file mode 100644
index 00000000000..b8be9e85813
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/FieldTestCase.java
@@ -0,0 +1,58 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+/**
+ * @author <a href="thomasg@yahoo-inc.com>Thomas Gundersen</a>
+ */
+public class FieldTestCase extends junit.framework.TestCase {
+    public FieldTestCase(String name) {
+        super(name);
+    }
+
+    public void testIdSettingConflict() {
+        DocumentType doc = new DocumentType("testdoc");
+        Field one = doc.addField("one", DataType.STRING);
+        one.setId(60, doc);
+
+        Field two = doc.addField("two", DataType.STRING);
+        two.setId(61, doc);
+
+        try {
+            Field three = doc.addField("three", DataType.STRING);
+            three.setId(60, doc);
+            fail("Allowed to set duplicate id");
+        } catch (IllegalArgumentException e) {
+            // Success
+        }
+    }
+
+    public void testSettingReservedId() {
+        DocumentType doc = new DocumentType("testdoc");
+        try {
+            Field one = doc.addField("one", DataType.STRING);
+            one.setId(127, doc);
+            fail("Allowed to set reserved id");
+        } catch (IllegalArgumentException e) {
+            // Success
+        }
+
+        try {
+            Field one = doc.addField("one", DataType.STRING);
+            one.setId(100, doc);
+            fail("Allowed to set reserved id");
+        } catch (IllegalArgumentException e) {
+            // Success
+        }
+
+        try {
+            Field one = doc.addField("one", DataType.STRING);
+            one.setId(-1, doc);
+            fail("Allowed to set reserved id");
+        } catch (IllegalArgumentException e) {
+            // Success
+        }
+        doc.removeField("one");
+        Field one = doc.addField("one", DataType.STRING);
+    }
+
+}
diff --git a/document/src/test/java/com/yahoo/document/GlobalIdTestCase.java b/document/src/test/java/com/yahoo/document/GlobalIdTestCase.java
new file mode 100644
index 00000000000..30b8f4c42ea
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/GlobalIdTestCase.java
@@ -0,0 +1,87 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import java.util.Arrays;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ * @since 5.1.10
+ */
+public class GlobalIdTestCase extends junit.framework.TestCase {
+    private final byte[] raw0 = new byte[0];
+    private final byte[] raw1_0 = new byte[]{(byte) 0};
+    private final byte[] raw2_11 = new byte[]{(byte) 1, (byte) 1};
+    private final byte[] raw2_minus1_1 = new byte[]{(byte) -1, (byte) 1};
+    private final byte[] raw12_1to12 = new byte[]{(byte) 1, (byte) 2, (byte) 3, (byte) 4, (byte) 5, (byte) 6, (byte) 7,
+            (byte) 8, (byte) 9, (byte) 10, (byte) 11, (byte) 12};
+    private final byte[] raw13 = new byte[]{(byte) 1, (byte) 2, (byte) 3, (byte) 4, (byte) 5, (byte) 6, (byte) 7,
+            (byte) 8, (byte) 9, (byte) 10, (byte) 11, (byte) 12, (byte) 13};
+
+    private final BucketIdFactory bucketIdFactory = new BucketIdFactory();
+
+    public void testRaw0() {
+        GlobalId gid = new GlobalId(raw0);
+        assertEquals(12, gid.getRawId().length);
+        byte[] raw = gid.getRawId();
+        for (byte b : raw) {
+            assertEquals((byte) 0, b);
+        }
+
+        GlobalId gid2 = new GlobalId(raw1_0);
+        assertEquals(12, gid2.getRawId().length);
+        byte[] raw2 = gid2.getRawId();
+        for (byte b : raw2) {
+            assertEquals((byte) 0, b);
+        }
+
+        assertEquals(gid, gid2);
+        assertTrue(Arrays.equals(raw, raw2));
+        assertEquals(gid.hashCode(), gid2.hashCode());
+    }
+
+    public void testLonger() {
+        GlobalId gid1 = new GlobalId(raw2_11);
+        GlobalId gid2 = new GlobalId(raw2_minus1_1);
+
+        assertFalse(gid1.equals(gid2));
+        assertFalse(gid1.hashCode() == gid2.hashCode());
+
+        GlobalId gid3 = new GlobalId(raw13);
+        GlobalId gid4 = new GlobalId(raw12_1to12);
+        assertEquals(gid3, gid4);
+        assertEquals(gid3.hashCode(), gid4.hashCode());
+    }
+
+    public void testCompareTo() {
+        GlobalId gid0 = new GlobalId(raw1_0);
+        GlobalId gid11 = new GlobalId(raw2_11);
+        GlobalId gidminus11 = new GlobalId(raw2_minus1_1);
+
+        assertEquals(-1, gid0.compareTo(gid11));
+        assertEquals(1, gid11.compareTo(gid0));
+
+        assertEquals(-1, gid0.compareTo(gidminus11));
+        assertEquals(1, gidminus11.compareTo(gid0));
+
+        assertEquals(-1, gid11.compareTo(gidminus11));
+        assertEquals(1, gidminus11.compareTo(gid11));
+    }
+
+    private void verifyGidToBucketIdMapping(String idString) {
+        DocumentId documentId = new DocumentId(idString);
+        GlobalId globalId = new GlobalId(documentId.getGlobalId());
+        BucketId bucketIdThroughGlobalId = globalId.toBucketId();
+        BucketId bucketIdThroughFactory = bucketIdFactory.getBucketId(documentId);
+        assertEquals(bucketIdThroughFactory, bucketIdThroughGlobalId);
+    }
+
+    public void testToBucketId() {
+        verifyGidToBucketIdMapping("userdoc:ns:1:abc");
+        verifyGidToBucketIdMapping("userdoc:ns:1000:abc");
+        verifyGidToBucketIdMapping("userdoc:hsgf:18446744073700000000:dfdfsdfg");
+        verifyGidToBucketIdMapping("groupdoc:ns:somegroup:hmm");
+        verifyGidToBucketIdMapping("doc:foo:test");
+        verifyGidToBucketIdMapping("doc:myns:http://foo.bar");
+        verifyGidToBucketIdMapping("doc:jsrthsdf:a234aleingzldkifvasdfgadf");
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/IdIdStringTest.java b/document/src/test/java/com/yahoo/document/IdIdStringTest.java
new file mode 100644
index 00000000000..bbdf45ec088
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/IdIdStringTest.java
@@ -0,0 +1,70 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import com.yahoo.document.idstring.IdIdString;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+/**
+ * Created with IntelliJ IDEA.
+ * User: magnarn
+ * Date: 10/16/12
+ * Time: 9:10 AM
+ */
+public class IdIdStringTest {
+    @Test
+    public void requireThatIdIdStringGeneratesProperString() throws Exception {
+        DocumentId docId = new DocumentId(new IdIdString("namespace", "type", "g=group", "foobar"));
+        assertEquals("id:namespace:type:g=group:foobar", docId.toString());
+    }
+
+    @Test
+    public void requireThatEmptyKeyValuesAreOk() throws Exception {
+        DocumentId docId = new DocumentId(new IdIdString("namespace", "type", "", "foobar"));
+        assertEquals("id:namespace:type::foobar", docId.toString());
+    }
+
+    @Test
+    public void requireThatIdIdStringCanBehaveLikeGroupDoc() throws Exception {
+        DocumentId docId1 = new DocumentId(new IdIdString("namespace", "type", "g=foo", "foo"));
+        DocumentId docId2 = new DocumentId(new IdIdString("namespace", "type", "g=foo", "bar"));
+        DocumentId docId3 = new DocumentId(new IdIdString("namespace", "type", "g=bar", "baz"));
+        assertEquals(docId1.getScheme().getLocation(), docId2.getScheme().getLocation());
+        assert(docId1.getScheme().getLocation() != docId3.getScheme().getLocation());
+    }
+
+    @Test
+    public void requireThatIdIdStringCanBehaveLikeUserDoc() throws Exception {
+        DocumentId docId1 = new DocumentId(new IdIdString("namespace", "type", "n=10", "foo"));
+        DocumentId docId2 = new DocumentId(new IdIdString("namespace", "type", "n=10", "bar"));
+        DocumentId docId3 = new DocumentId(new IdIdString("namespace", "type", "n=20", "baz"));
+        assertEquals(docId1.getScheme().getLocation(), docId2.getScheme().getLocation());
+        assert(docId1.getScheme().getLocation() != docId3.getScheme().getLocation());
+    }
+
+    @Test
+    public void requireThatIllegalKeyValuesThrow() throws Exception {
+        try {
+            new IdIdString("namespace", "type", "illegal=key", "foo");
+            fail();
+        } catch (IllegalArgumentException e) {
+        }
+    }
+
+    @Test
+    public void requireThatKeysWithoutValuesThrow() throws Exception {
+        try {
+            new IdIdString("namespace", "type", "illegal-pair", "foo");
+            fail();
+        } catch (IllegalArgumentException e) {
+        }
+    }
+
+    @Test
+    public void requireThatIdIdStringCanReplaceType() throws Exception {
+        String type = IdIdString.replaceType("id:namespace:type::foo", "newType");
+        assertEquals("id:namespace:newType::foo", type);
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/IncompatibleFieldTypesTest.java b/document/src/test/java/com/yahoo/document/IncompatibleFieldTypesTest.java
new file mode 100644
index 00000000000..9e0c4e353f2
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/IncompatibleFieldTypesTest.java
@@ -0,0 +1,44 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import com.yahoo.document.datatypes.*;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Tests for ticket 6394548
+ */
+public class IncompatibleFieldTypesTest {
+    private DataType arrayOfStrings;
+    private StructDataType struct;
+    private StructuredFieldValue root;
+
+    @Before
+    public void setUp() {
+        arrayOfStrings = new ArrayDataType(DataType.STRING);
+        struct = new StructDataType("fancypants");
+        struct.addField(new Field("stringarray", arrayOfStrings, false));
+        DataType weightedSetOfStrings = DataType.getWeightedSet(DataType.STRING, false, false);
+        struct.addField(new Field("stringws", weightedSetOfStrings, false));
+
+        root = struct.createFieldValue();
+        root.setFieldValue("stringarray", arrayOfStrings.createFieldValue());
+        root.setFieldValue("stringws", weightedSetOfStrings.createFieldValue());
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testAddingIncompatibleFieldToArrayFails() {
+        System.out.println(root.getFieldValue("stringarray").getDataType().createFieldValue().getClass().getName());
+        System.out.println(root.getFieldValue("stringarray").getDataType().createFieldValue().getDataType().toString());
+
+        ((Array)root.getFieldValue("stringarray")).add(new IntegerFieldValue(1234));
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testAddingIncompatibleFieldToWeightedSetFails() {
+        System.out.println(root.getFieldValue("stringws").getDataType().createFieldValue().getClass().getName());
+        System.out.println(root.getFieldValue("stringws").getDataType().createFieldValue().getDataType().toString());
+
+        ((WeightedSet<FieldValue>)root.getFieldValue("stringws")).put(new IntegerFieldValue(1234), 100);
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/NumericDataTypeTestCase.java b/document/src/test/java/com/yahoo/document/NumericDataTypeTestCase.java
new file mode 100644
index 00000000000..ebd63aa47b5
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/NumericDataTypeTestCase.java
@@ -0,0 +1,35 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.IntegerFieldValue;
+import org.junit.Test;
+
+import static org.hamcrest.CoreMatchers.*;
+import static org.junit.Assert.*;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ * @since 5.1.10
+ */
+public class NumericDataTypeTestCase {
+
+    @Test
+    public void basic() {
+        NumericDataType type = new NumericDataType("foo", 0, FieldValue.class, IntegerFieldValue.getFactory());
+        NumericDataType clonedType = type.clone();
+        assertThat(type, equalTo(clonedType));
+        assertThat(type, not(sameInstance(clonedType)));
+    }
+
+    @Test
+    public void create() {
+        try {
+            new NumericDataType("foo", 0, IntegerFieldValue.class, IntegerFieldValue.getFactory()).createFieldValue(new Object());
+            fail();
+        } catch (IllegalArgumentException e) {
+            assertEquals("Class class java.lang.Object not applicable to an class " +
+                         "com.yahoo.document.datatypes.IntegerFieldValue instance.", e.getMessage());
+        }
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/PositionTypeTestCase.java b/document/src/test/java/com/yahoo/document/PositionTypeTestCase.java
new file mode 100644
index 00000000000..3fb9c8d3f9e
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/PositionTypeTestCase.java
@@ -0,0 +1,44 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import com.yahoo.document.datatypes.IntegerFieldValue;
+import com.yahoo.document.datatypes.Struct;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class PositionTypeTestCase {
+
+    @Test
+    public void requireThatPositionFactoryWorks() {
+        Struct val = PositionDataType.valueOf(6, 9);
+        assertEquals(new IntegerFieldValue(6), val.getFieldValue(PositionDataType.FIELD_X));
+        assertEquals(new IntegerFieldValue(9), val.getFieldValue(PositionDataType.FIELD_Y));
+        assertEquals(2, val.getFieldCount());
+
+        val = PositionDataType.fromLong((6L << 32) + 9);
+        assertEquals(new IntegerFieldValue(6), val.getFieldValue(PositionDataType.FIELD_X));
+        assertEquals(new IntegerFieldValue(9), val.getFieldValue(PositionDataType.FIELD_Y));
+        assertEquals(2, val.getFieldCount());
+    }
+
+    @Test
+    public void requireThatAccessorsWork() {
+        Struct val = PositionDataType.valueOf(6, 9);
+        assertEquals(new IntegerFieldValue(6), PositionDataType.getXValue(val));
+        assertEquals(new IntegerFieldValue(9), PositionDataType.getYValue(val));
+    }
+
+    @Test
+    public void requireThatConstantsMatchCpp() {
+        assertEquals("position", PositionDataType.STRUCT_NAME);
+        assertEquals("x", PositionDataType.FIELD_X);
+        assertEquals("y", PositionDataType.FIELD_Y);
+        assertEquals("foo_zcurve", PositionDataType.getZCurveFieldName("foo"));
+        assertEquals("foo.position", PositionDataType.getPositionSummaryFieldName("foo"));
+        assertEquals("foo.distance", PositionDataType.getDistanceSummaryFieldName("foo"));
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/SimpleDocumentTestCase.java b/document/src/test/java/com/yahoo/document/SimpleDocumentTestCase.java
new file mode 100644
index 00000000000..853563165e4
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/SimpleDocumentTestCase.java
@@ -0,0 +1,33 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import com.yahoo.document.datatypes.IntegerFieldValue;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class SimpleDocumentTestCase {
+
+    @Test
+    public void requireThatAccessorsWorks() {
+        DocumentType type = new DocumentType("test");
+        type.addField("int", DataType.INT);
+        Document doc = new Document(type, "doc:scheme:");
+        SimpleDocument simple = new SimpleDocument(doc);
+
+        assertNull(simple.get("int"));
+        assertNull(doc.getFieldValue("int"));
+
+        simple.set("int", 69);
+        assertEquals(69, simple.get("int"));
+        assertEquals(new IntegerFieldValue(69), doc.getFieldValue("int"));
+
+        simple.remove("int");
+        assertNull(simple.get("int"));
+        assertNull(doc.getFieldValue("int"));
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/StructDataTypeTestCase.java b/document/src/test/java/com/yahoo/document/StructDataTypeTestCase.java
new file mode 100755
index 00000000000..6e5bfbb56d4
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/StructDataTypeTestCase.java
@@ -0,0 +1,68 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import com.yahoo.document.datatypes.Struct;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class StructDataTypeTestCase extends junit.framework.TestCase {
+    public void testSimpleInheritance() {
+        StructDataType personType = new StructDataType("person");
+        Field firstName = new Field("firstname", DataType.STRING);
+        Field lastName = new Field("lastname", DataType.STRING);
+        personType.addField(firstName);
+        personType.addField(lastName);
+
+        StructDataType employeeType = new StructDataType("employee");
+        Field empId = new Field("employeeid", DataType.INT);
+        employeeType.addField(empId);
+
+        assertEquals(2, personType.getFieldCount());
+        assertEquals("firstname", personType.getFields().toArray(new Field[0])[0].getName());
+        assertEquals("lastname", personType.getFields().toArray(new Field[0])[1].getName());
+        assertEquals(1, employeeType.getFieldCount());
+        assertEquals("employeeid", employeeType.getFields().toArray(new Field[0])[0].getName());
+
+        employeeType.inherit(personType);
+
+        assertEquals(2, personType.getFieldCount());
+        assertEquals("firstname", personType.getFields().toArray(new Field[0])[0].getName());
+        assertEquals("lastname", personType.getFields().toArray(new Field[0])[1].getName());
+        assertEquals(3, employeeType.getFieldCount());
+        assertEquals("employeeid", employeeType.getFields().toArray(new Field[0])[0].getName());
+        assertEquals("firstname", employeeType.getFields().toArray(new Field[0])[1].getName());
+        assertEquals("lastname", employeeType.getFields().toArray(new Field[0])[2].getName());
+    }
+
+    public void testCompatibleWith() {
+        StructDataType personType = new StructDataType("person");
+        Field firstName = new Field("firstname", DataType.STRING);
+        Field lastName = new Field("lastname", DataType.STRING);
+        personType.addField(firstName);
+        personType.addField(lastName);
+
+        StructDataType employeeType = new StructDataType("employee");
+        Field empId = new Field("employeeid", DataType.INT);
+        employeeType.addField(empId);
+        employeeType.inherit(personType);
+
+        Struct person = new Struct(personType);
+        Struct employee = new Struct(employeeType);
+
+        assertTrue(personType.isValueCompatible(person));
+        assertTrue(personType.isValueCompatible(employee));
+
+        assertTrue(employeeType.isValueCompatible(employee));
+        assertFalse(employeeType.isValueCompatible(person));
+
+        StructDataType containerType = new StructDataType("containerstruct");
+        Field structPolymorphic = new Field("structpolymorphic", personType);
+        containerType.addField(structPolymorphic);
+
+        Struct container = new Struct(containerType);
+        container.setFieldValue(structPolymorphic, person);
+        container.setFieldValue(structPolymorphic, employee);
+    }
+
+}
diff --git a/document/src/test/java/com/yahoo/document/TemporaryDataTypeTestCase.java b/document/src/test/java/com/yahoo/document/TemporaryDataTypeTestCase.java
new file mode 100644
index 00000000000..6f841aac821
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/TemporaryDataTypeTestCase.java
@@ -0,0 +1,24 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import com.yahoo.document.datatypes.StringFieldValue;
+import org.junit.Test;
+
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.CoreMatchers.nullValue;
+import static org.junit.Assert.assertThat;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ * @since 5.1.10
+ */
+public class TemporaryDataTypeTestCase {
+    @Test
+    public void requireNulls() {
+        TemporaryDataType type = new TemporaryDataType(0);
+        assertThat(type.createFieldValue(new Object()), nullValue());
+        assertThat(type.createFieldValue(), nullValue());
+        assertThat(type.getValueClass(), nullValue());
+        assertThat(type.isValueCompatible(new StringFieldValue("")), is(false));
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/TemporaryStructuredDataTypeTestCase.java b/document/src/test/java/com/yahoo/document/TemporaryStructuredDataTypeTestCase.java
new file mode 100644
index 00000000000..aad0505c365
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/TemporaryStructuredDataTypeTestCase.java
@@ -0,0 +1,24 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import org.junit.Test;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.not;
+import static org.junit.Assert.assertThat;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ * @since 5.1.10
+ */
+public class TemporaryStructuredDataTypeTestCase {
+    @Test
+    public void basic() {
+        TemporaryStructuredDataType type = TemporaryStructuredDataType.create("banana");
+        assertThat(type.getName(), equalTo("banana"));
+        int originalId = type.getId();
+        type.setName("apple");
+        assertThat(type.getName(), equalTo("apple"));
+        assertThat(originalId, not(equalTo(type.getId())));
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/annotation/AbstractTypesTest.java b/document/src/test/java/com/yahoo/document/annotation/AbstractTypesTest.java
new file mode 100755
index 00000000000..cc9cb8b2dd2
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/annotation/AbstractTypesTest.java
@@ -0,0 +1,150 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.DocumentTypeManager;
+import com.yahoo.document.Field;
+import com.yahoo.document.StructDataType;
+import com.yahoo.document.datatypes.IntegerFieldValue;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.document.datatypes.Struct;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public abstract class AbstractTypesTest {
+
+    protected DocumentTypeManager man;
+    protected StructDataType person;
+    protected AnnotationReferenceDataType personReference;
+    protected StructDataType relative;
+    protected AnnotationType dummy;
+    protected AnnotationType number;
+    protected AnnotationType personA;
+    protected AnnotationType relativeA;
+    protected AnnotationType banana;
+    protected AnnotationType apple;
+    protected AnnotationType grape;
+    protected DocumentType docType;
+
+    public AbstractTypesTest() {
+        man = new DocumentTypeManager();
+
+        person = new StructDataType("person");
+        person.addField(new Field("firstname", DataType.STRING));
+        person.addField(new Field("lastname", DataType.STRING));
+        person.addField(new Field("birthyear", DataType.INT));
+        man.register(person);
+
+        personA = new AnnotationType("person", person);
+        man.getAnnotationTypeRegistry().register(personA);
+
+        personReference = new AnnotationReferenceDataType(personA);
+        man.register(personReference);
+
+        relative = new StructDataType("relative");
+        relative.addField(new Field("title", DataType.STRING));
+        relative.addField(new Field("related", personReference));
+        man.register(relative);
+
+        dummy = new AnnotationType("dummy");
+        number = new AnnotationType("number", DataType.INT);
+        relativeA = new AnnotationType("relative", relative);
+        banana = new AnnotationType("banana");
+        apple = new AnnotationType("apple");
+        grape = new AnnotationType("grape");
+        man.getAnnotationTypeRegistry().register(dummy);
+        man.getAnnotationTypeRegistry().register(number);
+        man.getAnnotationTypeRegistry().register(relativeA);
+        man.getAnnotationTypeRegistry().register(banana);
+        man.getAnnotationTypeRegistry().register(apple);
+        man.getAnnotationTypeRegistry().register(grape);
+
+        docType = new DocumentType("dokk");
+        docType.addField("age", DataType.BYTE);
+        docType.addField("story", DataType.STRING);
+        docType.addField("date", DataType.INT);
+        docType.addField("friend", DataType.LONG);
+        man.register(docType);
+    }
+
+    protected StringFieldValue getAnnotatedString() {
+        StringFieldValue text = new StringFieldValue("help me help me i'm stuck inside a computer!");
+        {
+            AlternateSpanList alternateSpanList = new AlternateSpanList();
+            SpanTree tree = new SpanTree("ballooo", alternateSpanList);
+            text.setSpanTree(tree);
+
+            Span s1 = new Span(1, 2);
+            Span s2 = new Span(3, 4);
+            Span s3 = new Span(4, 5);
+            alternateSpanList.add(s1).add(s2).add(s3);
+            AlternateSpanList s4 = new AlternateSpanList();
+            Span s5 = new Span(7, 8);
+            Span s6 = new Span(8, 9);
+            s4.add(s5).add(s6);
+            alternateSpanList.add(s4);
+
+            tree.annotate(s2, dummy);
+            tree.annotate(s2, new Annotation(number, new IntegerFieldValue(1234)));
+
+            Struct mother = new Struct(person);
+            mother.setFieldValue("firstname", "jenny");
+            mother.setFieldValue("lastname", "olsen");
+            mother.setFieldValue("birthyear", 1909);
+            Annotation motherA = new Annotation(personA, mother);
+            tree.annotate(s2, motherA);
+
+            Struct daughter = new Struct(relative);
+            daughter.setFieldValue("title", "daughter");
+            daughter.setFieldValue("related", new AnnotationReference(personReference, motherA));
+            tree.annotate(s6, new Annotation(relativeA, daughter));
+
+            tree.annotate(s1, dummy);
+            tree.annotate(s3, dummy);
+            tree.annotate(s3, new Annotation(number, new IntegerFieldValue(2344)));
+            tree.annotate(s5, dummy);
+
+            List<SpanNode> alternateChildren = new ArrayList<>();
+            Span s7 = new Span(1, 4);
+            Span s8 = new Span(1, 9);
+            Span s9 = new Span(5, 6);
+            alternateChildren.add(s7);
+            alternateChildren.add(s8);
+            alternateChildren.add(s9);
+
+            alternateSpanList.addChildren(alternateChildren, 5.55);
+        }
+        {
+            SpanList root = new SpanList();
+            SpanTree tree = new SpanTree("fruits", root);
+            text.setSpanTree(tree);
+
+
+            Span s1 = new Span(5, 6);
+            Span s2 = new Span(11, 3);
+            Span s3 = new Span(14, 7);
+            root.add(s1).add(s2).add(s3);
+
+            tree.annotate(s1, banana);
+            tree.annotate(s1, grape);
+            tree.annotate(s2, banana);
+            tree.annotate(s3, apple);
+            tree.annotate(s3, grape);
+            tree.annotate(s3, grape);
+
+            SpanList s4 = new SpanList();
+            Span s5 = new Span(23,1);
+            s4.add(s5);
+            root.add(s4);
+
+            tree.annotate(s4, grape);
+            tree.annotate(s5, apple);
+        }
+        return text;
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/annotation/AlternateSpanListAdvTestCase.java b/document/src/test/java/com/yahoo/document/annotation/AlternateSpanListAdvTestCase.java
new file mode 100644
index 00000000000..e5950abe244
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/annotation/AlternateSpanListAdvTestCase.java
@@ -0,0 +1,429 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.datatypes.StringFieldValue;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.ListIterator;
+import java.util.TreeSet;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author <a href="mailto:mpraveen@yahoo-inc.com">Praveen Mohan</a>
+ *
+ * This test covers all possible scenarios in AlternateSpanList.
+ * If you really want to debug, just turn on the debug flag to true.
+ *
+ */
+public class AlternateSpanListAdvTestCase {
+
+    private AnnotationType at1 = new AnnotationType("person", DataType.STRING);
+    private AnnotationType at2 = new AnnotationType("street", DataType.STRING);
+    private AnnotationType at3 = new AnnotationType("city", DataType.STRING);
+
+    private StringFieldValue fv1 = (StringFieldValue) at1.getDataType().createFieldValue();
+    private StringFieldValue fv2 = (StringFieldValue) at2.getDataType().createFieldValue();
+    private StringFieldValue fv3 = (StringFieldValue) at3.getDataType().createFieldValue();
+    private StringFieldValue fv11 = (StringFieldValue) at1.getDataType().createFieldValue();
+    private StringFieldValue fv22 = (StringFieldValue) at2.getDataType().createFieldValue();
+    private StringFieldValue fv33 = (StringFieldValue) at3.getDataType().createFieldValue();
+    private StringFieldValue fv111 = (StringFieldValue) at1.getDataType().createFieldValue();
+    private StringFieldValue fv222 = (StringFieldValue) at1.getDataType().createFieldValue();
+
+    private SpanList root;
+    private SpanTree tree;
+    private SpanNode span1, span2, span3;
+    private SpanNode span11, span22, span33;
+    private SpanList alternate1, alternate2, branch0;
+    private List<SpanNode> subtreeList1, subtreeList2;
+    private AlternateSpanList branch;
+    private AlternateSpanList branch3, branch2;
+    private Annotation an1;
+
+    private boolean debug = false;
+
+    @Before
+    public void buildTree_List() {
+        root = new SpanList();
+        tree = new SpanTree("test", root);
+        branch = new AlternateSpanList();
+        span1 = new Span(0, 3);
+        span2 = new Span(1, 9);
+        span3 = new Span(12, 10);
+
+        span11 = new Span(0, 3);
+        span22 = new Span(1, 9);
+        span33 = new Span(12, 10);
+
+        an1 = new Annotation(at1, fv1);
+        Annotation an2 = new Annotation(at2, fv2);
+        Annotation an3 = new Annotation(at3, fv3);
+        Annotation an11 = new Annotation(at1, fv11);
+        Annotation an22 = new Annotation(at2, fv22);
+        Annotation an33 = new Annotation(at3, fv33);
+
+        alternate1 = new SpanList();
+        alternate1.add(span3);
+        alternate1.add(span2);
+        alternate1.add(span1);
+
+        alternate2 = new SpanList();
+        alternate2.add(span11);
+        alternate2.add(span22);
+        alternate2.add(span33);
+
+        tree.annotate(span1, an1);
+        tree.annotate(span2, an2);
+        tree.annotate(span3, an3);
+
+        tree.annotate(span11, an11);
+        tree.annotate(span22, an22);
+        tree.annotate(span33, an33);
+
+        subtreeList1 = new ArrayList<SpanNode>();
+        subtreeList1.add(alternate1);
+
+        subtreeList2 = new ArrayList<SpanNode>();
+        subtreeList2.add(alternate2);
+        branch.clearChildren();
+        branch.addChildren(1, subtreeList1, 20.0d);
+        branch.addChildren(2, subtreeList2, 50.0d);
+
+        root.add(branch);
+    }
+
+    @Test(expected = IndexOutOfBoundsException.class)
+    public void assertInvalidGetFrom() {
+        assertEquals(0, branch.getFrom(50));
+    }
+
+    @Test (expected = IllegalStateException.class)
+    public void assertSharingAnnotationInstance() {
+        SpanNode testNode = new Span(0, 2);
+        tree.annotate(testNode, an1);
+    }
+
+    @Test (expected = IllegalStateException.class)
+    public void assertSharingSpanTreeRoot() {
+        tree = new SpanTree("dummy", root);
+    }
+
+
+    @Test (expected = IllegalStateException.class)
+    public void assertAddSameNodeTwice() {
+        root.add(branch);
+    }
+
+    @Test (expected = IndexOutOfBoundsException.class)
+    public void assertInvalidAdd() {
+        SpanNode newNode = new Span(10, 10);
+        branch.add(branch.getNumSubTrees(), newNode);
+    }
+
+    @Test (expected = IndexOutOfBoundsException.class)
+    public void assertInvalidChildIteratorIndex() {
+        branch.childIterator(branch.getNumSubTrees());
+    }
+
+    @Test (expected = IllegalStateException.class)
+    public void assertReuseRemovedNode() {
+        alternate1.remove(span1);
+        branch.add(span1);
+    }
+
+    @Test
+    public void assertTree_NodeSet1() {
+        if (debug) consumeAnnotations(tree, root);
+        assertEquals(-1, root.getFrom());
+        assertEquals(-1, root.getTo());
+        assertEquals(-1, branch.getFrom());
+        assertEquals(-1, branch.getTo());
+        assertEquals(0, branch.getFrom(1));
+        assertEquals(22, branch.getTo(1));
+        assertEquals(0, branch.getFrom(2));
+        assertEquals(22, branch.getTo(2));
+        assertEquals(3, branch.getNumSubTrees());
+        int no = branch.getNumSubTrees();
+
+        TreeSet<Double> set = new TreeSet<Double>();
+        for (int i = 0; i < no; i ++) {
+            double prob = branch.getProbability(i);
+            set.add(prob);
+        }
+
+        branch.sortSubTreesByProbability();
+
+        Iterator<Double> iter = set.descendingIterator();
+        for (int i = 0; i < no; i ++) {
+            double prob = branch.getProbability(i);
+            double prob1 = iter.next();
+            assertTrue(prob == prob1);
+        }
+        branch.normalizeProbabilities();
+        double highest = 0;
+        for (int i = 0; i < no; i ++) {
+            double prob = branch.getProbability(i);
+            if (i == 0) {
+                highest = prob;
+                continue;
+            }
+            assertFalse(prob >= highest || prob > 1.0);
+            highest = prob;
+        }
+
+        ListIterator<SpanNode> it = branch.childIterator();
+        assertSame(alternate2, it.next());
+        assertSame(alternate1, it.next());
+        assertFalse(it.hasNext());
+
+        SpanNode sn;
+
+        it = branch.childIteratorRecursive();
+        assertSame(span11, it.next());
+        assertSame(span22, it.next());
+        assertSame(span33, it.next());
+        assertSame(alternate2, it.next());
+        assertSame(span3, it.next());
+        assertSame(span2, it.next());
+        assertSame(span1, it.next());
+        assertSame(alternate1, it.next());
+        assertFalse(it.hasNext());
+
+
+        it = branch.childIterator(1);
+        assertSame(alternate1, it.next());
+        assertFalse(it.hasNext());
+
+
+        SpanNode snNew = new Span(15, 20);
+        List<SpanNode> alternate3 = new ArrayList<SpanNode>();
+        alternate3.add(snNew);
+        List<SpanNode> l = branch.setChildren(0, alternate3, 200.0d);
+        assertFalse (l.get(0) != alternate2);
+        assertFalse (branch.getProbability(0) != 200.0);
+
+        String s = branch.toString();
+        tree.cleanup();
+        boolean dMode = debug;
+        debug = false;
+        consumeAnnotations(tree, root);
+        debug = dMode;
+
+        branch.setProbability(0, 125.0d);
+        branch.setProbability(1, 75.0d);
+        branch.setProbability(2, 25.0d);
+        branch.sortSubTreesByProbability();
+        double initial = 125.0;
+        for (int j = 0; j < no; j++) {
+            double prob = branch.getProbability(j);
+            assertFalse (prob != initial);
+            initial = initial - 50.0d;
+        }
+        no = branch.getNumSubTrees();
+
+        for (int j = 0; j < no; no --) {
+            branch.removeChildren(j);
+        }
+
+        assertFalse (branch.getNumSubTrees() != 1);
+        assertFalse(branch.getProbability(0) != 1.0);
+        branch.addChildren(1, subtreeList1, 20.0d);
+        branch.addChildren(2, subtreeList2, 50.0d);
+
+        no = branch.getNumSubTrees();
+        assertFalse (no != 3);
+        branch.clearChildren();
+        assertFalse (branch.getNumSubTrees() != 3);
+        assertFalse(branch.getProbability(0) != 1.0);
+
+        branch.addChildren(1, subtreeList1, 20.0d);
+        branch.addChildren(2, subtreeList2, 50.0d);
+        no = branch.getNumSubTrees();
+        assertFalse (no != 5);
+        branch.clearChildren(1);
+        assertTrue (branch.getNumSubTrees() == no);
+        assertEquals(branch.getFrom(1), -1);
+        assertEquals(branch.getTo(1), -1);
+        assertTrue(branch.getProbability(1) == 20.0);
+
+        ListIterator<SpanNode> lit = branch.childIteratorRecursive(1);
+        assertFalse(lit.hasNext());
+        SpanNode newNode = new Span(10, 10);
+        branch.add(0, newNode);
+        lit = branch.childIteratorRecursive();
+        assertTrue(lit.hasNext());
+        assertFalse(lit.next() != newNode);
+
+        branch.removeChildren(1);
+        assertTrue (branch.getNumSubTrees() == (no-1));
+
+        branch.removeChildren();
+        no = branch.getNumSubTrees();
+        assertTrue (no == 1);
+        assertTrue (branch.getProbability(0) == 1.0);
+        assertEquals(branch.getFrom(), -1);
+        assertEquals(branch.getTo(), -1);
+
+        buildTree_List();
+
+        CharSequence fieldValue = "ABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890";
+        String actual = branch.getText(1, fieldValue).toString();
+        String expected = "MNOPQRSTUVBCDEFGHIJABC";
+        assertEquals(actual, expected);
+    }
+
+    @Test
+    public void assertTree_NodeSet2() {
+        root = new SpanList();
+        tree = new SpanTree("test", root);
+        branch = new AlternateSpanList();
+
+        Annotation an111 = new Annotation(at1, fv111);
+        Annotation an222 = new Annotation(at2, fv222);
+
+        branch0 = new SpanList();
+        SpanNode span1 = new Span(0, 3);
+        SpanNode span2 = new Span(3, 3);
+        branch0.add(span1);
+        branch0.add(span2);
+        branch.add(branch0);
+
+        ArrayList<SpanNode> list1 = new ArrayList<SpanNode>();
+        branch2 = new AlternateSpanList();
+        SpanNode sn1 = new Span(6, 4);
+        SpanNode sn2 = new Span(10, 4);
+        branch2.add(sn1);
+        branch2.add(sn2);
+        list1.add(branch2);
+        branch.setProbability(0, 20.0d);
+        branch.addChildren(1, list1, 30.0d);
+
+        branch3 = new AlternateSpanList();
+        SpanNode sn3 = new Span(15, 5);
+        SpanNode sn4 = new Span(20, 5);
+        ArrayList<SpanNode> list2 = new ArrayList<SpanNode>();
+        list2.add(sn3);
+        list2.add(sn4);
+        branch3.addChildren(1, list2, 20.0d);
+        branch3.setProbability(0, 10.0d);
+
+        List<SpanNode> list3 = new ArrayList<SpanNode>();
+        list3.add(branch3);
+        branch2.addChildren(1, list3, 50.0d);
+        branch2.setProbability(0, 25.0d);
+
+        SpanNode sn5 = new Span(25, 3);
+        branch3.add(sn5);
+
+        root.add(branch);
+
+        // Never bother. Just for debugging.
+        if (debug) {
+            System.out.println("===========NodeSet2 ================");
+            consumeAnnotations(tree, root);
+        }
+
+        assertEquals(0, root.getFrom());
+        assertEquals(6, root.getTo());
+        assertEquals(0, branch.getFrom());
+        assertEquals(6, branch.getTo());
+        assertEquals(6, branch2.getFrom());
+        assertEquals(14, branch2.getTo());
+        assertEquals(25, branch2.getFrom(1));
+        assertEquals(28, branch2.getTo(1));
+        assertEquals(25, branch3.getFrom());
+        assertEquals(28, branch3.getTo());
+        assertEquals(15, branch3.getFrom(1));
+        assertEquals(25, branch3.getTo(1));
+        assertFalse ((branch.getNumSubTrees() != branch2.getNumSubTrees()) ||
+            (branch.getNumSubTrees() != branch3.getNumSubTrees()));
+
+        branch3.sortSubTreesByProbability();
+        assertEquals(15, branch3.getFrom());
+        assertEquals(25, branch3.getTo());
+        assertEquals(25, branch3.getFrom(1));
+        assertEquals(28, branch3.getTo(1));
+    }
+
+
+    @After
+    public void removeTree() {
+        tree = null;
+    }
+
+    public void consumeAnnotations(SpanTree tree, SpanList root) {
+        if (debug) System.out.println("\n\nSpanList: [" + root.getFrom() + ", " + root.getTo() + "] num Children: " + root.numChildren());
+        if (debug) System.out.println("-------------------");
+        Iterator<SpanNode> childIterator = root.childIterator();
+        while (childIterator.hasNext()) {
+            SpanNode node = childIterator.next();
+            if (debug) System.out.println("\n\nSpan Node (" + node + "): [" + node.getFrom() + ", " + node.getTo() + "] ");
+            if (node instanceof AlternateSpanList) {
+                parseAlternateLists(tree, (AlternateSpanList)node);
+                if (debug) System.out.println("---- Alternate SpanList complete ---");
+            } else if (node instanceof SpanList) {
+                if (debug) System.out.println("Encountered another span list");
+                SpanList spl = (SpanList) node;
+                ListIterator<SpanNode> lli = spl.childIterator();
+                while (lli.hasNext()) System.out.print(" " + lli.next() + " ");
+                consumeAnnotations(tree, (SpanList) node);
+            } else {
+                   if (debug) System.out.println("\nGetting annotations for this span node: [" + node.getFrom() + ", " + node.getTo() + "] ");
+                   getAnnotationsForNode(tree, node);
+            }
+        }
+        if (debug) System.out.println("\nGetting annotations for the SpanList itself : [" + root.getFrom() + ", " + root.getTo() + "] ");
+        getAnnotationsForNode(tree, root);
+    }
+
+    public void parseAlternateLists(SpanTree tree, AlternateSpanList aspl) {
+        int no = aspl.getNumSubTrees();
+        if (debug) System.out.println("Parsing Alternate span list. No of subtrees: " + no);
+        int ctr = 0;
+        while (ctr < no) {
+            if (debug) System.out.println("\nSubTree: " + ctr);
+            ListIterator<SpanNode> lIter = aspl.childIteratorRecursive(ctr);
+            while (lIter.hasNext()) {
+                SpanNode spnNode = lIter.next();
+                if (debug) System.out.println("Parsing span node: [" + spnNode.getFrom() + ", " + spnNode.getTo() + "] ");
+                if (spnNode instanceof AlternateSpanList) {
+                    if (debug) System.out.println("A child alternate span list found. Recursing");
+                    parseAlternateLists(tree, (AlternateSpanList)spnNode);
+                }
+
+                getAnnotationsForNode(tree, spnNode);
+            }
+            ctr ++;
+        }
+    }
+
+    public void getAnnotationsForNode(SpanTree tree, SpanNode node) {
+        Iterator<Annotation> iter = tree.iterator(node);
+        boolean annotationPresent = false;
+        while (iter.hasNext()) {
+            annotationPresent = true;
+            Annotation xx = iter.next();
+            StringFieldValue fValue = (StringFieldValue) xx.getFieldValue();
+            if (debug) System.out.println("Annotation: " + xx);
+            if (fValue == null) {
+                if (debug) System.out.println("Field Value is null");
+                return;
+            } else {
+                if (debug) System.out.println("Field Value: " + fValue.getString());
+            }
+        }
+        if (!annotationPresent) {
+            if (debug) System.out.println("****No annotations found for the span node: [" + node.getFrom() + ", " + node.getTo() + "] ");
+        }
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/annotation/AlternateSpanListTestCase.java b/document/src/test/java/com/yahoo/document/annotation/AlternateSpanListTestCase.java
new file mode 100755
index 00000000000..f6131a42e8a
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/annotation/AlternateSpanListTestCase.java
@@ -0,0 +1,250 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.document.serialization.*;
+import com.yahoo.io.GrowableByteBuffer;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.ListIterator;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertSame;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class AlternateSpanListTestCase extends AbstractTypesTest {
+
+    @Test
+    public void testSerializeDeserialize() {
+        {
+            AlternateSpanList alternateSpanList = new AlternateSpanList();
+            serializeAndAssert(alternateSpanList);
+        }
+        {
+            AlternateSpanList alternateSpanList = new AlternateSpanList();
+            Span s1 = new Span(1, 2);
+            Span s2 = new Span(3, 4);
+            Span s3 = new Span(4, 5);
+            alternateSpanList.add(s1).add(s2).add(s3);
+            AlternateSpanList s4 = new AlternateSpanList();
+            Span s5 = new Span(7, 8);
+            Span s6 = new Span(8, 9);
+            s4.add(s5).add(s6);
+            alternateSpanList.add(s4);
+
+            List<SpanNode> alternateChildren = new ArrayList<>();
+            Span s7 = new Span(1, 4);
+            Span s8 = new Span(1, 9);
+            Span s9 = new Span(5, 10);
+            alternateChildren.add(s7);
+            alternateChildren.add(s8);
+            alternateChildren.add(s9);
+
+            alternateSpanList.addChildren(alternateChildren, 5.55);
+
+            serializeAndAssert(alternateSpanList);
+        }
+    }
+
+    private void serializeAndAssert(AlternateSpanList alternateSpanList) {
+        GrowableByteBuffer buffer;
+        {
+            buffer = new GrowableByteBuffer(1024);
+            DocumentSerializer serializer = DocumentSerializerFactory.create42(buffer);
+            StringFieldValue value = new StringFieldValue("lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj lk");
+            SpanTree tree = new SpanTree("bababa", alternateSpanList);
+            value.setSpanTree(tree);
+            serializer.write(null, value);
+            buffer.flip();
+        }
+        AlternateSpanList alternateSpanList2;
+        {
+            DocumentDeserializer deserializer = DocumentDeserializerFactory.create42(man, buffer);
+            StringFieldValue value = new StringFieldValue();
+            deserializer.read(null, value);
+            alternateSpanList2 = (AlternateSpanList)value.getSpanTree("bababa").getRoot();
+        }
+
+        assertEquals(alternateSpanList, alternateSpanList2);
+        assertNotSame(alternateSpanList, alternateSpanList2);
+    }
+
+    @Test
+    public void testToString() {
+        SpanList root = new SpanList();
+        AlternateSpanList branch = new AlternateSpanList();
+
+        SpanNode sn1 = new Span(0, 5);
+        SpanNode span1 = new Span(0, 3);
+        root.add(sn1);
+        branch.add(span1);
+        root.add(branch);
+
+        SpanNode span11 = new Span(0, 3);
+        SpanNode span22 = new Span(1, 9);
+        SpanNode span33 = new Span(12, 10);
+
+        SpanList alternate = new SpanList();
+        alternate.add(span11);
+        alternate.add(span22);
+        alternate.add(span33);
+
+        List<SpanNode> subtreeList = new ArrayList<>();
+        subtreeList.add(alternate);
+
+        branch.setProbability(0, 100.0d);
+        branch.addChildren(subtreeList, 50.0d);
+
+        assertNotNull(root.toString());
+        assertNotNull(branch.toString());
+    }
+
+    @Test
+    public void testSortRecursive() {
+        AlternateSpanList root = new AlternateSpanList();
+        Span a1 = new Span(0, 1);
+        SpanList b1 = new SpanList();
+        SpanList c1 = new SpanList();
+        Span d1 = new Span(9, 1);
+        root.add(d1).add(c1).add(a1).add(b1);
+
+        Span aB2 = new Span(1, 1);
+        Span bB2 = new Span(2, 1);
+        Span cB2 = new Span(3, 1);
+        Span dB2 = new Span(4, 1);
+        b1.add(dB2).add(cB2).add(bB2).add(aB2);
+
+        Span aC2 = new Span(5, 1);
+        Span bC2 = new Span(6, 1);
+        Span cC2 = new Span(7, 1);
+        Span dC2 = new Span(8, 1);
+        c1.add(cC2).add(aC2).add(bC2).add(dC2);
+
+        Span altA1 = new Span(0, 1);
+        Span altB1 = new Span(1, 1);
+        List<SpanNode> altList = new ArrayList<>(2);
+        altList.add(altB1);
+        altList.add(altA1);
+        root.addChildren(1, altList, 0.5);
+
+        root.sortChildrenRecursive();
+        assertSame(a1, root.children().get(0));
+        assertSame(b1, root.children().get(1));
+        assertSame(c1, root.children().get(2));
+        assertSame(d1, root.children().get(3));
+        assertSame(aB2, b1.children().get(0));
+        assertSame(bB2, b1.children().get(1));
+        assertSame(cB2, b1.children().get(2));
+        assertSame(dB2, b1.children().get(3));
+        assertSame(aC2, c1.children().get(0));
+        assertSame(bC2, c1.children().get(1));
+        assertSame(cC2, c1.children().get(2));
+        assertSame(dC2, c1.children().get(3));
+
+        assertSame(altA1, root.children(1).get(0));
+        assertSame(altB1, root.children(1).get(1));
+    }
+
+    @Test
+    public void testIterator() {
+        AlternateSpanList asl1 = new AlternateSpanList();
+
+        Span span10 = new Span(1, 2);
+        Span span20 = new Span(2, 3);
+        Span span30 = new Span(3, 4);
+        List<SpanNode> subTree0 = new ArrayList<>(3);
+        subTree0.add(span10);
+        subTree0.add(span20);
+        subTree0.add(span30);
+
+        Span span11 = new Span(1, 2);
+        Span span21 = new Span(2, 3);
+        Span span31 = new Span(3, 4);
+        List<SpanNode> subTree1 = new ArrayList<>(3);
+        subTree1.add(span11);
+        subTree1.add(span21);
+        subTree1.add(span31);
+
+        Span span12 = new Span(1, 2);
+        Span span22 = new Span(2, 3);
+        Span span32 = new Span(3, 4);
+        List<SpanNode> subTree2 = new ArrayList<>(3);
+        subTree2.add(span12);
+        subTree2.add(span22);
+        subTree2.add(span32);
+
+        asl1.addChildren(0, subTree0, 0.1);
+        asl1.addChildren(1, subTree1, 0.1);
+        asl1.addChildren(2, subTree2, 0.1);
+
+        ListIterator<SpanNode> it = asl1.childIterator();
+        assertSame(span10, it.next());
+        assertSame(span20, it.next());
+        assertSame(span30, it.next());
+        assertSame(span11, it.next());
+        assertSame(span21, it.next());
+        assertSame(span31, it.next());
+        assertSame(span12, it.next());
+        assertSame(span22, it.next());
+        assertSame(span32, it.next());
+        assertFalse(it.hasNext());
+    }
+
+    @Test
+    public void testIteratorRecursive() {
+        AlternateSpanList asl1 = new AlternateSpanList();
+
+        Span span10 = new Span(1, 1);
+        Span span20 = new Span(2, 1);
+        Span span30 = new Span(3, 1);
+        List<SpanNode> subTree0 = new ArrayList<>(3);
+        subTree0.add(span10);
+        subTree0.add(span20);
+        subTree0.add(span30);
+
+        Span span11 = new Span(4, 1);
+        Span span21 = new Span(5, 1);
+        Span span31 = new Span(6, 1);
+        SpanList sl112 = new SpanList();
+        sl112.add(span11);
+        sl112.add(span21);
+        sl112.add(span31);
+        SpanList sl11 = new SpanList();
+        sl11.add(sl112);
+        List<SpanNode> subTree1 = new ArrayList<>(1);
+        subTree1.add(sl11);
+
+        Span span12 = new Span(7, 1);
+        Span span22 = new Span(8, 1);
+        Span span32 = new Span(9, 1);
+        List<SpanNode> subTree2 = new ArrayList<>(3);
+        subTree2.add(span12);
+        subTree2.add(span22);
+        subTree2.add(span32);
+
+        asl1.addChildren(0, subTree0, 0.1);
+        asl1.addChildren(1, subTree1, 0.1);
+        asl1.addChildren(2, subTree2, 0.1);
+
+        ListIterator<SpanNode> it = asl1.childIteratorRecursive();
+        assertSame(span10, it.next());
+        assertSame(span20, it.next());
+        assertSame(span30, it.next());
+        assertSame(span11, it.next());
+        assertSame(span21, it.next());
+        assertSame(span31, it.next());
+        assertSame(sl112, it.next());
+        assertSame(sl11, it.next());
+        assertSame(span12, it.next());
+        assertSame(span22, it.next());
+        assertSame(span32, it.next());
+        assertFalse(it.hasNext());
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/annotation/AnnotationTestCase.java b/document/src/test/java/com/yahoo/document/annotation/AnnotationTestCase.java
new file mode 100644
index 00000000000..65ae706ec9a
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/annotation/AnnotationTestCase.java
@@ -0,0 +1,129 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.datatypes.IntegerFieldValue;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.document.datatypes.Struct;
+import com.yahoo.document.serialization.*;
+import com.yahoo.io.GrowableByteBuffer;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotSame;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class AnnotationTestCase extends AbstractTypesTest {
+
+    @Test
+    public void testBasic() {
+        AnnotationType begTagType = new AnnotationType("begin_tag");
+        Annotation a = new Annotation(begTagType);
+        Annotation b = new Annotation(begTagType);
+
+        assertEquals(a, b);
+        assertEquals(a.hashCode(), b.hashCode());
+        assertNotSame(a, b);
+
+        Annotation c = new Annotation(new AnnotationType("determiner"));
+
+        assertFalse(a.equals(c));
+        assertFalse(c.equals(a));
+        assertFalse(b.equals(c));
+        assertFalse(c.equals(b));
+
+        assertFalse(a.hashCode() == c.hashCode());
+        assertFalse(c.hashCode() == a.hashCode());
+        assertFalse(b.hashCode() == c.hashCode());
+        assertFalse(c.hashCode() == b.hashCode());
+    }
+
+    @Test
+    public void testFieldValues() {
+        AnnotationType atype = new AnnotationType("foobar", DataType.STRING);
+        StringFieldValue sfv = new StringFieldValue("balloo");
+
+        Annotation a = new Annotation(atype);
+        a.setFieldValue(sfv);
+    }
+
+    @Test
+    public void testSerializeDeserialize() {
+        {
+            Annotation annotation = new Annotation(dummy);
+            serializeAndAssert(annotation);
+        }
+        {
+            Annotation annotation = new Annotation(number, new IntegerFieldValue(56));
+            serializeAndAssert(annotation);
+        }
+        {
+            Struct value = new Struct(person);
+            value.setFieldValue("firstname", "Barack");
+            value.setFieldValue("lastname", "Obama");
+            value.setFieldValue("birthyear", 1909);
+            Annotation annotation = new Annotation(personA, value);
+            serializeAndAssert(annotation);
+        }
+    }
+
+    /**
+     * A test case taken from real use to verify the API ease of use
+     */
+    @Test
+    public void testApi() {
+        // Prepare
+        AnnotationType type1 = new AnnotationType("type1", DataType.STRING);
+        AnnotationType type2 = new AnnotationType("type2", DataType.INT);
+        StringFieldValue output = new StringFieldValue("foo bar");
+        SpanTree tree;
+
+        // no shortcuts
+        {
+            SpanList root = new SpanList();
+            tree = new SpanTree("SpanTree1", root);
+            SpanNode node = new Span(0, 3);
+            tree.annotate(node, new Annotation(type1, new StringFieldValue("text")));
+            tree.annotate(node, new Annotation(type2, new IntegerFieldValue(1)));
+            root.add(node);
+            output.setSpanTree(tree);
+        }
+
+        // short
+        {
+            SpanList root = new SpanList();
+            output.setSpanTree(new SpanTree("SpanTree2", root));
+            SpanNode node = root.add(new Span(0, 3));
+            node.annotate(type1, "text").annotate(type2, 1);
+        }
+
+        // shorter
+        {
+            SpanList root = output.setSpanTree(new SpanTree("SpanTree3")).spanList();
+            root.span(0, 3).annotate(type1, "text").annotate(type2, 1);
+        }
+
+        // shortest
+        {
+            output.setSpanTree(new SpanTree("SpanTree4")).spanList().span(0, 3).annotate(type1, "text")
+                  .annotate(type2, 1);
+        }
+    }
+
+    private void serializeAndAssert(Annotation annotation) {
+        GrowableByteBuffer buffer = new GrowableByteBuffer(1024);
+        DocumentSerializer serializer = DocumentSerializerFactory.create42(buffer);
+        serializer.write(annotation);
+        buffer.flip();
+
+        DocumentDeserializer deserializer = DocumentDeserializerFactory.create42(man, buffer);
+        Annotation annotation2 = new Annotation();
+        deserializer.read(annotation2);
+
+        assertEquals(annotation, annotation2);
+        assertNotSame(annotation, annotation2);
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/annotation/AnnotationTypeRegistryTestCase.java b/document/src/test/java/com/yahoo/document/annotation/AnnotationTypeRegistryTestCase.java
new file mode 100644
index 00000000000..273d30b5426
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/annotation/AnnotationTypeRegistryTestCase.java
@@ -0,0 +1,50 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+import com.yahoo.document.DataType;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class AnnotationTypeRegistryTestCase extends junit.framework.TestCase {
+    public void testRegisterUnregister() {
+        AnnotationTypeRegistry reg = new AnnotationTypeRegistry();
+        assertEquals(0, reg.getTypes().size());
+
+        AnnotationType one = new AnnotationType("one");
+        AnnotationType another = new AnnotationType("one");
+
+        //should work; re-registering type with same name and same id:
+        reg.register(one);
+        assertEquals(1, reg.getTypes().size());
+        reg.register(another);
+        assertEquals(1, reg.getTypes().size());
+
+        AnnotationType oneWithData = new AnnotationType("one", DataType.INT);
+
+        reg.register(oneWithData);
+        assertEquals(1, reg.getTypes().size());
+
+
+        AnnotationType two = new AnnotationType("two");
+        AnnotationType three = new AnnotationType("three");
+
+        reg.register(two);
+        assertEquals(2, reg.getTypes().size());
+        reg.register(three);
+        assertEquals(3, reg.getTypes().size());
+
+
+        reg.unregister("one");
+        assertEquals(2, reg.getTypes().size());
+        assertEquals("two", reg.getType("two").getName());
+        assertEquals("three", reg.getType("three").getName());
+
+        reg.unregister(two.getId());
+        assertEquals(1, reg.getTypes().size());
+        assertEquals("three", reg.getType("three").getName());
+
+        reg.unregister(three);
+        assertEquals(0, reg.getTypes().size());
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/annotation/AnnotationTypeTestCase.java b/document/src/test/java/com/yahoo/document/annotation/AnnotationTypeTestCase.java
new file mode 100644
index 00000000000..dcceb403564
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/annotation/AnnotationTypeTestCase.java
@@ -0,0 +1,66 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+import com.yahoo.document.DataType;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class AnnotationTypeTestCase extends junit.framework.TestCase {
+    public void testBasic() {
+        AnnotationType a = new AnnotationType("foo");
+        AnnotationType b = new AnnotationType("foo");
+
+        assertEquals(a, b);
+        assertEquals(a.hashCode(), b.hashCode());
+        assertEquals(a.hashCode(), a.getId());
+        assertEquals(b.hashCode(), b.getId());
+
+        AnnotationType c = new AnnotationType("bar");
+        assertEquals(c.hashCode(), c.getId());
+
+        assertFalse(a.equals(c));
+        assertFalse(c.equals(a));
+        assertFalse(b.equals(c));
+        assertFalse(c.equals(b));
+
+        assertFalse(a.hashCode() == c.hashCode());
+        assertFalse(c.hashCode() == a.hashCode());
+        assertFalse(b.hashCode() == c.hashCode());
+        assertFalse(c.hashCode() == b.hashCode());
+    }
+
+    public void testBasic2() {
+        AnnotationType a = new AnnotationType("foo", DataType.INT);
+        AnnotationType b = new AnnotationType("foo", DataType.INT);
+
+        assertEquals(a, b);
+        assertEquals(a.hashCode(), b.hashCode());
+        assertEquals(a.hashCode(), a.getId());
+        assertEquals(b.hashCode(), b.getId());
+
+        AnnotationType c = new AnnotationType("foo", DataType.FLOAT);
+        assertEquals(c.hashCode(), c.getId());
+
+        assertEquals(a, c);
+        assertEquals(a.hashCode(), c.hashCode());
+        assertEquals(a.hashCode(), a.getId());
+        assertEquals(c.hashCode(), c.getId());
+    }
+
+    public void testPolymorphy() {
+        AnnotationType suuper = new AnnotationType("super");
+        AnnotationType sub = new AnnotationType("sub");
+        sub.inherit(suuper);
+
+        //reference type for super annotation type
+        AnnotationReferenceDataType refType = new AnnotationReferenceDataType(suuper);
+
+        Annotation superAnnotation = new Annotation(suuper);
+        Annotation subAnnotation = new Annotation(sub);
+
+        AnnotationReference ref1 = new AnnotationReference(refType, superAnnotation);
+        //this would fail without polymorphy support:
+        AnnotationReference ref2 = new AnnotationReference(refType, subAnnotation);
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/annotation/AnnotationTypesTestCase.java b/document/src/test/java/com/yahoo/document/annotation/AnnotationTypesTestCase.java
new file mode 100644
index 00000000000..7a2d3d006b2
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/annotation/AnnotationTypesTestCase.java
@@ -0,0 +1,22 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+import com.yahoo.document.datatypes.DoubleFieldValue;
+import org.junit.Test;
+
+import static org.junit.Assert.fail;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a>
+ */
+public class AnnotationTypesTestCase {
+
+    @Test
+    public void requireThatProximityBreakAcceptsDoubleWeight() {
+        try {
+            new Annotation(AnnotationTypes.PROXIMITY_BREAK, new DoubleFieldValue(6.9));
+        } catch (Exception e) {
+            fail("this is required for ticket #665166, do not change");
+        }
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/annotation/Bug4155865TestCase.java b/document/src/test/java/com/yahoo/document/annotation/Bug4155865TestCase.java
new file mode 100644
index 00000000000..6cf9ee8f683
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/annotation/Bug4155865TestCase.java
@@ -0,0 +1,379 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.datatypes.StringFieldValue;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.ListIterator;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class Bug4155865TestCase {
+
+    private SpanTree tree;
+
+    @Before
+    public void buildTree() {
+        AnnotationType at1 = new AnnotationType("person", DataType.STRING);
+        AnnotationType at2 = new AnnotationType("street", DataType.STRING);
+        AnnotationType at3 = new AnnotationType("city", DataType.STRING);
+
+        StringFieldValue fv1 = (StringFieldValue) at1.getDataType().createFieldValue();
+        fv1.assign("Praveen");
+        Annotation an1 = new Annotation(at1, fv1);
+
+        StringFieldValue fv2 = (StringFieldValue) at2.getDataType().createFieldValue();
+        fv2.assign("Bommanahalli");
+        Annotation an2 = new Annotation(at2, fv2);
+
+        StringFieldValue fv3 = (StringFieldValue) at3.getDataType().createFieldValue();
+        fv3.assign("Bangalore");
+        Annotation an3 = new Annotation(at3, fv3);
+
+
+        StringFieldValue fv11 = (StringFieldValue) at1.getDataType().createFieldValue();
+        fv11.assign("Elancheran");
+        Annotation an11 = new Annotation(at1, fv11);
+
+        StringFieldValue fv22 = (StringFieldValue) at2.getDataType().createFieldValue();
+        fv22.assign("Kagadaspura");
+        Annotation an22 = new Annotation(at2, fv22);
+
+        StringFieldValue fv33 = (StringFieldValue) at3.getDataType().createFieldValue();
+        fv33.assign("Delhi");
+        Annotation an33 = new Annotation(at3, fv33);
+
+        StringFieldValue fv111 = (StringFieldValue) at1.getDataType().createFieldValue();
+        fv111.assign("Govindan");
+        Annotation an111 = new Annotation(at1, fv111);
+
+        StringFieldValue fv222 = (StringFieldValue) at1.getDataType().createFieldValue();
+        fv222.assign("Kenneth");
+        Annotation an222 = new Annotation(at2, fv222);
+
+        SpanList root = new SpanList();
+        tree = new SpanTree("test", root);
+        AlternateSpanList branch = new AlternateSpanList();
+        SpanList branch2 = new SpanList();
+
+
+        SpanNode sn1 = new Span(0, 5);
+        SpanNode sn2 = new Span(5, 10);
+        SpanNode sn3 = new Span(15, 10);
+        SpanNode sn4 = new Span(15, 20);
+
+        SpanNode span1 = new Span(0, 3);
+        SpanNode span2 = new Span(1, 9);
+        SpanNode span3 = new Span(12, 10);
+
+        root.add(sn4);
+        root.add(sn3);
+        root.add(sn2);
+        root.add(sn1);
+
+        SpanNode spn1 = new Span(4, 5);
+        branch2.add(spn1);
+
+        AlternateSpanList branch3 = new AlternateSpanList();
+
+        SpanNode spn2 = new Span(1, 4);
+        SpanNode spn3 = new Span(6, 10);
+        tree.annotate(spn2, an111);
+        tree.annotate(spn3, an222);
+
+        List<SpanNode> stList = new ArrayList<SpanNode>();
+        stList.add(spn2);
+        List<SpanNode> stList1 = new ArrayList<SpanNode>();
+        stList1.add(spn3);
+        branch3.addChildren(stList, 45.0);
+        branch3.addChildren(stList1, 25.0);
+
+        root.add(branch2);
+        branch.add(branch3);
+        root.add(branch);
+
+        SpanNode span11 = new Span(0, 3);
+        SpanNode span22 = new Span(1, 9);
+        SpanNode span33 = new Span(12, 10);
+
+        SpanList alternate2 = new SpanList();
+        alternate2.add(span3);
+        alternate2.add(span2);
+        alternate2.add(span1);
+
+        SpanList alternate1 = new SpanList();
+        alternate1.add(span11);
+        alternate1.add(span22);
+        alternate1.add(span33);
+
+        tree.annotate(span1, an1);
+        tree.annotate(span2, an2);
+        tree.annotate(span3, an3);
+
+        tree.annotate(span11, an11);
+        tree.annotate(span22, an22);
+        tree.annotate(span33, an33);
+
+        List<SpanNode> subtreeList1 = new ArrayList<SpanNode>();
+        subtreeList1.add(alternate1);
+
+        List<SpanNode> subtreeList2 = new ArrayList<SpanNode>();
+        subtreeList2.add(alternate2);
+
+        branch.addChildren(subtreeList1, 50.0d);
+        branch.addChildren(subtreeList2, 100.0d);
+    }
+
+    @Test
+    public void assertTree() {
+        final SpanList root = (SpanList) tree.getRoot();
+        //Level 0:
+        assertEquals(0, root.getFrom());
+        assertEquals(35, root.getTo());
+        assertEquals(6, root.numChildren());
+
+        //Level 1:
+        assertTrue(root.children().get(0) instanceof Span);
+        assertEquals(15, root.children().get(0).getFrom());
+        assertEquals(35, root.children().get(0).getTo());
+        assertFalse(root.children().get(0).childIterator().hasNext());
+        assertFalse(tree.iterator(root.children().get(0)).hasNext());
+
+        assertTrue(root.children().get(1) instanceof Span);
+        assertEquals(15, root.children().get(1).getFrom());
+        assertEquals(25, root.children().get(1).getTo());
+        assertFalse(root.children().get(1).childIterator().hasNext());
+        assertFalse(tree.iterator(root.children().get(1)).hasNext());
+
+        assertTrue(root.children().get(2) instanceof Span);
+        assertEquals(5, root.children().get(2).getFrom());
+        assertEquals(15, root.children().get(2).getTo());
+        assertFalse(root.children().get(2).childIterator().hasNext());
+        assertFalse(tree.iterator(root.children().get(2)).hasNext());
+
+        assertTrue(root.children().get(3) instanceof Span);
+        assertEquals(0, root.children().get(3).getFrom());
+        assertEquals(5, root.children().get(3).getTo());
+        assertFalse(root.children().get(3).childIterator().hasNext());
+        assertFalse(tree.iterator(root.children().get(3)).hasNext());
+
+        assertTrue(root.children().get(4) instanceof SpanList);
+        assertFalse(root.children().get(4) instanceof AlternateSpanList);
+        assertEquals(4, root.children().get(4).getFrom());
+        assertEquals(9, root.children().get(4).getTo());
+        assertTrue(root.children().get(4).childIterator().hasNext());
+        assertFalse(tree.iterator(root.children().get(4)).hasNext());
+
+        assertTrue(root.children().get(5) instanceof AlternateSpanList);
+        assertEquals(-1, root.children().get(5).getFrom());
+        assertEquals(-1, root.children().get(5).getTo());
+        assertTrue(root.children().get(5).childIterator().hasNext());
+        assertFalse(tree.iterator(root.children().get(5)).hasNext());
+
+
+        //Level 2:
+        final SpanList list1 = (SpanList) root.children().get(4);
+        assertFalse(list1 instanceof AlternateSpanList);
+        assertEquals(1, list1.numChildren());
+
+        assertTrue(list1.children().get(0) instanceof Span);
+        assertEquals(4, list1.children().get(0).getFrom());
+        assertEquals(9, list1.children().get(0).getTo());
+        assertFalse(list1.children().get(0).childIterator().hasNext());
+        assertFalse(tree.iterator(list1.children().get(0)).hasNext());
+
+        final AlternateSpanList altList1 = (AlternateSpanList) root.children().get(5);
+        assertEquals(3, altList1.getNumSubTrees());
+
+        List<SpanNode> subTree0 = altList1.children(0);
+        assertEquals(1, subTree0.size());
+
+        assertTrue(subTree0.get(0) instanceof AlternateSpanList); //TODO: Assert on this!!
+        assertEquals(3, ((AlternateSpanList) subTree0.get(0)).getNumSubTrees());
+        List<SpanNode> subTree0_0 = ((AlternateSpanList) subTree0.get(0)).children(0);
+        assertEquals(0, subTree0_0.size());
+        List<SpanNode> subTree0_1 = ((AlternateSpanList) subTree0.get(0)).children(1);
+        assertEquals(1, subTree0_1.size());
+        List<SpanNode> subTree0_2 = ((AlternateSpanList) subTree0.get(0)).children(2);
+        assertEquals(1, subTree0_2.size());
+
+        assertEquals(-1, subTree0.get(0).getFrom());
+        assertEquals(-1, subTree0.get(0).getTo());
+        assertTrue(subTree0.get(0).childIterator().hasNext());
+        assertFalse(tree.iterator(subTree0.get(0)).hasNext());
+
+        List<SpanNode> subTree1 = altList1.children(1);
+        assertEquals(1, subTree1.size());
+
+        assertTrue(subTree1.get(0) instanceof SpanList);
+        assertFalse(subTree1.get(0) instanceof AlternateSpanList);
+        assertEquals(0, subTree1.get(0).getFrom());
+        assertEquals(22, subTree1.get(0).getTo());
+        assertTrue(subTree1.get(0).childIterator().hasNext());
+        assertFalse(tree.iterator(subTree1.get(0)).hasNext());
+
+        List<SpanNode> subTree2 = altList1.children(2);
+        assertEquals(1, subTree2.size());
+
+        assertTrue(subTree2.get(0) instanceof SpanList);
+        assertFalse(subTree2.get(0) instanceof AlternateSpanList);
+        assertEquals(0, subTree2.get(0).getFrom());
+        assertEquals(22, subTree2.get(0).getTo());
+        assertTrue(subTree2.get(0).childIterator().hasNext());
+        assertFalse(tree.iterator(subTree2.get(0)).hasNext());
+
+        //NOTE subTree2 has children
+
+
+        //Level 3
+        assertTrue(subTree0_0.isEmpty());
+
+        final Span subTree0_1_0 = (Span) subTree0_1.get(0);
+        assertEquals(1, subTree0_1_0.getFrom());
+        assertEquals(5, subTree0_1_0.getTo());
+        assertFalse(subTree0_1_0.childIterator().hasNext());
+        final Iterator<Annotation> subTree0_1_0AnnIterator = tree.iterator(subTree0_1_0);
+        assertTrue(subTree0_1_0AnnIterator.hasNext());
+        Annotation subTree0_1_0Annotation = subTree0_1_0AnnIterator.next();
+        //TODO: Assert on annotation
+        assertFalse(subTree0_1_0AnnIterator.hasNext());
+
+
+        final Span subTree0_2_0 = (Span) subTree0_2.get(0);
+        assertEquals(6, subTree0_2_0.getFrom());
+        assertEquals(16, subTree0_2_0.getTo());
+        assertFalse(subTree0_2_0.childIterator().hasNext());
+        final Iterator<Annotation> subTree0_2_0AnnIterator = tree.iterator(subTree0_2_0);
+        assertTrue(subTree0_2_0AnnIterator.hasNext());
+        Annotation subTree0_2_0Annotation = subTree0_2_0AnnIterator.next();
+        //TODO: Assert on annotation
+        assertFalse(subTree0_2_0AnnIterator.hasNext());
+
+
+        final SpanList sl = (SpanList) subTree1.get(0);
+        assertFalse(sl instanceof AlternateSpanList);
+        assertEquals(3, sl.children().size());
+
+        assertTrue(sl.children().get(0) instanceof Span);
+        assertEquals(0, sl.children().get(0).getFrom());
+        assertEquals(3, sl.children().get(0).getTo());
+        assertFalse(sl.children().get(0).childIterator().hasNext());
+        final Iterator<Annotation> iterator0 = tree.iterator(sl.children().get(0));
+        assertTrue(iterator0.hasNext());
+        Annotation iterator0Annotation = iterator0.next();
+        //TODO: Assert on annotation
+        assertFalse(iterator0.hasNext());
+
+        assertTrue(sl.children().get(1) instanceof Span);
+        assertEquals(1, sl.children().get(1).getFrom());
+        assertEquals(10, sl.children().get(1).getTo());
+        assertFalse(sl.children().get(1).childIterator().hasNext());
+        final Iterator<Annotation> iterator1 = tree.iterator(sl.children().get(1));
+        assertTrue(iterator1.hasNext());
+        Annotation iterator1Annotation = iterator1.next();
+        //TODO: Assert on annotation
+        assertFalse(iterator1.hasNext());
+
+        assertTrue(sl.children().get(2) instanceof Span);
+        assertEquals(12, sl.children().get(2).getFrom());
+        assertEquals(22, sl.children().get(2).getTo());
+        assertFalse(sl.children().get(2).childIterator().hasNext());
+        final Iterator<Annotation> iterator2 = tree.iterator(sl.children().get(2));
+        assertTrue(iterator2.hasNext());
+        Annotation iterator2Annotation = iterator2.next();
+        //TODO: Assert on annotation
+        assertFalse(iterator2.hasNext());
+
+        final SpanList sl2 = (SpanList) subTree2.get(0);
+        assertFalse (sl2 instanceof AlternateSpanList);
+        assertEquals(3, sl2.children().size());
+
+        assertTrue(sl2.children().get(0) instanceof Span);
+        assertEquals(12, sl2.children().get(0).getFrom());
+        assertEquals(22, sl2.children().get(0).getTo());
+        assertFalse(sl2.children().get(0).childIterator().hasNext());
+        final Iterator<Annotation> iterator3 = tree.iterator(sl2.children().get(0));
+        assertTrue(iterator3.hasNext());
+        Annotation iterator3Annotation = iterator3.next();
+        //TODO: Assert on annotation
+        assertFalse(iterator3.hasNext());
+
+        assertTrue(sl2.children().get(1) instanceof Span);
+        assertEquals(1, sl2.children().get(1).getFrom());
+        assertEquals(10, sl2.children().get(1).getTo());
+        assertFalse(sl2.children().get(1).childIterator().hasNext());
+        final Iterator<Annotation> iterator4 = tree.iterator(sl2.children().get(1));
+        assertTrue(iterator4.hasNext());
+        Annotation iterator4Annotation = iterator4.next();
+        //TODO: Assert on annotation
+        assertFalse(iterator4.hasNext());
+
+        assertTrue(sl2.children().get(2) instanceof Span);
+        assertEquals(0, sl2.children().get(2).getFrom());
+        assertEquals(3, sl2.children().get(2).getTo());
+        assertFalse(sl2.children().get(2).childIterator().hasNext());
+        final Iterator<Annotation> iterator5 = tree.iterator(sl2.children().get(2));
+        assertTrue(iterator5.hasNext());
+        Annotation iterator5Annotation = iterator5.next();
+        //TODO: Assert on annotation
+        assertFalse(iterator5.hasNext());
+
+    }
+
+    @After
+    public void removeTree() {
+        tree = null;
+    }
+
+    public void parseAlternateLists(SpanTree tree, AlternateSpanList aspl) {
+        int no = aspl.getNumSubTrees();
+        System.out.println("Parsing Alternate span list. No of subtrees: " + no);
+        int ctr = 0;
+        while (ctr < no) {
+            System.out.println("\nSubTree: " + ctr);
+            ListIterator<SpanNode> lIter = aspl.childIteratorRecursive(ctr);
+            while (lIter.hasNext()) {
+                SpanNode spnNode = lIter.next();
+                System.out.println("Parsing span node: [" + spnNode.getFrom() + ", " + spnNode.getTo() + "] ");
+                if (spnNode instanceof AlternateSpanList) {
+                    System.out.println("A child alternate span list found. Recursing");
+                    parseAlternateLists(tree, (AlternateSpanList)spnNode);
+                }
+
+                getAnnotationsForNode(tree, spnNode);
+            }
+            ctr ++;
+        }
+    }
+
+    public void getAnnotationsForNode(SpanTree tree, SpanNode node) {
+        Iterator<Annotation> iter = tree.iterator(node);
+        boolean annotationPresent = false;
+        while (iter.hasNext()) {
+            annotationPresent = true;
+            Annotation xx = iter.next();
+            StringFieldValue fValue = (StringFieldValue) xx.getFieldValue();
+            System.out.println("Annotation: " + xx);
+            if (fValue == null) {
+                System.out.println("Field Value is null");
+                return;
+            } else {
+                System.out.println("Field Value: " + fValue.getString());
+            }
+        }
+        if (!annotationPresent) {
+            System.out.println("****No annotations found for the span node: [" + node.getFrom() + ", " + node.getTo() + "] ");
+        }
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/annotation/Bug4164299TestCase.java b/document/src/test/java/com/yahoo/document/annotation/Bug4164299TestCase.java
new file mode 100644
index 00000000000..3a88fb98f48
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/annotation/Bug4164299TestCase.java
@@ -0,0 +1,140 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+import com.yahoo.document.datatypes.StringFieldValue;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.ListIterator;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author <a href="mailto:mpraveen@yahoo-inc.com">Praveen Mohan</a>
+ *
+ * This test checks if sub-trees are sorted appropriately by probability
+ * within an alternate span list and getFrom() and getTo() are updated properly
+ * when the trees are sorted.
+ *
+ */
+public class Bug4164299TestCase {
+
+    private SpanTree tree;
+
+    @Before
+    public void buildTree() {
+        SpanList root = new SpanList();
+        AlternateSpanList branch = new AlternateSpanList();
+        tree = new SpanTree("test", root);
+
+        SpanNode span1 = new Span(0, 2);
+        SpanNode span2 = new Span(2, 2);
+        SpanNode span3 = new Span(4, 2);
+
+        SpanNode span11 = new Span(10, 2);
+        SpanNode span22 = new Span(12, 2);
+        SpanNode span33 = new Span(14, 2);
+
+        branch.add(span3);
+        branch.add(span2);
+        branch.add(span1);
+
+        List<SpanNode> subtreeList = new ArrayList<SpanNode>();
+        subtreeList.add(span11);
+        subtreeList.add(span22);
+        subtreeList.add(span33);
+        branch.addChildren(1, subtreeList, 50.0d);
+        root.add(branch);
+
+    }
+
+    @Test
+    public void assertTree() {
+        final AlternateSpanList branch =  (AlternateSpanList)((SpanList)tree.getRoot()).children().get(0);
+        assertEquals(0, branch.getFrom());
+        assertEquals(6, branch.getTo());
+        assertEquals(10, branch.getFrom(1));
+        assertEquals(16, branch.getTo(1));
+        branch.sortSubTreesByProbability();
+
+        assertEquals(10, branch.getFrom());
+        assertEquals(16, branch.getTo());
+        assertEquals(0, branch.getFrom(1));
+        assertEquals(6, branch.getTo(1));
+    }
+
+    @After
+    public void removeTree() {
+        tree = null;
+    }
+
+    public void consumeAnnotations(SpanTree tree, SpanList root) {
+        System.out.println("\n\nSpanList: [" + root.getFrom() + ", " + root.getTo() + "] num Children: " + root.numChildren());
+        System.out.println("-------------------");
+        Iterator<SpanNode> childIterator = root.childIterator();
+        while (childIterator.hasNext()) {
+            SpanNode node = childIterator.next();
+            System.out.println("\n\nSpan Node (" + node + "): [" + node.getFrom() + ", " + node.getTo() + "] ");
+            if (node instanceof AlternateSpanList) {
+                parseAlternateLists(tree, (AlternateSpanList)node);
+                System.out.println("---- Alternate SpanList complete ---");
+            } else if (node instanceof SpanList) {
+                System.out.println("Encountered another span list");
+                SpanList spl = (SpanList) node;
+                ListIterator<SpanNode> lli = spl.childIterator();
+                while (lli.hasNext()) System.out.print(" " + lli.next() + " ");
+                consumeAnnotations(tree, (SpanList) node);
+            } else {
+                   System.out.println("\nGetting annotations for this span node: [" + node.getFrom() + ", " + node.getTo() + "] ");
+                   getAnnotationsForNode(tree, node);
+            }
+        }
+        System.out.println("\nGetting annotations for the SpanList itself : [" + root.getFrom() + ", " + root.getTo() + "] ");
+        getAnnotationsForNode(tree, root);
+    }
+
+    public void parseAlternateLists(SpanTree tree, AlternateSpanList aspl) {
+        int no = aspl.getNumSubTrees();
+        System.out.println("Parsing Alternate span list. No of subtrees: " + no);
+        int ctr = 0;
+        while (ctr < no) {
+            System.out.println("\nSubTree: " + ctr);
+            ListIterator<SpanNode> lIter = aspl.childIteratorRecursive(ctr);
+            while (lIter.hasNext()) {
+                SpanNode spnNode = lIter.next();
+                System.out.println("Parsing span node: [" + spnNode.getFrom() + ", " + spnNode.getTo() + "] ");
+                if (spnNode instanceof AlternateSpanList) {
+                    System.out.println("A child alternate span list found. Recursing");
+                    parseAlternateLists(tree, (AlternateSpanList)spnNode);
+                }
+
+                getAnnotationsForNode(tree, spnNode);
+            }
+            ctr ++;
+        }
+    }
+
+    public void getAnnotationsForNode(SpanTree tree, SpanNode node) {
+        Iterator<Annotation> iter = tree.iterator(node);
+        boolean annotationPresent = false;
+        while (iter.hasNext()) {
+            annotationPresent = true;
+            Annotation xx = iter.next();
+            StringFieldValue fValue = (StringFieldValue) xx.getFieldValue();
+            System.out.println("Annotation: " + xx);
+            if (fValue == null) {
+                System.out.println("Field Value is null");
+                return;
+            } else {
+                System.out.println("Field Value: " + fValue.getString());
+            }
+        }
+        if (!annotationPresent) {
+            System.out.println("****No annotations found for the span node: [" + node.getFrom() + ", " + node.getTo() + "] ");
+        }
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/annotation/Bug4259784TestCase.java b/document/src/test/java/com/yahoo/document/annotation/Bug4259784TestCase.java
new file mode 100644
index 00000000000..035f6a95a19
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/annotation/Bug4259784TestCase.java
@@ -0,0 +1,129 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+import com.yahoo.document.ArrayDataType;
+import com.yahoo.document.Document;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.DocumentTypeManager;
+import com.yahoo.document.DocumentTypeManagerConfigurer;
+import com.yahoo.document.Field;
+import com.yahoo.document.StructDataType;
+import com.yahoo.document.datatypes.Array;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.document.datatypes.Struct;
+import com.yahoo.io.GrowableByteBuffer;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class Bug4259784TestCase extends junit.framework.TestCase {
+
+    @Test
+    public void testSerialize() {
+        DocumentTypeManager manager = new DocumentTypeManager();
+        DocumentTypeManagerConfigurer.configure(manager, "file:src/test/java/com/yahoo/document/annotation/documentmanager.bug4259784.cfg");
+
+        DocumentType type = manager.getDocumentType("blog");
+        Document doc = new Document(type, "doc:this:is:a:test");
+        doc.setFieldValue("body", new StringFieldValue("bla bla bla bla bla bla bla" +
+                                                       "bla bla bla bla bla bla bla"));
+        annotate(doc, manager);
+
+        GrowableByteBuffer buf = new GrowableByteBuffer();
+        doc.serialize(buf);
+    }
+
+
+    private void annotate(Document document, DocumentTypeManager manager) {
+        AnnotationTypeRegistry registry = manager.getAnnotationTypeRegistry();
+
+		AnnotationType company = registry.getType("company");
+		AnnotationType industry = registry.getType("industry");
+		AnnotationType person  = registry.getType("person");
+		AnnotationType location = registry.getType("location");
+
+	    SpanTree tree = new SpanTree("testannotations");
+        SpanList root = (SpanList) tree.getRoot();
+
+	    SpanNode span1 = new Span(0,5);
+	    SpanNode span2 = new Span(5,10);
+	    SpanNode span3 = new Span(10,15);
+	    SpanNode span4 = new Span(15,20);
+	    SpanNode span5 = new Span(6,10);
+	    SpanNode span6 = new Span(8,4);
+        SpanNode span7 = new Span(4, 2);
+
+	    root.add(span1);
+	    root.add(span2);
+	    root.add(span4);
+        root.add(span5);
+        root.add(span6);
+
+        AlternateSpanList aspl = new AlternateSpanList();
+        aspl.add(span7);
+        List<SpanNode> subtree1 = new ArrayList<SpanNode>();
+        subtree1.add(span3);
+        aspl.addChildren(1, subtree1, 33.0d);
+
+        root.add(aspl);
+
+	    Struct personValue = (Struct) person.getDataType().createFieldValue();
+	    personValue.setFieldValue("name", "Richard Bair");
+	    Annotation personAn = new Annotation(person, personValue);
+	    tree.annotate(span1, personAn);
+
+		Struct companyValue = (Struct) company.getDataType().createFieldValue();
+		companyValue.setFieldValue("name", "Sun");
+        Annotation compAn = new Annotation(company, companyValue);
+        tree.annotate(span2, compAn);
+
+        Struct locationVal = new Struct(manager.getDataType("annotation.location"));
+		locationVal.setFieldValue("lat", 37.774929);
+		locationVal.setFieldValue("lon", -122.419415);
+        Annotation locAnnotation = new Annotation(location, locationVal);
+        tree.annotate(span3, locAnnotation);
+
+
+        Struct dirValue1 = new Struct(manager.getDataType("annotation.person"));
+        dirValue1.setFieldValue("name", "Jonathan Schwartz");
+        Annotation dirAnnotation1 = new Annotation(person, dirValue1);
+        tree.annotate(span5, dirAnnotation1);
+
+        Struct dirValue2 = new Struct(manager.getDataType("annotation.person"));
+        dirValue2.setFieldValue("name", "Scott Mcnealy");
+        Annotation dirAnnotation2 = new Annotation(person, dirValue2);
+        tree.annotate(span6, dirAnnotation2);
+
+
+        Struct indValue = new Struct(manager.getDataType("annotation.industry"));
+        indValue.setFieldValue("vertical", "Manufacturing");
+        Annotation indAn = new Annotation(industry, indValue);
+        tree.annotate(span4, indAn);
+
+
+        Field compLocField = ((StructDataType) company.getDataType()).getField("place");
+        AnnotationReferenceDataType annType = (AnnotationReferenceDataType) compLocField.getDataType();
+        FieldValue compLocFieldVal = new AnnotationReference(annType, locAnnotation);
+		companyValue.setFieldValue(compLocField, compLocFieldVal);
+		companyValue.setFieldValue("vertical", "software");
+
+
+
+		Field dirField = ((StructDataType) company.getDataType()).getField("directors");
+		Array<FieldValue> dirFieldVal = new Array<FieldValue>(dirField.getDataType());
+		AnnotationReferenceDataType annRefType = (AnnotationReferenceDataType) ((ArrayDataType) dirField.getDataType()).getNestedType();
+		dirFieldVal.add(new AnnotationReference(annRefType, dirAnnotation1));
+		dirFieldVal.add(new AnnotationReference(annRefType, dirAnnotation2));
+		companyValue.setFieldValue(dirField, dirFieldVal);
+
+        tree.clearAnnotations(span3);
+
+        StringFieldValue body = (StringFieldValue) document.getFieldValue(document.getDataType().getField("body"));
+		body.setSpanTree(tree);
+	    document.setFieldValue(document.getDataType().getField("body"), body);
+	}
+
+}
+
diff --git a/document/src/test/java/com/yahoo/document/annotation/Bug4261985TestCase.java b/document/src/test/java/com/yahoo/document/annotation/Bug4261985TestCase.java
new file mode 100644
index 00000000000..4d6d040bcf8
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/annotation/Bug4261985TestCase.java
@@ -0,0 +1,153 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+import com.yahoo.document.ArrayDataType;
+import com.yahoo.document.Document;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.DocumentTypeManager;
+import com.yahoo.document.DocumentTypeManagerConfigurer;
+import com.yahoo.document.Field;
+import com.yahoo.document.StructDataType;
+import com.yahoo.document.datatypes.Array;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.document.datatypes.Struct;
+import com.yahoo.io.GrowableByteBuffer;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class Bug4261985TestCase {
+
+    @Test
+    public void testAnnotate() {
+        DocumentTypeManager manager = new DocumentTypeManager();
+        DocumentTypeManagerConfigurer.configure(manager, "file:src/test/java/com/yahoo/document/annotation/documentmanager.bug4261985.cfg");
+
+        DocumentType type = manager.getDocumentType("blog");
+        Document doc = new Document(type, "doc:this:is:a:test");
+        doc.setFieldValue("body", new StringFieldValue("bla bla bla bla bla bla bla" +
+                                                       "bla bla bla bla bla bla bla"));
+        annotate(doc, manager);
+
+        GrowableByteBuffer buf = new GrowableByteBuffer();
+        doc.serialize(buf);
+    }
+
+    public void annotate(Document document, DocumentTypeManager manager) {
+        AnnotationTypeRegistry registry = manager.getAnnotationTypeRegistry();
+
+		AnnotationType company = registry.getType("company");
+		AnnotationType industry = registry.getType("industry");
+		AnnotationType person  = registry.getType("person");
+		AnnotationType location = registry.getType("location");
+		AnnotationType bigshots = registry.getType("bigshots");
+
+		if (company.inherits(industry)) {
+			System.out.println("Company Inherits Industry");
+		} else {
+			System.out.println("FAIL: COMPANY DOES NOT INHERIT INDUSTRY");
+            throw new RuntimeException("FAIL: COMPANY DOES NOT INHERIT INDUSTRY, though it does in SD file");
+		}
+
+	    SpanTree tree = new SpanTree("testannotations");
+        SpanList root = (SpanList) tree.getRoot();
+
+	    SpanNode span1 = new Span(0,5);
+	    SpanNode span2 = new Span(5,10);
+	    SpanNode span3 = new Span(10,15);
+	    SpanNode span4 = new Span(15,20);
+	    SpanNode span5 = new Span(6,10);
+	    SpanNode span6 = new Span(8,4);
+        SpanNode span7 = new Span(4, 2);
+        SpanNode span8 = new Span(12, 6);
+
+	    root.add(span1);
+	    root.add(span2);
+	    //root.add(span3);
+	    root.add(span4);
+        root.add(span5);
+        root.add(span6);
+        //root.add(span7);
+        root.add(span8);
+
+        AlternateSpanList aspl = new AlternateSpanList();
+        aspl.add(span7);
+        List<SpanNode> subtree1 = new ArrayList<SpanNode>();
+        subtree1.add(span3);
+        aspl.addChildren(1, subtree1, 33.0d);
+
+        root.add(aspl);
+
+	    Struct personValue = (Struct) person.getDataType().createFieldValue();
+	    personValue.setFieldValue("name", "Richard Bair");
+	    Annotation personAn = new Annotation(person, personValue);
+	    tree.annotate(span1, personAn);
+
+		Struct companyValue = (Struct) company.getDataType().createFieldValue();
+		companyValue.setFieldValue("name", "Sun");
+
+        Struct locationVal = new Struct(manager.getDataType("annotation.location"));
+		locationVal.setFieldValue("lat", 37.774929);
+		locationVal.setFieldValue("lon", -122.419415);
+        Annotation locAnnotation = new Annotation(location, locationVal);
+        Field compLocField = ((StructDataType) company.getDataType()).getField("place");
+		//FieldValue compLocFieldVal = new FieldValue(compLocField.getDataType());
+        AnnotationReferenceDataType annType = (AnnotationReferenceDataType) compLocField.getDataType();
+        FieldValue compLocFieldVal = null;
+        //if (scenario.equals("createFieldValue")) {
+         //   compLocFieldVal = annType.createFieldValue(new AnnotationReference(annType, locAnnotation));
+        //} else {
+            compLocFieldVal = new AnnotationReference(annType, locAnnotation);
+        //}
+		companyValue.setFieldValue(compLocField, compLocFieldVal);
+
+		companyValue.setFieldValue("vertical", "software");
+		Struct dirValue1 = new Struct(manager.getDataType("annotation.person"));
+	    dirValue1.setFieldValue("name", "Jonathan Schwartz");
+	    Annotation dirAnnotation1 = new Annotation(person, dirValue1);
+	    Struct dirValue2 = new Struct(manager.getDataType("annotation.person"));
+	    dirValue2.setFieldValue("name", "Scott Mcnealy");
+	    Annotation dirAnnotation2 = new Annotation(person, dirValue2);
+		Field dirField = ((StructDataType) company.getDataType()).getField("directors");
+		Array<FieldValue> dirFieldVal = new Array<FieldValue>(dirField.getDataType());
+		AnnotationReferenceDataType annRefType = (AnnotationReferenceDataType) ((ArrayDataType) dirField.getDataType()).getNestedType();
+		dirFieldVal.add(new AnnotationReference(annRefType, dirAnnotation1));
+		dirFieldVal.add(new AnnotationReference(annRefType, dirAnnotation2));
+		companyValue.setFieldValue(dirField, dirFieldVal);
+		Annotation compAn = new Annotation(company, companyValue);
+		tree.annotate(span2, compAn);
+
+		Struct bigshotsValue = (Struct) bigshots.getDataType().createFieldValue();
+		Field ceosField = ((StructDataType) bigshots.getDataType()).getField("ceos");
+		//FieldValue compLocFieldVal = new FieldValue(compLocField.getDataType());
+        AnnotationReferenceDataType annType1 = (AnnotationReferenceDataType) ceosField.getDataType();
+        FieldValue ceosFieldVal = new AnnotationReference(annType1, compAn);
+		bigshotsValue.setFieldValue(ceosField, ceosFieldVal);
+
+		Annotation bigshotsAn = new Annotation(bigshots, bigshotsValue);
+		tree.annotate(span8, bigshotsAn);
+
+		Field selfField = ((StructDataType) bigshots.getDataType()).getField("self");
+		AnnotationReferenceDataType annType2 = (AnnotationReferenceDataType) selfField.getDataType();
+		FieldValue selfFieldVal = new AnnotationReference(annType2, bigshotsAn);
+		bigshotsValue.setFieldValue(selfField, selfFieldVal);
+		bigshotsAn = new Annotation(bigshots, bigshotsValue);
+		tree.annotate(span8, bigshotsAn);
+
+	    tree.annotate(span3, locAnnotation);
+        tree.annotate(span5, dirAnnotation1);
+        tree.annotate(span6, dirAnnotation2);
+
+		Struct indValue = new Struct(manager.getDataType("annotation.industry"));
+		indValue.setFieldValue("vertical", "Manufacturing");
+		Annotation indAn = new Annotation(industry, indValue);
+		tree.annotate(span4, indAn);
+
+        StringFieldValue body = (StringFieldValue) document.getFieldValue(document.getDataType().getField("body"));
+		body.setSpanTree(tree);
+	    document.setFieldValue(document.getDataType().getField("body"), body);
+	}
+}
+
diff --git a/document/src/test/java/com/yahoo/document/annotation/Bug4475379TestCase.java b/document/src/test/java/com/yahoo/document/annotation/Bug4475379TestCase.java
new file mode 100755
index 00000000000..c75e7300773
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/annotation/Bug4475379TestCase.java
@@ -0,0 +1,151 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+import com.yahoo.document.Document;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.DocumentTypeManager;
+import com.yahoo.document.DocumentTypeManagerConfigurer;
+import com.yahoo.document.datatypes.FloatFieldValue;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.document.datatypes.Struct;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.junit.Assert.assertThat;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class Bug4475379TestCase {
+
+    @Test
+    public void testClone() {
+        DocumentTypeManager manager = new DocumentTypeManager();
+        DocumentTypeManagerConfigurer.configure(manager, "file:src/test/java/com/yahoo/document/annotation/documentmanager.bug4475379.cfg");
+
+        DocumentType type = manager.getDocumentType("blog");
+        Document doc = new Document(type, "doc:this:is:a:test");
+        doc.setFieldValue("body", new StringFieldValue(""));
+        annotate(manager, doc);
+
+        Document anotherDoc = doc.clone();
+        assertThat(doc, equalTo(anotherDoc));
+    }
+
+    public void annotate(DocumentTypeManager manager, Document document) {
+        AnnotationTypeRegistry registry = manager.getAnnotationTypeRegistry();
+
+        AnnotationType company = registry.getType("company");
+        AnnotationType industry = registry.getType("industry");
+        AnnotationType person = registry.getType("person");
+        AnnotationType location = registry.getType("location");
+
+        Annotation compAn1;
+        Annotation personAn1;
+        Annotation locAn1;
+        Annotation indAn1;
+        Annotation compAn2;
+        Annotation personAn2;
+        Annotation locAn2;
+        Annotation indAn2;
+
+        {
+            Struct companyValue1 = (Struct) company.getDataType().createFieldValue();
+            companyValue1.setFieldValue("name", new StringFieldValue("Sun"));
+            companyValue1.setFieldValue("ceo", new StringFieldValue("Scott Mcnealy"));
+            companyValue1.setFieldValue("lat", new FloatFieldValue(37.7f));
+            companyValue1.setFieldValue("lon", new FloatFieldValue(-122.44f));
+            companyValue1.setFieldValue("alt", 60.456);
+            companyValue1.setFieldValue("vertical", new StringFieldValue("software"));
+            compAn1 = new Annotation(company, companyValue1);
+        }
+        {
+            Struct personValue1 = new Struct(manager.getDataType("annotation.person"));
+            personValue1.setFieldValue("name", new StringFieldValue("Richard Bair"));
+            personAn1 = new Annotation(person, personValue1);
+        }
+        {
+            Struct locValue1 = new Struct(manager.getDataType("annotation.location"));
+            locValue1.setFieldValue("name", new StringFieldValue("Prinsens Gate"));
+            locAn1 = new Annotation(location, locValue1);
+        }
+        {
+            Struct indValue1 = new Struct(manager.getDataType("annotation.industry"));
+            indValue1.setFieldValue("vertical", new StringFieldValue("Software Services"));
+            indAn1 = new Annotation(industry, indValue1);
+        }
+        {
+            Struct companyValue2 = (Struct) company.getDataType().createFieldValue();
+            companyValue2.setFieldValue("name", new StringFieldValue("Yahoo"));
+            companyValue2.setFieldValue("ceo", new StringFieldValue("Carol Bartz"));
+            companyValue2.setFieldValue("lat", new FloatFieldValue(32.1f));
+            companyValue2.setFieldValue("lon", new FloatFieldValue(-48.44f));
+            companyValue2.setFieldValue("alt", 33.56);
+            companyValue2.setFieldValue("vertical", new StringFieldValue("Research"));
+            compAn2 = new Annotation(company, companyValue2);
+        }
+        {
+            Struct personValue2 = new Struct(manager.getDataType("annotation.person"));
+            personValue2.setFieldValue("name", new StringFieldValue("Kim Johansen"));
+            personAn2 = new Annotation(person, personValue2);
+        }
+        {
+            Struct locValue2 = new Struct(manager.getDataType("annotation.location"));
+            locValue2.setFieldValue("name", new StringFieldValue("RT Nagar"));
+            locAn2 = new Annotation(location, locValue2);
+        }
+        {
+            Struct indValue2 = new Struct(manager.getDataType("annotation.industry"));
+            indValue2.setFieldValue("vertical", new StringFieldValue("Software Consulting"));
+            indAn2 = new Annotation(industry, indValue2);
+        }
+
+        SpanTree tree = new SpanTree("test");
+        SpanList root = (SpanList) tree.getRoot();
+        AlternateSpanList branch = new AlternateSpanList();
+
+        SpanNode span1 = new Span(0, 3);
+        SpanNode span2 = new Span(1, 9);
+        SpanNode span3 = new Span(12, 10);
+
+        SpanNode span11 = new Span(0, 3);
+        SpanNode span22 = new Span(1, 9);
+        SpanNode span33 = new Span(12, 10);
+
+        SpanList alternate1 = new SpanList();
+        alternate1.add(span3);
+        alternate1.add(span2);
+        alternate1.add(span1);
+
+        SpanList alternate2 = new SpanList();
+        alternate2.add(span11);
+        alternate2.add(span22);
+        alternate2.add(span33);
+
+        tree.annotate(span1, compAn1);
+        tree.annotate(span2, personAn1);
+        tree.annotate(span3, locAn1);
+        tree.annotate(span1, indAn1);
+
+        tree.annotate(span11, compAn2);
+        tree.annotate(span22, personAn2);
+        tree.annotate(span33, locAn2);
+        tree.annotate(span11, indAn2);
+
+        List<SpanNode> subtreeList1 = new ArrayList<>();
+        subtreeList1.add(alternate1);
+
+        List<SpanNode> subtreeList2 = new ArrayList<>();
+        subtreeList2.add(alternate2);
+        branch.addChildren(1, subtreeList1, 20.0d);
+        branch.addChildren(2, subtreeList2, 50.0d);
+        root.add(branch);
+
+        StringFieldValue body = (StringFieldValue) document.getFieldValue(document.getDataType().getField("body"));
+        body.setSpanTree(tree);
+        document.setFieldValue(document.getDataType().getField("body"), body);
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/annotation/Bug6394548TestCase.java b/document/src/test/java/com/yahoo/document/annotation/Bug6394548TestCase.java
new file mode 100644
index 00000000000..ee448bb79b0
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/annotation/Bug6394548TestCase.java
@@ -0,0 +1,123 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+import com.yahoo.document.Document;
+import com.yahoo.document.DocumentTypeManager;
+import com.yahoo.document.DocumentTypeManagerConfigurer;
+import com.yahoo.document.StructDataType;
+import com.yahoo.document.datatypes.*;
+import com.yahoo.document.serialization.*;
+import com.yahoo.io.GrowableByteBuffer;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.Collections;
+
+import static org.junit.Assert.*;
+
+public class Bug6394548TestCase {
+    @Test
+    public void testSerializeAndDeserializeMultipleAdjacentStructAnnotations() {
+        DocumentTypeManager manager = new DocumentTypeManager();
+        DocumentTypeManagerConfigurer.configure(manager, "file:src/test/java/com/yahoo/document/annotation/documentmanager.6394548.cfg");
+
+        AnnotationTypeRegistry registry = manager.getAnnotationTypeRegistry();
+        AnnotationType featureSetType = registry.getType("morty.RICK_FEATURESET");
+        assertNotNull(featureSetType);
+
+        Document doc = new Document(manager.getDocumentType("article"), "doc:article:test");
+        StringFieldValue sfv = new StringFieldValue("badger waltz");
+
+        SpanList root = new SpanList();
+        SpanNode node = new Span(0, sfv.getString().length());
+        root.add(node);
+
+        SpanTree tree = new SpanTree("rick_features", root);
+        for (int i = 0; i < 2; ++i) {
+            tree.annotate(createBigFeatureSetAnnotation(featureSetType));
+        }
+
+        sfv.setSpanTree(tree);
+        doc.setFieldValue("title", sfv);
+        System.out.println(doc.toXml());
+        String annotationsBefore = dumpAllAnnotations(tree);
+
+        GrowableByteBuffer buffer = new GrowableByteBuffer();
+        DocumentSerializer serializer = DocumentSerializerFactory.create42(buffer);
+        serializer.write(doc);
+
+        buffer.flip();
+        DocumentDeserializer deserializer = DocumentDeserializerFactory.create42(manager, buffer);
+        Document doc2 = new Document(deserializer);
+
+        System.out.println(doc2.toXml());
+
+        StringFieldValue readString = (StringFieldValue)doc2.getFieldValue("title");
+        SpanTree readTree = readString.getSpanTree("rick_features");
+        assertNotNull(readTree);
+        String annotationsAfter = dumpAllAnnotations(readTree);
+
+        System.out.println("before:\n" + annotationsBefore);
+        System.out.println("after:\n" + annotationsAfter);
+
+        assertEquals(annotationsBefore, annotationsAfter);
+    }
+
+    private String dumpAllAnnotations(SpanTree tree) {
+        ArrayList<String> tmp = new ArrayList<>();
+        for (Annotation anno : tree) {
+            Struct s = (Struct)anno.getFieldValue();
+            tmp.add(s.toXml());
+        }
+        Collections.sort(tmp);
+        StringBuilder annotations = new StringBuilder();
+        for (String s : tmp) {
+            annotations.append(s);
+        }
+        return annotations.toString();
+    }
+
+    private Annotation createBigFeatureSetAnnotation(AnnotationType featuresetAnno) {
+        StructDataType featuresetType = (StructDataType)featuresetAnno.getDataType();
+        Struct featureset = featuresetType.createFieldValue();
+        System.out.println("featureset type: " + featureset.getDataType().toString());
+
+        MapFieldValue<StringFieldValue, IntegerFieldValue> discreteValued
+                = (MapFieldValue<StringFieldValue, IntegerFieldValue>)featuresetType.getField("discretevaluedfeatures").getDataType().createFieldValue();
+        discreteValued.put(new StringFieldValue("foo"), new IntegerFieldValue(1234));
+        discreteValued.put(new StringFieldValue("bar"), new IntegerFieldValue(567890123));
+        featureset.setFieldValue("discretevaluedfeatures", discreteValued);
+
+        MapFieldValue<StringFieldValue, DoubleFieldValue> realValued
+                = (MapFieldValue<StringFieldValue, DoubleFieldValue>)featuresetType.getField("realvaluedfeatures").getDataType().createFieldValue();
+        realValued.put(new StringFieldValue("foo"), new DoubleFieldValue(0.75));
+        realValued.put(new StringFieldValue("bar"), new DoubleFieldValue(1.5));
+        featureset.setFieldValue("realvaluedfeatures", realValued);
+
+        Array<StringFieldValue> nested = (Array<StringFieldValue>)featureset.getField("foo10").getDataType().createFieldValue();
+        nested.add(new StringFieldValue("baz"));
+        nested.add(new StringFieldValue("blargh"));
+        featureset.setFieldValue("foo10", nested);
+
+        featureset.setFieldValue("foo1", new StringFieldValue("asdf"));
+        featureset.setFieldValue("foo4", new StringFieldValue("qwerty"));
+        featureset.setFieldValue("foo2", new IntegerFieldValue(555));
+        featureset.setFieldValue("foo2", new IntegerFieldValue(8));
+        featureset.setFieldValue("foo7", new IntegerFieldValue(1337));
+        featureset.setFieldValue("foo8", new IntegerFieldValue(967867));
+        featureset.setFieldValue("foo9", new DoubleFieldValue(123.45));
+
+        Array<StringFieldValue> attributes = (Array<StringFieldValue>)featureset.getField("foo6").getDataType().createFieldValue();
+        attributes.add(new StringFieldValue("adam"));
+        attributes.add(new StringFieldValue("jamie"));
+        attributes.add(new StringFieldValue("grant"));
+        attributes.add(new StringFieldValue("tory"));
+        attributes.add(new StringFieldValue("kari"));
+        featureset.setFieldValue("variantattribute", attributes);
+
+        Annotation anno = new Annotation(featuresetAnno);
+        anno.setFieldValue(featureset);
+
+        return anno;
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/annotation/Bug6425939TestCase.java b/document/src/test/java/com/yahoo/document/annotation/Bug6425939TestCase.java
new file mode 100644
index 00000000000..d34d0da8e82
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/annotation/Bug6425939TestCase.java
@@ -0,0 +1,66 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.DocumentTypeManager;
+import com.yahoo.document.Field;
+import com.yahoo.document.StructDataType;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.document.datatypes.Struct;
+import com.yahoo.document.serialization.*;
+import com.yahoo.io.GrowableByteBuffer;
+import org.junit.Before;
+import org.junit.Test;
+import static org.junit.Assert.*;
+
+public class Bug6425939TestCase {
+    private DocumentTypeManager man;
+    private StructDataType person;
+    private AnnotationType personA;
+
+    @Before
+    public void setUp() {
+        man = new DocumentTypeManager();
+
+        person = new StructDataType("personStruct");
+        person.addField(new Field("foo", DataType.STRING));
+        person.addField(new Field("bar", DataType.INT));
+        man.register(person);
+
+        personA = new AnnotationType("person", person);
+        man.getAnnotationTypeRegistry().register(personA);
+    }
+
+    @Test
+    public void canDeserializeAnnotationsOnZeroLengthStrings() {
+        StringFieldValue emptyString = new StringFieldValue("");
+        emptyString.setSpanTree(createSpanTree());
+
+        GrowableByteBuffer buffer = new GrowableByteBuffer(1024);
+        DocumentSerializer serializer = DocumentSerializerFactory.create42(buffer);
+        Field strField = new Field("flarn", DataType.STRING);
+        serializer.write(strField, emptyString);
+        buffer.flip();
+
+        // Should not throw exception if bug 6425939 is fixed:
+        DocumentDeserializer deserializer = DocumentDeserializerFactory.create42(man, buffer);
+        StringFieldValue deserializedString = new StringFieldValue();
+        deserializer.read(strField, deserializedString);
+
+        assertEquals("", deserializedString.getString());
+        SpanTree readTree = deserializedString.getSpanTree("SpanTree1");
+        assertNotNull(readTree);
+    }
+
+    private SpanTree createSpanTree() {
+        SpanList root = new SpanList();
+        SpanTree tree = new SpanTree("SpanTree1", root);
+        SpanNode node = new Span(0, 0);
+        Struct ps = new Struct(person);
+        ps.setFieldValue("foo", "epic badger");
+        ps.setFieldValue("bar", 54321);
+        tree.annotate(node, new Annotation(personA, ps));
+        root.add(node);
+        return tree;
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/annotation/DocTestCase.java b/document/src/test/java/com/yahoo/document/annotation/DocTestCase.java
new file mode 100644
index 00000000000..565cbd6c370
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/annotation/DocTestCase.java
@@ -0,0 +1,425 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+import com.yahoo.document.ArrayDataType;
+import com.yahoo.document.DocumentTypeManager;
+import com.yahoo.document.Field;
+import com.yahoo.document.StructDataType;
+import com.yahoo.document.datatypes.Array;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.document.datatypes.Struct;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.ListIterator;
+
+/**
+ * Contains code snippets that are used in the documentation. Not really a test case.
+ *
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class DocTestCase extends junit.framework.TestCase {
+
+    private class Processing {
+        private Service getService() {
+            return null;
+        }
+    }
+
+    private class Service {
+        private DocumentTypeManager getDocumentTypeManager() {
+            return null;
+        }
+    }
+
+    private Processing processing = null;
+
+
+    public void testSimple1() {
+        StringFieldValue text = new StringFieldValue("<html><head><title>Diary</title></head><body>I live in San Francisco</body></html>");
+                                                    //012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789
+        SpanList root = new SpanList();
+        root.add(new Span(0, 19))
+                .add(new Span(19, 5))
+                .add(new Span(24, 21))
+                .add(new Span(45, 23))
+                .add(new Span(68, 14));
+
+        SpanTree tree = new SpanTree("html", root);
+        text.setSpanTree(tree);
+    }
+
+    public void simple2() {
+        //the following line works inside process(Document, Arguments, Processing) in a DocumentProcessor
+        AnnotationTypeRegistry atr = processing.getService().getDocumentTypeManager().getAnnotationTypeRegistry();
+
+        StringFieldValue text = new StringFieldValue("<html><head><title>Diary</title></head><body>I live in San Francisco</body></html>");
+                                                    //012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789
+        AnnotationType textType = atr.getType("text");
+        AnnotationType markup = atr.getType("markup");
+
+        SpanList root = new SpanList();
+        SpanTree tree = new SpanTree("html", root);
+
+
+        Span span1 = new Span(0, 19);
+        root.add(span1);
+        tree.annotate(span1, markup);
+
+        Span span2 = new Span(19, 5);
+        root.add(span2);
+        tree.annotate(span2, textType);
+
+        Span span3 = new Span(24, 21);
+        root.add(span3);
+        tree.annotate(span3, markup);
+
+        Span span4 = new Span(45, 23);
+        root.add(span4);
+        tree.annotate(span4, textType);
+
+        Span span5 = new Span(68, 14);
+        root.add(span5);
+        tree.annotate(span5, markup);
+
+
+        text.setSpanTree(tree);
+    }
+
+    public void simple3() {
+        //the following line works inside process(Document, Arguments, Processing) in a DocumentProcessor
+        AnnotationTypeRegistry atr = processing.getService().getDocumentTypeManager().getAnnotationTypeRegistry();
+
+        StringFieldValue text = new StringFieldValue("<html><head><title>Diary</title></head><body>I live in San Francisco</body></html>");
+                                                    //012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789
+
+        SpanList root = new SpanList();
+        SpanTree tree = new SpanTree("html", root);
+
+        AnnotationType textType = atr.getType("text");
+        AnnotationType beginTag = atr.getType("begintag");
+        AnnotationType endTag = atr.getType("endtag");
+        AnnotationType bodyType = atr.getType("body");
+        AnnotationType headerType = atr.getType("header");
+
+        SpanList header = new SpanList();
+        {
+            Span span1 = new Span(6, 6);
+            Span span2 = new Span(12, 7);
+            Span span3 = new Span(19, 5);
+            Span span4 = new Span(24, 8);
+            Span span5 = new Span(32, 7);
+            header.add(span1)
+                    .add(span2)
+                    .add(span3)
+                    .add(span4)
+                    .add(span5);
+            tree.annotate(span1, beginTag)
+                    .annotate(span2, beginTag)
+                    .annotate(span3, textType)
+                    .annotate(span4, endTag)
+                    .annotate(span5, endTag)
+                    .annotate(header, headerType);
+        }
+
+        SpanList body = new SpanList();
+        {
+            Span span1 = new Span(39, 6);
+            Span span2 = new Span(45, 23);
+            Span span3 = new Span(68, 7);
+            body.add(span1)
+                    .add(span2)
+                    .add(span3);
+            tree.annotate(span1, beginTag)
+                    .annotate(span2, textType)
+                    .annotate(span3, endTag)
+                    .annotate(body, bodyType);
+        }
+
+        {
+            Span span1 = new Span(0, 6);
+            Span span2 = new Span(75, 7);
+            root.add(span1)
+                    .add(header)
+                    .add(body)
+                    .add(span2);
+            tree.annotate(span1, beginTag)
+                    .annotate(span2, endTag);
+        }
+
+        text.setSpanTree(tree);
+    }
+
+
+    public void simple4() {
+        //the following line works inside process(Document, Arguments, Processing) in a DocumentProcessor
+        AnnotationTypeRegistry atr = processing.getService().getDocumentTypeManager().getAnnotationTypeRegistry();
+
+        StringFieldValue text = new StringFieldValue("<html><head><title>Diary</title></head><body>I live in San Francisco</body></html>");
+                                                    //012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789
+
+        SpanList root = new SpanList();
+        SpanTree tree = new SpanTree("html", root);
+
+        AnnotationType textType = atr.getType("text");
+        AnnotationType beginTag = atr.getType("begintag");
+        AnnotationType endTag = atr.getType("endtag");
+        AnnotationType bodyType = atr.getType("body");
+        AnnotationType headerType = atr.getType("header");
+        AnnotationType cityType = atr.getType("city");
+
+        Struct position = (Struct) cityType.getDataType().createFieldValue();
+        position.setFieldValue("latitude", 37.774929);
+        position.setFieldValue("longitude", -122.419415);
+        Annotation city = new Annotation(cityType, position);
+
+        SpanList header = new SpanList();
+        {
+            Span span1 = new Span(6, 6);
+            Span span2 = new Span(12, 7);
+            Span span3 = new Span(19, 5);
+            Span span4 = new Span(24, 8);
+            Span span5 = new Span(32, 7);
+            header.add(span1)
+                    .add(span2)
+                    .add(span3)
+                    .add(span4)
+                    .add(span5);
+            tree.annotate(span1, beginTag)
+                    .annotate(span2, beginTag)
+                    .annotate(span3, textType)
+                    .annotate(span4, endTag)
+                    .annotate(span4, endTag)
+                    .annotate(header, headerType);
+        }
+
+        SpanList textNode = new SpanList();
+        {
+            Span span1 = new Span(45, 10);
+            Span span2 = new Span(55, 13);
+            textNode.add(span1)
+                    .add(span2);
+            tree.annotate(span2, city)
+                    .annotate(textNode, textType);
+        }
+
+        SpanList body = new SpanList();
+        {
+            Span span1 = new Span(39, 6);
+            Span span2 = new Span(68, 7);
+            body.add(span1)
+                    .add(textNode)
+                    .add(span2);
+            tree.annotate(span1, beginTag)
+                    .annotate(span2, endTag)
+                    .annotate(body, bodyType);
+        }
+
+        {
+            Span span1 = new Span(0, 6);
+            Span span2 = new Span(75, 7);
+            root.add(span1)
+                    .add(header)
+                    .add(body)
+                    .add(span2);
+            tree.annotate(span1, beginTag)
+                    .annotate(span2, endTag);
+        }
+
+        text.setSpanTree(tree);
+    }
+
+
+    public void simple5() {
+        //the following two lines work inside process(Document, Arguments, Processing) in a DocumentProcessor
+        DocumentTypeManager dtm = processing.getService().getDocumentTypeManager();
+        AnnotationTypeRegistry atr = dtm.getAnnotationTypeRegistry();
+
+        StringFieldValue text = new StringFieldValue("<body><p>I live in San </p>Francisco</body>");
+                                                    //012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789
+
+        SpanList root = new SpanList();
+        SpanTree tree = new SpanTree("html", root);
+
+        StructDataType positionType = (StructDataType) dtm.getDataType("position");
+
+        AnnotationType textType = atr.getType("text");
+        AnnotationType beginTag = atr.getType("begintag");
+        AnnotationType endTag = atr.getType("endtag");
+        AnnotationType bodyType = atr.getType("body");
+        AnnotationType paragraphType = atr.getType("paragraph");
+        AnnotationType cityType = atr.getType("city");
+
+        Struct position = new Struct(positionType);
+        position.setFieldValue("latitude", 37.774929);
+        position.setFieldValue("longitude", -122.419415);
+
+        Annotation sanAnnotation = new Annotation(textType);
+        Annotation franciscoAnnotation = new Annotation(textType);
+
+        Struct positionWithRef = (Struct) cityType.getDataType().createFieldValue();
+        positionWithRef.setFieldValue("position", position);
+
+        Field referencesField = ((StructDataType) cityType.getDataType()).getField("references");
+        Array<FieldValue> refList = new Array<FieldValue>(referencesField.getDataType());
+        AnnotationReferenceDataType annRefType = (AnnotationReferenceDataType) ((ArrayDataType) referencesField.getDataType()).getNestedType();
+        refList.add(new AnnotationReference(annRefType, sanAnnotation));
+        refList.add(new AnnotationReference(annRefType, franciscoAnnotation));
+        positionWithRef.setFieldValue(referencesField, refList);
+
+        Annotation city = new Annotation(cityType, positionWithRef);
+
+        SpanList paragraph = new SpanList();
+        {
+            Span span1 = new Span(6, 3);
+            Span span2 = new Span(9, 10);
+            Span span3 = new Span(19, 4);
+            Span span4 = new Span(23, 4);
+            paragraph.add(span1)
+                    .add(span2)
+                    .add(span3)
+                    .add(span4);
+            tree.annotate(span1, beginTag)
+                    .annotate(span2, textType)
+                    .annotate(span3, sanAnnotation)
+                    .annotate(span4, endTag)
+                    .annotate(paragraph, paragraphType);
+        }
+
+        {
+            Span span1 = new Span(0, 6);
+            Span span2 = new Span(27, 9);
+            Span span3 = new Span(36, 8);
+            root.add(span1)
+                    .add(paragraph)
+                    .add(span2)
+                    .add(span3);
+
+            tree.annotate(span1, beginTag)
+                    .annotate(span2, franciscoAnnotation)
+                    .annotate(span3, endTag)
+                    .annotate(root, bodyType)
+                    .annotate(city);
+        }
+
+        text.setSpanTree(tree);
+    }
+
+    public void simple6() {
+        StringFieldValue text = new StringFieldValue("<html><head><title>Diary</title></head><body>I live in San Francisco</body></html>");
+                                                    //012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789
+
+        SpanTree tree = text.getSpanTree("html");
+        SpanList root = (SpanList) tree.getRoot();
+        //TODO: Note that the above could have been a Span or an AlternateSpanList!
+
+        ListIterator<SpanNode> nodeIt = root.childIterator();
+
+        AnnotationType beginTag = new AnnotationType("begintag");
+        AnnotationType endTag = new AnnotationType("endtag");
+
+
+        while (nodeIt.hasNext()) {
+            SpanNode node = nodeIt.next();
+            boolean nodeHadMarkupAnnotation = removeMarkupAnnotation(tree, node);
+            if (nodeHadMarkupAnnotation) {
+                nodeIt.remove();
+                List<Span> replacementNodes = analyzeMarkup(tree, node, text, beginTag, endTag);
+                for (SpanNode repl : replacementNodes) {
+                    nodeIt.add(repl);
+                }
+            }
+        }
+    }
+
+    /**
+     * Removes annotations of type 'markup' from the given node.
+     *
+     * @param tree the tree to remove annotations from
+     * @param node the node to remove annotations of type 'markup' from
+     * @return true if the given node had 'markup' annotations, false otherwise
+     */
+    private boolean removeMarkupAnnotation(SpanTree tree, SpanNode node) {
+        //get iterator over all annotations on this node:
+        Iterator<Annotation> annotationIt = tree.iterator(node);
+
+        while (annotationIt.hasNext()) {
+            Annotation annotation = annotationIt.next();
+            if (annotation.getType().getName().equals("markup")) {
+                //this node has an annotation of type markup, remove it:
+                annotationIt.remove();
+                //return true, this node had a markup annotation:
+                return true;
+            }
+        }
+        //this node did not have a markup annotation:
+        return false;
+    }
+
+    /**
+     * NOTE: This method is provided only for completeness. It analyzes spans annotated with
+     * &quot;markup&quot;, and splits them into several shorter spans annotated with &quot;begintag&quot;
+     * and &quot;endtag&quot;.
+     *
+     * @param tree       the span tree to annotate into
+     * @param input      a SpanNode that is annotated with &quot;markup&quot;.
+     * @param text       the text that the SpanNode covers
+     * @param beginTag   the type to use for begintag annotations
+     * @param endTagType the type to use for endtag annotations
+     * @return a list of new spans to replace the input
+     */
+    private List<Span> analyzeMarkup(SpanTree tree, SpanNode input, StringFieldValue text,
+                                     AnnotationType beginTag, AnnotationType endTagType) {
+        //we know that this node is annotated with "markup"
+        String coveredText = input.getText(text.getString()).toString();
+        int spanOffset = input.getFrom();
+        int tagStart = -1;
+        boolean endTag = false;
+        List<Span> tags = new ArrayList<Span>();
+        for (int i = 0; i < coveredText.length(); i++) {
+            if (coveredText.charAt(i) == '<') {
+                //we're in a tag
+                tagStart = i;
+                continue;
+            }
+            if (coveredText.charAt(i) == '>' && tagStart > -1) {
+                Span span = new Span(spanOffset + tagStart, (i + 1) - tagStart);
+                tags.add(span);
+                if (endTag) {
+                    tree.annotate(span, endTagType);
+                } else {
+                    tree.annotate(span, beginTag);
+                }
+                tagStart = -1;
+            }
+            if (tagStart > -1 && i == (tagStart + 1)) {
+                if (coveredText.charAt(i) == '/') {
+                    endTag = true;
+                } else {
+                    endTag = false;
+                }
+            }
+        }
+        return tags;
+    }
+
+    public void simple7() {
+        StringFieldValue text = new StringFieldValue("<html><head><title>Diary</title></head><body>I live in San Francisco</body></html>");
+                                                    //012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789
+
+        SpanTree tree = text.getSpanTree("html");
+
+        Iterator<Annotation> annotationIt = tree.iterator();
+
+        while (annotationIt.hasNext()) {
+            Annotation annotation = annotationIt.next();
+            if (annotation.getType().getName().equals("markup")) {
+                //we have an annotation of type markup, remove it:
+                annotationIt.remove();
+            }
+        }
+    }
+
+}
diff --git a/document/src/test/java/com/yahoo/document/annotation/DummySpanNodeTestCase.java b/document/src/test/java/com/yahoo/document/annotation/DummySpanNodeTestCase.java
new file mode 100644
index 00000000000..f238b906f6b
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/annotation/DummySpanNodeTestCase.java
@@ -0,0 +1,29 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+import org.junit.Test;
+
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.CoreMatchers.nullValue;
+import static org.junit.Assert.assertThat;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ * @since 5.1.10
+ */
+public class DummySpanNodeTestCase {
+
+    @Test
+    public void basic() {
+        DummySpanNode node = DummySpanNode.INSTANCE;
+        assertThat(node.getFrom(), is(0));
+        assertThat(node.getTo(), is(0));
+        assertThat(node.getLength(), is(0));
+        assertThat(node.getText("baba"), nullValue());
+        assertThat(node.isLeafNode(), is(true));
+        assertThat(node.childIterator().hasNext(), is(false));
+        assertThat(node.childIterator().hasPrevious(), is(false));
+        assertThat(node.childIteratorRecursive().hasNext(), is(false));
+        assertThat(node.childIteratorRecursive().hasPrevious(), is(false));
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/annotation/IndexKeyAnnotationTypeSpanTreeAdvTest.java b/document/src/test/java/com/yahoo/document/annotation/IndexKeyAnnotationTypeSpanTreeAdvTest.java
new file mode 100644
index 00000000000..790813abe73
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/annotation/IndexKeyAnnotationTypeSpanTreeAdvTest.java
@@ -0,0 +1,13 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class IndexKeyAnnotationTypeSpanTreeAdvTest extends SpanTreeAdvTest {
+    @Override
+    public void populateSpanTree() {
+        super.populateSpanTree();
+        tree.createIndex(SpanTree.IndexKey.ANNOTATION_TYPE);
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/annotation/IndexKeyAnnotationTypeSpanTreeTestCase.java b/document/src/test/java/com/yahoo/document/annotation/IndexKeyAnnotationTypeSpanTreeTestCase.java
new file mode 100644
index 00000000000..8bad1bdc37d
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/annotation/IndexKeyAnnotationTypeSpanTreeTestCase.java
@@ -0,0 +1,14 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class IndexKeyAnnotationTypeSpanTreeTestCase extends SpanTreeTestCase {
+
+    @Override
+    public void setUp() {
+        super.setUp();
+        tree.createIndex(SpanTree.IndexKey.ANNOTATION_TYPE);
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/annotation/IndexKeySpanNodeSpanTreeAdvTest.java b/document/src/test/java/com/yahoo/document/annotation/IndexKeySpanNodeSpanTreeAdvTest.java
new file mode 100644
index 00000000000..49c34d1fee4
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/annotation/IndexKeySpanNodeSpanTreeAdvTest.java
@@ -0,0 +1,14 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class IndexKeySpanNodeSpanTreeAdvTest extends SpanTreeAdvTest {
+
+    @Override
+    public void populateSpanTree() {
+        super.populateSpanTree();
+        tree.createIndex(SpanTree.IndexKey.SPAN_NODE);
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/annotation/IndexKeySpanNodeSpanTreeTestCase.java b/document/src/test/java/com/yahoo/document/annotation/IndexKeySpanNodeSpanTreeTestCase.java
new file mode 100644
index 00000000000..dc685f3079d
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/annotation/IndexKeySpanNodeSpanTreeTestCase.java
@@ -0,0 +1,14 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class IndexKeySpanNodeSpanTreeTestCase extends SpanTreeTestCase {
+
+    @Override
+    public void setUp() {
+        super.setUp();
+        tree.createIndex(SpanTree.IndexKey.SPAN_NODE);
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/annotation/IndexKeySpanTreeTestCase.java b/document/src/test/java/com/yahoo/document/annotation/IndexKeySpanTreeTestCase.java
new file mode 100644
index 00000000000..243367e4f54
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/annotation/IndexKeySpanTreeTestCase.java
@@ -0,0 +1,57 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+import org.junit.Test;
+
+import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.assertThat;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class IndexKeySpanTreeTestCase {
+
+    @Test
+    public void testIndexKeys() throws Exception {
+        SpanTree tree = new SpanTree("something");
+        assertThat(tree.getCurrentIndexes().isEmpty(), is(true));
+
+        tree.createIndex(SpanTree.IndexKey.SPAN_NODE);
+        assertThat(tree.getCurrentIndexes().size(), is(1));
+        assertThat(tree.getCurrentIndexes().iterator().next(), is(SpanTree.IndexKey.SPAN_NODE));
+
+        tree.clearIndexes();
+        assertThat(tree.getCurrentIndexes().isEmpty(), is(true));
+
+        tree.createIndex(SpanTree.IndexKey.ANNOTATION_TYPE);
+        assertThat(tree.getCurrentIndexes().size(), is(1));
+        assertThat(tree.getCurrentIndexes().iterator().next(), is(SpanTree.IndexKey.ANNOTATION_TYPE));
+
+        tree.clearIndexes();
+        assertThat(tree.getCurrentIndexes().isEmpty(), is(true));
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testSwitchIndexes()  {
+        SpanTree tree = new SpanTree("something");
+        assertThat(tree.getCurrentIndexes().isEmpty(), is(true));
+        tree.createIndex(SpanTree.IndexKey.SPAN_NODE);
+        tree.createIndex(SpanTree.IndexKey.ANNOTATION_TYPE);
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testSwitchIndexes2()  {
+        SpanTree tree = new SpanTree("something");
+        assertThat(tree.getCurrentIndexes().isEmpty(), is(true));
+        tree.createIndex(SpanTree.IndexKey.ANNOTATION_TYPE);
+        tree.createIndex(SpanTree.IndexKey.SPAN_NODE);
+    }
+
+    @Test
+    public void testSwitchIndexes3()  {
+        SpanTree tree = new SpanTree("something");
+        assertThat(tree.getCurrentIndexes().isEmpty(), is(true));
+        tree.createIndex(SpanTree.IndexKey.ANNOTATION_TYPE);
+        tree.clearIndex(SpanTree.IndexKey.SPAN_NODE);
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/annotation/PeekableListIteratorTestCase.java b/document/src/test/java/com/yahoo/document/annotation/PeekableListIteratorTestCase.java
new file mode 100644
index 00000000000..d63cd324033
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/annotation/PeekableListIteratorTestCase.java
@@ -0,0 +1,335 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.ListIterator;
+import java.util.NoSuchElementException;
+
+/**
+ * Tests that PeekableListIterator behaves as expected.
+ *
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class PeekableListIteratorTestCase extends junit.framework.TestCase {
+    private List<String> strings;
+
+    @Override
+    public void setUp() {
+        strings = new ArrayList<String>();
+        strings.add("0");
+        strings.add("1");
+        strings.add("2");
+        strings.add("3");
+        strings.add("4");
+        strings.add("5");
+    }
+
+    public void testSimpleListIterator() {
+        ListIterator<String> it = strings.listIterator();
+        assertEquals("0", it.next());
+        assertEquals("1", it.next());
+        //cursor is before "2"
+        assertEquals("1", it.previous());
+        //cursor is before "1"
+        assertEquals("0", it.previous());
+        //cursor is before "0"
+        //removing 0, it's the last one returned by next() or previous():
+        it.remove();
+        assertEquals("1", it.next());
+    }
+
+    public void testVarious() {
+        PeekableListIterator<String> it = new PeekableListIterator<String>(strings.listIterator());
+
+        assertTrue(it.hasNext());
+        assertEquals("0", it.peek());
+        assertTrue(it.hasNext());
+        assertEquals("0", it.next());
+        assertEquals("1", it.next());
+        assertEquals("2", it.next());
+        assertEquals("3", it.peek());
+        assertEquals("3", it.peek());
+        assertEquals("3", it.peek());
+        assertTrue(it.hasNext());
+        assertTrue(it.hasNext());
+        assertTrue(it.hasNext());
+        assertTrue(it.hasNext());
+        assertTrue(it.hasNext());
+        assertTrue(it.hasNext());
+        assertTrue(it.hasNext());
+        assertTrue(it.hasNext());
+        assertTrue(it.hasNext());
+        assertTrue(it.hasNext());
+        assertTrue(it.hasNext());
+        assertTrue(it.hasNext());
+        assertTrue(it.hasNext());
+        assertEquals("3", it.next());
+        //cursor is now before 4
+        it.add("banana");
+        it.add("apple");
+        //added before 4
+        assertTrue(it.hasNext());
+        assertTrue(it.hasNext());
+        assertTrue(it.hasNext());
+        assertTrue(it.hasNext());
+        assertEquals("4", it.peek());
+        assertEquals("4", it.peek());
+        assertEquals("4", it.peek());
+        assertEquals("4", it.peek());
+        //removing 3
+        it.remove();
+        assertEquals("4", it.peek());
+        assertEquals("4", it.next());
+        //replacing 4 with orange:
+        it.set("orange");
+        assertEquals("5", it.peek());
+        assertEquals("5", it.next());
+        assertFalse(it.hasNext());
+        assertNull(it.peek());
+        try {
+            it.next();
+            fail("Shouldn't have worked.");
+        } catch (NoSuchElementException nsee) {
+            // empty
+        }
+    }
+
+    public void testRemoveFirst() {
+        PeekableListIterator<String> it = new PeekableListIterator<String>(strings.listIterator());
+        try {
+            it.remove();  //this shouldn't work
+            fail("shouldn't work");
+        } catch (IllegalStateException ise) {
+            //nada
+        }
+
+        assertEquals("0", it.next());
+        it.remove();
+
+        assertEquals(5, strings.size());
+        assertEquals("1", strings.get(0));
+    }
+
+    public void testPeekThenRemove() {
+        PeekableListIterator<String> it = new PeekableListIterator<String>(strings.listIterator());
+
+        it.peek();
+        try {
+            it.remove();  //this should not work!
+            fail("should have gotten exception");
+        } catch (IllegalStateException ise) {
+            //niks
+        }
+
+        assertEquals("0", it.next());
+        it.remove();  //this should work
+        assertEquals(5, strings.size());
+        assertEquals("1", strings.get(0));
+        assertEquals("2", strings.get(1));
+        assertEquals("3", strings.get(2));
+        assertEquals("4", strings.get(3));
+        assertEquals("5", strings.get(4));
+    }
+
+    public void testPeekNextRemove() {
+        PeekableListIterator<String> it = new PeekableListIterator<String>(strings.listIterator());
+
+        assertEquals("0", it.peek());
+        assertEquals("0", it.next());
+        it.remove();
+        assertEquals(5, strings.size());
+        assertEquals("1", strings.get(0));
+        assertEquals("2", strings.get(1));
+        assertEquals("3", strings.get(2));
+        assertEquals("4", strings.get(3));
+        assertEquals("5", strings.get(4));
+    }
+
+    public void testPeekNextPeekRemove() {
+        PeekableListIterator<String> it = new PeekableListIterator<String>(strings.listIterator());
+
+        assertEquals("0", it.peek());
+        assertEquals("0", it.next());
+        assertEquals("1", it.peek());
+        it.remove();
+        assertEquals(5, strings.size());
+        assertEquals("1", strings.get(0));
+        assertEquals("2", strings.get(1));
+        assertEquals("3", strings.get(2));
+        assertEquals("4", strings.get(3));
+        assertEquals("5", strings.get(4));
+    }
+
+    public void testPeekNextNextRemove() {
+        PeekableListIterator<String> it = new PeekableListIterator<String>(strings.listIterator());
+
+        assertEquals("0", it.peek());
+        assertEquals("0", it.next());
+        assertEquals("1", it.next());
+        it.remove();
+        assertEquals(5, strings.size());
+        assertEquals("0", strings.get(0));
+        assertEquals("2", strings.get(1));
+        assertEquals("3", strings.get(2));
+        assertEquals("4", strings.get(3));
+        assertEquals("5", strings.get(4));
+    }
+
+    public void testNextPeekRemove() {
+        PeekableListIterator<String> it = new PeekableListIterator<String>(strings.listIterator());
+
+        assertEquals("0", it.next());
+        assertEquals("1", it.peek());
+        it.remove();
+        assertEquals(5, strings.size());
+        assertEquals("1", strings.get(0));
+        assertEquals("2", strings.get(1));
+        assertEquals("3", strings.get(2));
+        assertEquals("4", strings.get(3));
+        assertEquals("5", strings.get(4));
+    }
+
+    public void testAddSimple() {
+        PeekableListIterator<String> it = new PeekableListIterator<String>(strings.listIterator());
+
+        it.add("a");
+        it.add("b");
+        it.add("c");
+        assertEquals("0", it.next());
+
+        assertEquals(9, strings.size());
+        assertEquals("a", strings.get(0));
+        assertEquals("b", strings.get(1));
+        assertEquals("c", strings.get(2));
+        assertEquals("0", strings.get(3));
+        assertEquals("1", strings.get(4));
+        assertEquals("2", strings.get(5));
+        assertEquals("3", strings.get(6));
+        assertEquals("4", strings.get(7));
+        assertEquals("5", strings.get(8));
+    }
+
+    public void testPeekAdd() {
+        PeekableListIterator<String> it = new PeekableListIterator<String>(strings.listIterator());
+
+        it.peek();
+        it.add("a");
+        it.add("b");
+        it.add("c");
+        assertEquals("0", it.next());
+
+        assertEquals(9, strings.size());
+        assertEquals("a", strings.get(0));
+        assertEquals("b", strings.get(1));
+        assertEquals("c", strings.get(2));
+        assertEquals("0", strings.get(3));
+        assertEquals("1", strings.get(4));
+        assertEquals("2", strings.get(5));
+        assertEquals("3", strings.get(6));
+        assertEquals("4", strings.get(7));
+        assertEquals("5", strings.get(8));
+    }
+
+
+    public void testSetFirst() {
+        PeekableListIterator<String> it = new PeekableListIterator<String>(strings.listIterator());
+        try {
+            it.set("kjell");  //this shouldn't work
+            fail("shouldn't work");
+        } catch (IllegalStateException ise) {
+            //nada
+        }
+
+        assertEquals("0", it.next());
+        it.set("elvis");
+
+        assertEquals(6, strings.size());
+        assertEquals("elvis", strings.get(0));
+    }
+
+    public void testPeekThenSet() {
+        PeekableListIterator<String> it = new PeekableListIterator<String>(strings.listIterator());
+
+        it.peek();
+        try {
+            it.set("elvis");  //this should not work!
+            fail("should have gotten exception");
+        } catch (IllegalStateException ise) {
+            //niks
+        }
+
+        assertEquals("0", it.next());
+        it.set("presley");  //this should work
+        assertEquals(6, strings.size());
+        assertEquals("presley", strings.get(0));
+        assertEquals("1", strings.get(1));
+        assertEquals("2", strings.get(2));
+        assertEquals("3", strings.get(3));
+        assertEquals("4", strings.get(4));
+        assertEquals("5", strings.get(5));
+    }
+
+    public void testPeekNextSet() {
+        PeekableListIterator<String> it = new PeekableListIterator<String>(strings.listIterator());
+
+        assertEquals("0", it.peek());
+        assertEquals("0", it.next());
+        it.set("buddy");
+        assertEquals(6, strings.size());
+        assertEquals("buddy", strings.get(0));
+        assertEquals("1", strings.get(1));
+        assertEquals("2", strings.get(2));
+        assertEquals("3", strings.get(3));
+        assertEquals("4", strings.get(4));
+        assertEquals("5", strings.get(5));
+    }
+
+    public void testPeekNextPeekSet() {
+        PeekableListIterator<String> it = new PeekableListIterator<String>(strings.listIterator());
+
+        assertEquals("0", it.peek());
+        assertEquals("0", it.next());
+        assertEquals("1", it.peek());
+        it.set("holly");
+        assertEquals(6, strings.size());
+        assertEquals("holly", strings.get(0));
+        assertEquals("1", strings.get(1));
+        assertEquals("2", strings.get(2));
+        assertEquals("3", strings.get(3));
+        assertEquals("4", strings.get(4));
+        assertEquals("5", strings.get(5));
+    }
+
+    public void testPeekNextNextSet() {
+        PeekableListIterator<String> it = new PeekableListIterator<String>(strings.listIterator());
+
+        assertEquals("0", it.peek());
+        assertEquals("0", it.next());
+        assertEquals("1", it.next());
+        it.set("jerry");
+        assertEquals(6, strings.size());
+        assertEquals("0", strings.get(0));
+        assertEquals("jerry", strings.get(1));
+        assertEquals("2", strings.get(2));
+        assertEquals("3", strings.get(3));
+        assertEquals("4", strings.get(4));
+        assertEquals("5", strings.get(5));
+    }
+
+    public void testNextPeekSet() {
+        PeekableListIterator<String> it = new PeekableListIterator<String>(strings.listIterator());
+
+        assertEquals("0", it.next());
+        assertEquals("1", it.peek());
+        it.set("lee");
+        assertEquals(6, strings.size());
+        assertEquals("lee", strings.get(0));
+        assertEquals("1", strings.get(1));
+        assertEquals("2", strings.get(2));
+        assertEquals("3", strings.get(3));
+        assertEquals("4", strings.get(4));
+        assertEquals("5", strings.get(5));
+    }
+
+}
diff --git a/document/src/test/java/com/yahoo/document/annotation/SpanListAdvTestCase.java b/document/src/test/java/com/yahoo/document/annotation/SpanListAdvTestCase.java
new file mode 100644
index 00000000000..c43385f1572
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/annotation/SpanListAdvTestCase.java
@@ -0,0 +1,284 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.datatypes.StringFieldValue;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Collection;
+import java.util.ListIterator;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author <a href="mailto:mpraveen@yahoo-inc.com">Praveen Mohan</a>
+ *
+ * This test covers all possible scenarios in a SpanList.
+ *
+ */
+
+public class SpanListAdvTestCase {
+
+    private boolean debug = false;
+
+    private AnnotationType at1 = new AnnotationType("person", DataType.STRING);
+    private AnnotationType at2 = new AnnotationType("street", DataType.STRING);
+    private AnnotationType at3 = new AnnotationType("city", DataType.STRING);
+
+    private SpanList root = new SpanList();
+    public SpanTree tree = new SpanTree("test", root);
+
+    private SpanNode span1, span2, span3;
+    private SpanNode span11, span22, span33;
+    private SpanNode span111, span222, span333;
+    private SpanList alternate1, alternate2, alternate3;
+
+    @Before
+    public void buildTree() {
+        tree.cleanup();
+        span1 = new Span(0, 3);
+        span2 = new Span(4, 6);
+        span3 = new Span(6, 10);
+
+        span11 = new Span(0, 2);
+        span22 = new Span(2, 10);
+        span33 = new Span(12, 10);
+
+        span111 = new Span(5, 10);
+        span222 = new Span(15, 5);
+        span333 = new Span(20, 10);
+
+        alternate1 = new SpanList();
+        alternate1.add(span3);
+        alternate1.add(span2);
+        alternate1.add(span1);
+
+        alternate2 = new SpanList();
+        alternate2.add(span11);
+        alternate2.add(span22);
+        alternate2.add(span33);
+
+        alternate3 = new SpanList();
+        alternate3.add(span111);
+        alternate3.add(span222);
+
+        root.add(span333);
+
+        tree.annotate(span1, at1);
+        tree.annotate(span2, at2);
+        tree.annotate(span3, at3);
+
+        tree.annotate(span11, at1);
+        tree.annotate(span22, at2);
+        tree.annotate(span33, at3);
+
+        tree.annotate(span111, at1);
+        tree.annotate(span222, at2);
+        tree.annotate(span333, at3);
+
+        alternate1.add(alternate3);
+
+        root.add(alternate1);
+        root.add(alternate2);
+    }
+
+    @Test
+    public void assertTree() {
+
+        if (debug) {
+            consumeAnnotations((SpanList)tree.getRoot());
+        }
+
+        assertEquals(0, root.getFrom());
+        assertEquals(30, root.getTo());
+        assertEquals(0, alternate1.getFrom());
+        assertEquals(20, alternate1.getTo());
+        assertEquals(0, alternate2.getFrom());
+        assertEquals(22, alternate2.getTo());
+        assertEquals(5, alternate3.getFrom());
+        assertEquals(20, alternate3.getTo());
+        assertFalse(root.numChildren() != 3 || alternate1.numChildren() != 4 || alternate2.numChildren() != 3 || alternate3.numChildren() != 2);
+
+        ArrayList<SpanNode> al = new ArrayList<SpanNode>();
+        al.add(span333);
+        al.add(alternate1);
+        al.add(alternate2);
+        ListIterator<SpanNode> iter = root.childIterator();
+        while (iter.hasNext()) {
+            SpanNode sn = iter.next();
+            int i = 0;
+            for (i = 0; i < al.size(); i ++) {
+                if (sn == al.get(i)) {
+                    break;
+                }
+            }
+            assertFalse(i >= al.size());
+        }
+
+        iter = root.childIteratorRecursive();
+        boolean nodeFound = false;
+        while (iter.hasNext()) {
+            SpanNode sn = iter.next();
+            if (sn == span222 || sn == span111) {
+                nodeFound = true;
+                break;
+            }
+        }
+        assertTrue(nodeFound);
+
+        alternate1.sortChildren();
+        SpanNode ssn = new Span(2, 1);
+        alternate1.add(ssn);
+        alternate1.sortChildren();
+
+        iter = alternate1.childIterator();
+        int from = -1;
+        while (iter.hasNext()) {
+            SpanNode sn = iter.next();
+            if (from == -1) {
+                from = sn.getFrom();
+                continue;
+            }
+            assertFalse(sn.getFrom() < from);
+            from = sn.getFrom();
+        }
+        alternate1.remove(ssn);
+
+        SpanList sl = alternate3.remove(span111);
+        assertFalse (sl != alternate3);
+        assertFalse(span111.isValid());
+
+        alternate1.remove(alternate3);
+        assertFalse(alternate3.isValid());
+        assertFalse(span222.isValid());
+
+        int noofChild = alternate1.numChildren();
+        for (int i = 0; i < alternate1.numChildren(); ) {
+            alternate1.remove(i);
+        }
+        assertFalse(alternate1.numChildren() != 0);
+
+        int noAnnotations = tree.numAnnotations();
+        tree.cleanup();
+        assertFalse(tree.numAnnotations() != (noAnnotations - 5));
+
+        root.clearChildren();
+        tree.cleanup();
+        assertFalse(tree.numAnnotations() != 0);
+
+        sl = new SpanList(alternate1);
+        root.add(sl);
+        assertFalse(root.getFrom() != -1 || root.getTo() != -1);
+
+        SpanNode newSpan1 = new Span(0, 10);
+        SpanNode newSpan2 = new Span(12, 8);
+        alternate3 = new SpanList();
+        alternate3.add(newSpan1);
+        alternate3.add(newSpan2);
+        alternate2.add(alternate3);
+
+        SpanList newA2 = new SpanList(alternate2);
+        root.add(newA2);
+        assertFalse(root.getFrom() != newA2.getFrom() || root.getTo() != newA2.getTo());
+        assertFalse(newA2.numChildren() != 4);
+    }
+
+    @After
+    public void removeTree() {
+        tree = null;
+    }
+
+
+    public void consumeAnnotations(SpanList root) {
+        if (debug) System.out.println("\n\nSpanList: [" + root.getFrom() + ", " + root.getTo() + "] num Children: " + root.numChildren());
+        if (debug) System.out.println("-------------------");
+        Iterator<SpanNode> childIterator = root.childIterator();
+		while (childIterator.hasNext()) {
+			SpanNode node = childIterator.next();
+            //System.out.println("Span Node: " + node); // + " Span Text: " + node.getText(fieldValStr));
+            if (debug) System.out.println("\n\nSpan Node: [" + node.getFrom() + ", " + node.getTo() + "] ");
+            if (node instanceof AlternateSpanList) {
+                parseAlternateLists((AlternateSpanList)node);
+                if (debug) System.out.println("---- Alternate SpanList complete ---");
+            } else if (node instanceof SpanList) {
+                if (debug) System.out.println("Encountered another span list");
+                SpanList spl = (SpanList) node;
+                ListIterator<SpanNode> lli = spl.childIterator();
+                while (lli.hasNext()) System.out.print(" " + lli.next() + " ");
+                consumeAnnotations((SpanList) node);
+            } else {
+       			if (debug) System.out.println("\nGetting annotations for this span node: [" + node.getFrom() + ", " + node.getTo() + "] ");
+   	    	    getAnnotationsForNode(node);
+            }
+		}
+        if (debug) System.out.println("\nGetting annotations for the SpanList itself : [" + root.getFrom() + ", " + root.getTo() + "] ");
+        getAnnotationsForNode(root);
+    }
+
+    public void parseAlternateLists(AlternateSpanList aspl) {
+        int no = aspl.getNumSubTrees();
+        if (debug) System.out.println("Parsing Alternate span list. No of subtrees: " + no);
+        int ctr = 0;
+        while (ctr < no) {
+            if (debug) System.out.println("\nSubTree: " + ctr + " probability: " + aspl.getProbability(ctr));
+            ListIterator<SpanNode> lIter = aspl.childIterator(ctr);
+            while (lIter.hasNext()) {
+                SpanNode spnNode = lIter.next();
+                if (debug) System.out.println("Parsing span node: [" + spnNode.getFrom() + ", " + spnNode.getTo() + "] ");
+                if (spnNode instanceof AlternateSpanList) {
+                    if (debug) System.out.println("A child alternate span list found. Recursing");
+                    parseAlternateLists((AlternateSpanList)spnNode);
+                } else if (spnNode instanceof SpanList) {
+                    if (debug) System.out.println("A child span list found. Recursing");
+                    consumeAnnotations((SpanList)spnNode);
+                } else {
+                    //System.out.println("Span Node (from alternate spanlist): " + spnNode);
+                    getAnnotationsForNode(spnNode);
+                }
+            }
+            ctr ++;
+        }
+    }
+
+
+    public void parseFieldForAnnotations(StringFieldValue sfv) {
+        Collection<SpanTree> c = sfv.getSpanTrees();
+        Iterator<SpanTree> iiter = c.iterator();
+        while (iiter.hasNext()) {
+            if (debug) System.out.println(sfv + " has annotations");
+            tree = iiter.next();
+            SpanList root = (SpanList) tree.getRoot();
+            consumeAnnotations(root);
+        }
+    }
+
+
+    public void getAnnotationsForNode(SpanNode node) {
+        Iterator<Annotation> iter = tree.iterator(node);
+        boolean annotationPresent = false;
+		while (iter.hasNext()) {
+            annotationPresent = true;
+			Annotation xx = iter.next();
+			AnnotationType t = xx.getType();
+			StringFieldValue fValue = (StringFieldValue) xx.getFieldValue();
+			if (debug) System.out.println("Annotation: " + xx);
+            if (fValue == null) {
+                if (debug) System.out.println("Field Value is null");
+                return;
+            } else {
+                if (debug) System.out.println("Field Value: " + fValue.getString());
+            }
+		}
+        if (!annotationPresent) {
+            if (debug) System.out.println("****No annotations found for the span node: [" + node.getFrom() + ", " + node.getTo() + "] ");
+        }
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/annotation/SpanListTestCase.java b/document/src/test/java/com/yahoo/document/annotation/SpanListTestCase.java
new file mode 100755
index 00000000000..73e2c65fd5f
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/annotation/SpanListTestCase.java
@@ -0,0 +1,354 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.document.serialization.*;
+import com.yahoo.io.GrowableByteBuffer;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.fail;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class SpanListTestCase extends AbstractTypesTest {
+
+    @Test
+    public void testSerializeDeserialize() {
+        {
+            SpanList spanList = new SpanList();
+            serializeAndAssert(spanList);
+        }
+        {
+            SpanList spanList = new SpanList();
+            Span s1 = new Span(1, 2);
+            Span s2 = new Span(3, 4);
+            Span s3 = new Span(4, 5);
+            spanList.add(s1).add(s2).add(s3);
+            SpanList s4 = new SpanList();
+            Span s5 = new Span(7, 8);
+            Span s6 = new Span(8, 9);
+            s4.add(s5).add(s6);
+            spanList.add(s4);
+
+            serializeAndAssert(spanList);
+        }
+    }
+
+    private void serializeAndAssert(SpanList spanList) {
+        GrowableByteBuffer buffer;
+        {
+            buffer = new GrowableByteBuffer(1024);
+            DocumentSerializer serializer = DocumentSerializerFactory.create42(buffer);
+            StringFieldValue value = new StringFieldValue("lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj lk");
+            SpanTree tree = new SpanTree("bababa", spanList);
+            value.setSpanTree(tree);
+            serializer.write(null, value);
+            buffer.flip();
+        }
+        SpanList spanList2;
+        {
+            DocumentDeserializer deserializer = DocumentDeserializerFactory.create42(man, buffer);
+            StringFieldValue value = new StringFieldValue();
+            deserializer.read(null, value);
+            spanList2 = (SpanList)value.getSpanTree("bababa").getRoot();
+        }
+
+        assertEquals(spanList, spanList2);
+        assertNotSame(spanList, spanList2);
+    }
+
+    @Test
+    public void testFromAndTo() {
+        Span s1 = new Span(1, 2);
+        Span s2 = new Span(5, 10);
+
+        SpanList list = new SpanList();
+
+        assertEquals(-1, list.getFrom());
+        assertEquals(-1, list.getTo());
+
+        list.add(s1);
+        assertEquals(1, list.getFrom());
+        assertEquals(1, list.getFrom());
+        assertEquals(3, list.getTo());
+        assertEquals(3, list.getTo());
+
+        list.add(s2);
+        assertEquals(1, list.getFrom());
+        assertEquals(1, list.getFrom());
+        assertEquals(15, list.getTo());
+        assertEquals(15, list.getTo());
+
+        list.clearChildren();
+        assertEquals(-1, list.getFrom());
+        assertEquals(-1, list.getTo());
+
+        s1 = new Span(1, 2);
+        s2 = new Span(5, 10);
+
+        list.add(s1);
+        assertEquals(1, list.getFrom());
+        assertEquals(1, list.getFrom());
+        assertEquals(3, list.getTo());
+        assertEquals(3, list.getTo());
+
+        list.add(s2);
+        assertEquals(1, list.getFrom());
+        assertEquals(1, list.getFrom());
+        assertEquals(15, list.getTo());
+        assertEquals(15, list.getTo());
+
+        list.remove(s1);
+        assertEquals(5, list.getFrom());
+        assertEquals(5, list.getFrom());
+        assertEquals(15, list.getTo());
+        assertEquals(15, list.getTo());
+
+        list.remove(s2);
+        assertEquals(-1, list.getFrom());
+        assertEquals(-1, list.getTo());
+    }
+
+    @Test
+    public void testSortRecursive() {
+        SpanList root = new SpanList();
+        Span a1 = new Span(0, 1);
+        SpanList b1 = new SpanList();
+        SpanList c1 = new SpanList();
+        Span d1 = new Span(9, 1);
+        root.add(d1).add(c1).add(a1).add(b1);
+
+        Span aB2 = new Span(1, 1);
+        Span bB2 = new Span(2, 1);
+        Span cB2 = new Span(3, 1);
+        Span dB2 = new Span(4, 1);
+        b1.add(dB2).add(cB2).add(bB2).add(aB2);
+
+        Span aC2 = new Span(5, 1);
+        Span bC2 = new Span(6, 1);
+        Span cC2 = new Span(7, 1);
+        Span dC2 = new Span(8, 1);
+        c1.add(cC2).add(aC2).add(bC2).add(dC2);
+
+        root.sortChildrenRecursive();
+        assertSame(a1, root.children().get(0));
+        assertSame(b1, root.children().get(1));
+        assertSame(c1, root.children().get(2));
+        assertSame(d1, root.children().get(3));
+        assertSame(aB2, b1.children().get(0));
+        assertSame(bB2, b1.children().get(1));
+        assertSame(cB2, b1.children().get(2));
+        assertSame(dB2, b1.children().get(3));
+        assertSame(aC2, c1.children().get(0));
+        assertSame(bC2, c1.children().get(1));
+        assertSame(cC2, c1.children().get(2));
+        assertSame(dC2, c1.children().get(3));
+    }
+
+    @Test
+    public void testTwoLevelFromAndTo() {
+        SpanList root = new SpanList();
+        SpanList l1 = new SpanList();
+        root.add(l1);
+
+        Span s1 = new Span(0, 20);
+        Span s2 = new Span(20, 20);
+
+        l1.add(s1).add(s2);
+
+        assertEquals(0, root.getFrom());
+        assertEquals(40, root.getTo());
+        assertEquals(0, l1.getFrom());
+        assertEquals(40, l1.getTo());
+
+        Span s3 = new Span(40, 20);
+        l1.add(s3);
+
+        assertEquals(0, root.getFrom());
+        assertEquals(60, root.getTo());
+        assertEquals(0, l1.getFrom());
+        assertEquals(60, l1.getTo());
+    }
+
+    @Test
+    public void testAddingToManyRoots() {
+        Span s1 = new Span(1, 1);
+        Span s2 = new Span(2, 1);
+        Span s3 = new Span(3, 1);
+
+        SpanList sl1 = new SpanList();
+        sl1.add(s1).add(s2).add(s3);
+
+        SpanList sl2 = new SpanList();
+        try {
+            sl2.add(s1).add(s2).add(s3);
+            fail("Should have failed here!!");
+        } catch (IllegalStateException ise) {
+            //OK!
+        }
+
+        SpanTree tree = new SpanTree("foo", sl1);
+        assertSame(tree, sl1.getParent());
+        assertSame(sl1, tree.getRoot());
+
+        SpanList sl3 = new SpanList();
+        sl1.add(sl3);
+        assertSame(sl3, sl1.children().get(3));
+        assertSame(sl1, sl3.getParent());
+
+        assertSame(tree, sl3.getSpanTree());
+        assertSame(tree, sl1.getSpanTree());
+
+        assertNull(sl2.getSpanTree());
+    }
+
+    @Test
+    public void testRemoveInvalidate() {
+        SpanList sl1 = new SpanList();
+        Span s1 = new Span(1, 2);
+
+        sl1.add(s1);
+
+        SpanList sl2 = new SpanList();
+        try {
+            sl2.add(s1);
+            fail("Should have failed.");
+        } catch (IllegalStateException ise) {
+            //OK!
+        }
+
+        sl1.remove(0);
+
+        try {
+            sl2.add(s1);
+            fail("Should have failed.");
+        } catch (IllegalStateException ise) {
+            //OK!
+        }
+    }
+
+    @Test
+    public void testMoveSimple() {
+        SpanList sl1 = new SpanList();
+        SpanList sl2 = new SpanList();
+        Span s1 = new Span(1, 2);
+
+        sl1.add(s1);
+        assertEquals(1, sl1.children().size());
+        assertEquals(0, sl2.children().size());
+        sl1.move(s1, sl2);
+        assertEquals(0, sl1.children().size());
+        assertEquals(1, sl2.children().size());
+        sl2.move(0, sl1);
+        assertEquals(1, sl1.children().size());
+        assertEquals(0, sl2.children().size());
+    }
+
+    @Test
+    public void testMoveAlternate() {
+        AlternateSpanList asl1 = new AlternateSpanList();
+        AlternateSpanList asl2 = new AlternateSpanList();
+        Span s1 = new Span(1, 2);
+
+        asl1.add(s1);
+        assertEquals(1, asl1.children().size());
+        assertEquals(0, asl2.children().size());
+        asl1.move(s1, asl2);
+        assertEquals(0, asl1.children().size());
+        assertEquals(1, asl2.children().size());
+        asl2.move(0, asl1);
+        assertEquals(1, asl1.children().size());
+        assertEquals(0, asl2.children().size());
+    }
+
+    @Test
+    public void testMoveAlternateAdvances() {
+        AlternateSpanList asl1 = new AlternateSpanList();
+        AlternateSpanList asl2 = new AlternateSpanList();
+        Span s1 = new Span(1, 2);
+
+        asl1.addChildren(new ArrayList<SpanNode>(), 50d);
+        asl1.addChildren(new ArrayList<SpanNode>(), 50d);
+        asl2.addChildren(new ArrayList<SpanNode>(), 50d);
+        asl2.addChildren(new ArrayList<SpanNode>(), 50d);
+
+        asl1.add(s1);
+        assertEquals(1, asl1.children(0).size());
+        assertEquals(0, asl1.children(1).size());
+        assertEquals(0, asl1.children(2).size());
+        assertEquals(0, asl2.children(0).size());
+        assertEquals(0, asl2.children(1).size());
+        assertEquals(0, asl2.children(2).size());
+        asl1.move(s1, asl2, 2);
+        assertEquals(0, asl1.children(0).size());
+        assertEquals(0, asl1.children(1).size());
+        assertEquals(0, asl1.children(2).size());
+        assertEquals(0, asl2.children(0).size());
+        assertEquals(0, asl2.children(1).size());
+        assertEquals(1, asl2.children(2).size());
+        asl2.move(2, 0, asl1, 1);
+        assertEquals(0, asl1.children(0).size());
+        assertEquals(1, asl1.children(1).size());
+        assertEquals(0, asl1.children(2).size());
+        assertEquals(0, asl2.children(0).size());
+        assertEquals(0, asl2.children(1).size());
+        assertEquals(0, asl2.children(2).size());
+    }
+
+    @Test
+    public void testGetStringFieldValue() {
+        StringFieldValue text = getAnnotatedString();
+        {
+            SpanTree tree = text.getSpanTree("ballooo");
+            assertSame(text, tree.getStringFieldValue());
+
+            AlternateSpanList root = (AlternateSpanList)tree.getRoot();
+            Iterator<SpanNode> it = root.childIteratorRecursive();
+
+            while (it.hasNext()) {
+                assertSame(text, it.next().getStringFieldValue());
+            }
+        }
+        {
+            SpanTree tree = text.getSpanTree("fruits");
+            assertSame(text, tree.getStringFieldValue());
+
+            SpanList root = (SpanList)tree.getRoot();
+            Iterator<SpanNode> it = root.childIteratorRecursive();
+
+            while (it.hasNext()) {
+                assertSame(text, it.next().getStringFieldValue());
+            }
+        }
+        {
+            SpanTree tree = text.removeSpanTree("ballooo");
+            assertNull(tree.getStringFieldValue());
+
+            AlternateSpanList root = (AlternateSpanList)tree.getRoot();
+            Iterator<SpanNode> it = root.childIteratorRecursive();
+
+            while (it.hasNext()) {
+                assertNull(it.next().getStringFieldValue());
+            }
+        }
+        {
+            SpanTree tree = text.removeSpanTree("fruits");
+            assertNull(tree.getStringFieldValue());
+
+            SpanList root = (SpanList)tree.getRoot();
+            Iterator<SpanNode> it = root.childIteratorRecursive();
+
+            while (it.hasNext()) {
+                assertNull(it.next().getStringFieldValue());
+            }
+        }
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/annotation/SpanNodeAdvTestCase.java b/document/src/test/java/com/yahoo/document/annotation/SpanNodeAdvTestCase.java
new file mode 100644
index 00000000000..9277d939168
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/annotation/SpanNodeAdvTestCase.java
@@ -0,0 +1,333 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.datatypes.StringFieldValue;
+import org.junit.After;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.ListIterator;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author <a href="mailto:mpraveen@yahoo-inc.com">Praveen Mohan</a>
+ *
+ * This test covers all possible scenarios in SpanNode.
+ */
+
+
+
+public class SpanNodeAdvTestCase {
+
+    private boolean debug = false;
+
+    private AnnotationType at1 = new AnnotationType("person", DataType.STRING);
+    private AnnotationType at2 = new AnnotationType("street", DataType.STRING);
+    private AnnotationType at3 = new AnnotationType("city", DataType.STRING);
+
+    public SpanTree tree = null;
+
+    private SpanNode span1, span2, span3;
+    private SpanNode span11, span22, span33;
+    private SpanNode span111, span222, span333;
+
+    private AlternateSpanList root, alternate1, alternate2;
+    private SpanList branch, branch1;
+    private List<SpanNode> subtreeList1, subtreeList2, subtreeList3, subtreeList4;
+    private Annotation an111;
+
+
+    public void populateSpanTree() {
+
+        root = new AlternateSpanList();
+        tree = new SpanTree("test", root);
+
+        span1 = new Span(10, 8);
+        span2 = new Span(5, 10);
+        span3 = new Span(13, 2);
+
+        span11 = new Span(0, 2);
+        span22 = new Span(2, 10);
+        span33 = new Span(8, 10);
+
+        span111 = new Span(5, 10);
+        span222 = new Span(10, 10);
+        span333 = new Span(20, 10);
+
+        an111 = new Annotation(at1);
+
+        tree.annotate(span1, at1).annotate(span2, at2).annotate(span3, at3);
+        tree.annotate(span11, at1).annotate(span22, at2).annotate(span33, at3);
+        tree.annotate(span111, an111).annotate(span222, at2).annotate(span333, at3).annotate(span333, at1);
+        tree.annotate(span222, at2);
+
+        root.add(span3);
+        root.add(span2);
+        root.add(span1);
+
+        alternate1 = new AlternateSpanList();
+        alternate1.add(span11);
+        branch = new SpanList();
+        branch.add(span22);
+        subtreeList1 = new ArrayList<SpanNode>();
+        subtreeList1.add(branch);
+        alternate1.addChildren(1, subtreeList1, 50.0d);
+
+        alternate2 = new AlternateSpanList();
+        alternate2.add(span33);
+        subtreeList2 = new ArrayList<SpanNode>();
+        subtreeList2.add(span111);
+        subtreeList2.add(span222);
+        alternate2.addChildren(1, subtreeList2, 70.0d);
+
+        subtreeList3 = new ArrayList<SpanNode>();
+        branch1 = new SpanList();
+        branch1.add(span333);
+        subtreeList3.add(branch1);
+        alternate2.addChildren(2, subtreeList3, 90.0d);
+        branch.add(alternate2);
+
+        subtreeList4 = new ArrayList<SpanNode>();
+        subtreeList4.add(alternate1);
+        root.addChildren(1, subtreeList4, 10.0d);
+    }
+
+	@Test
+	public void assertIsValid() {
+		populateSpanTree();
+		root.clearChildren(1);
+		root.remove(span3);
+
+        assertTrue(root.isValid());
+        assertTrue(span1.isValid());
+        assertTrue(span2.isValid());
+
+        assertFalse(span3.isValid());
+		assertFalse(alternate1.isValid());
+        assertFalse(span11.isValid());
+		assertFalse(branch.isValid());
+		assertFalse(span22.isValid());
+		assertFalse(alternate2.isValid());
+		assertFalse(span33.isValid());
+		assertFalse(branch1.isValid());
+		assertFalse(span333.isValid());
+		assertFalse(span111.isValid());
+		assertFalse(span222.isValid());
+	}
+
+    @Test
+	public void assertIsLeafNode() {
+		populateSpanTree();
+		assertFalse(root.isLeafNode());
+		assertFalse(alternate1.isLeafNode());
+		assertFalse(branch.isLeafNode());
+		assertTrue(span11.isLeafNode());
+		assertTrue(span22.isLeafNode());
+		assertFalse(alternate2.isLeafNode());
+		assertTrue(span33.isLeafNode());
+		assertFalse(branch1.isLeafNode());
+		assertTrue(span111.isLeafNode());
+		assertTrue(span222.isLeafNode());
+		assertTrue(span333.isLeafNode());
+	}
+
+	@Test
+	public void assertOverlaps() {
+		populateSpanTree();
+		assertTrue(span1.overlaps(span2));
+		assertTrue(span1.overlaps(span3));
+		assertTrue(span2.overlaps(span3));
+		assertFalse(span11.overlaps(span22));
+		assertTrue(span22.overlaps(span33));
+		assertFalse(span11.overlaps(span33));
+		assertTrue(span111.overlaps(span222));
+		assertFalse(span111.overlaps(span333));
+		assertFalse(span222.overlaps(span333));
+		assertTrue(span1.overlaps(span222));
+		assertFalse(span1.overlaps(span333));
+		assertTrue(span2.overlaps(span222));
+		assertFalse(span3.overlaps(span22));
+
+		assertTrue(span2.overlaps(span1));
+		assertTrue(span3.overlaps(span1));
+		assertTrue(span3.overlaps(span2));
+		assertFalse(span22.overlaps(span11));
+		assertTrue(span33.overlaps(span22));
+		assertFalse(span33.overlaps(span11));
+		assertTrue(span222.overlaps(span111));
+		assertFalse(span333.overlaps(span111));
+		assertFalse(span333.overlaps(span222));
+		assertTrue(span222.overlaps(span1));
+		assertFalse(span333.overlaps(span1));
+		assertTrue(span222.overlaps(span2));
+		assertTrue(span1.overlaps(span1));
+		assertFalse(span22.overlaps(span3));
+		assertFalse(root.overlaps(alternate1));
+		assertTrue(alternate2.overlaps(span22));
+		assertTrue(branch.overlaps(root));
+		assertTrue(branch.overlaps(alternate2));
+		assertTrue(root.overlaps(alternate2));
+		assertTrue(span22.overlaps(root));
+
+	}
+
+	@Test
+	public void assertContains() {
+		populateSpanTree();
+		assertTrue(span222.contains(span1));
+		assertFalse(span1.contains(span222));
+		assertTrue(span1.contains(span3));
+		assertFalse(span33.contains(span111));
+		assertTrue(span222.contains(span3));
+		assertTrue(span111.contains(span2));
+		assertTrue(span2.contains(span111));
+		assertTrue(branch.contains(root));
+		assertTrue(branch.contains(alternate2));
+		assertTrue(root.contains(alternate2));
+		assertFalse(alternate2.contains(span22));
+	}
+
+    @Test
+	public void assertCompareTo() {
+		populateSpanTree();
+		assertEquals(1 , span1.compareTo(span2));
+		assertEquals(-1, span2.compareTo(span1));
+		assertEquals(-1 , span2.compareTo(span3));
+		assertEquals(1, span3.compareTo(span2));
+		assertEquals(0, span2.compareTo(span111));
+		assertEquals(1, root.compareTo(branch));
+		assertEquals(-1, alternate1.compareTo(root));
+		assertEquals(1, branch.compareTo(span22));
+		assertEquals(-1, branch.compareTo(alternate2));
+		assertEquals(1, alternate2.compareTo(root));
+		assertEquals(-1, span111.compareTo(root));
+		assertEquals(0, span333.compareTo(branch1));
+		assertEquals(0, alternate2.compareTo(span33));
+		root.removeChildren();
+		tree.cleanup();
+		assertEquals(1, span11.compareTo(root));
+	}
+
+    @Test
+	public void assertGetParent() {
+		populateSpanTree();
+		assertEquals(root, span1.getParent());
+		assertEquals(root, span2.getParent());
+        assertEquals(root, span3.getParent());
+		assertEquals(root, alternate1.getParent());
+		assertEquals(alternate1, span11.getParent());
+		assertEquals(alternate1, branch.getParent());
+		assertEquals(branch, span22.getParent());
+		assertEquals(branch, alternate2.getParent());
+		assertEquals(alternate2, span33.getParent());
+		assertEquals(alternate2, span111.getParent());
+		assertEquals(branch1, span333.getParent());
+		assertEquals(alternate2, branch1.getParent());
+		assertEquals(alternate1, span11.getParent());
+	}
+
+
+    @After
+    public void tearDown() {
+        tree = null;
+    }
+
+
+    public void consumeAnnotations(SpanList root) {
+        if (root instanceof AlternateSpanList) {
+            parseAlternateLists((AlternateSpanList)root);
+            if (debug) System.out.println("\nGetting annotations for the SpanList itself : [" + root.getFrom() + ", " + root.getTo() + "] ");
+            getAnnotationsForNode(root);
+            return;
+        }
+        if (debug) System.out.println("\n\nSpanList: [" + root.getFrom() + ", " + root.getTo() + "] num Children: " + root.numChildren());
+        if (debug) System.out.println("-------------------");
+        Iterator<SpanNode> childIterator = root.childIterator();
+		while (childIterator.hasNext()) {
+			SpanNode node = childIterator.next();
+            //System.out.println("Span Node: " + node); // + " Span Text: " + node.getText(fieldValStr));
+            if (debug) System.out.println("\n\nSpan Node: [" + node.getFrom() + ", " + node.getTo() + "] ");
+            if (node instanceof AlternateSpanList) {
+                parseAlternateLists((AlternateSpanList)node);
+                if (debug) System.out.println("---- Alternate SpanList complete ---");
+            } else if (node instanceof SpanList) {
+                if (debug) System.out.println("Encountered another span list");
+                SpanList spl = (SpanList) node;
+                ListIterator<SpanNode> lli = spl.childIterator();
+                while (lli.hasNext()) System.out.print(" " + lli.next() + " ");
+                consumeAnnotations((SpanList) node);
+            } else {
+       			if (debug) System.out.println("\nGetting annotations for this span node: [" + node.getFrom() + ", " + node.getTo() + "] ");
+   	    	    getAnnotationsForNode(node);
+            }
+		}
+        if (debug) System.out.println("\nGetting annotations for the SpanList itself : [" + root.getFrom() + ", " + root.getTo() + "] ");
+        getAnnotationsForNode(root);
+    }
+
+    public void parseAlternateLists(AlternateSpanList aspl) {
+        int no = aspl.getNumSubTrees();
+        if (debug) System.out.println("Parsing Alternate span list. No of subtrees: " + no);
+        int ctr = 0;
+        while (ctr < no) {
+            if (debug) System.out.println("\nSubTree: " + ctr + " probability: " + aspl.getProbability(ctr));
+            ListIterator<SpanNode> lIter = aspl.childIterator(ctr);
+            while (lIter.hasNext()) {
+                SpanNode spnNode = lIter.next();
+                if (debug) System.out.println("Parsing span node: [" + spnNode.getFrom() + ", " + spnNode.getTo() + "] ");
+                if (spnNode instanceof AlternateSpanList) {
+                    if (debug) System.out.println("A child alternate span list found. Recursing");
+                    parseAlternateLists((AlternateSpanList)spnNode);
+                } else if (spnNode instanceof SpanList) {
+                    if (debug) System.out.println("A child span list found. Recursing");
+                    consumeAnnotations((SpanList)spnNode);
+                } else {
+                    //System.out.println("Span Node (from alternate spanlist): " + spnNode);
+                    getAnnotationsForNode(spnNode);
+                }
+            }
+            ctr ++;
+        }
+    }
+
+
+    public void parseFieldForAnnotations(StringFieldValue sfv) {
+        Collection<SpanTree> c = sfv.getSpanTrees();
+        Iterator<SpanTree> iiter = c.iterator();
+        while (iiter.hasNext()) {
+            if (debug) System.out.println(sfv + " has annotations");
+            tree = iiter.next();
+            SpanList root = (SpanList) tree.getRoot();
+            consumeAnnotations(root);
+        }
+    }
+
+
+    public void getAnnotationsForNode(SpanNode node) {
+        Iterator<Annotation> iter = tree.iterator(node);
+        boolean annotationPresent = false;
+		while (iter.hasNext()) {
+            annotationPresent = true;
+			Annotation xx = iter.next();
+			AnnotationType t = xx.getType();
+			StringFieldValue fValue = (StringFieldValue) xx.getFieldValue();
+			if (debug) System.out.println("Annotation: " + xx);
+            if (fValue == null) {
+                if (debug) System.out.println("Field Value is null");
+                return;
+            } else {
+                if (debug) System.out.println("Field Value: " + fValue.getString());
+            }
+		}
+        if (!annotationPresent) {
+            if (debug) System.out.println("****No annotations found for the span node: [" + node.getFrom() + ", " + node.getTo() + "] ");
+        }
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/annotation/SpanNodeTestCase.java b/document/src/test/java/com/yahoo/document/annotation/SpanNodeTestCase.java
new file mode 100644
index 00000000000..b7d17779ea9
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/annotation/SpanNodeTestCase.java
@@ -0,0 +1,646 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.ListIterator;
+import java.util.NoSuchElementException;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class SpanNodeTestCase extends junit.framework.TestCase {
+
+    public void testOverlaps() {
+        //qwertyuiopasdfghjklzxcvbnm
+        //012345678901234567890123456789
+        Span a = new Span(0, 6);  //qwerty
+        Span b = new Span(6, 6);  //uiopas
+        Span c = new Span(12, 5); //dfghj
+        Span d = new Span(17, 6); //klzxcv
+        Span e = new Span(23, 3); //bnm
+
+        assertFalse(a.overlaps(b));
+        assertFalse(b.overlaps(a));
+
+        assertFalse(b.overlaps(c));
+        assertFalse(c.overlaps(b));
+
+        assertFalse(c.overlaps(d));
+        assertFalse(d.overlaps(c));
+
+        assertFalse(d.overlaps(e));
+        assertFalse(e.overlaps(d));
+
+
+        Span all = new Span(0, 25);
+
+        assertTrue(a.overlaps(all));
+        assertTrue(all.overlaps(a));
+
+        assertTrue(b.overlaps(all));
+        assertTrue(all.overlaps(b));
+
+        assertTrue(c.overlaps(all));
+        assertTrue(all.overlaps(c));
+
+        assertTrue(all.overlaps(d));
+        assertTrue(d.overlaps(all));
+
+        assertTrue(all.overlaps(e));
+        assertTrue(e.overlaps(all));
+
+
+        Span f = new Span(3, 7); // rtyuiop
+
+        assertTrue(a.overlaps(f));
+        assertTrue(f.overlaps(a));
+
+        assertTrue(b.overlaps(f));
+        assertTrue(f.overlaps(b));
+
+        assertFalse(c.overlaps(f));
+        assertFalse(f.overlaps(c));
+
+        assertFalse(d.overlaps(f));
+        assertFalse(f.overlaps(d));
+
+        assertFalse(e.overlaps(f));
+        assertFalse(f.overlaps(e));
+
+        assertTrue(all.overlaps(f));
+        assertTrue(f.overlaps(all));
+
+
+        Span g = new Span(0, 7);  //qwertyu
+
+        assertTrue(a.overlaps(g));
+        assertTrue(g.overlaps(a));
+
+        assertTrue(b.overlaps(g));
+        assertTrue(g.overlaps(b));
+
+        assertFalse(c.overlaps(g));
+        assertFalse(g.overlaps(c));
+
+        assertFalse(d.overlaps(g));
+        assertFalse(g.overlaps(d));
+
+        assertFalse(e.overlaps(g));
+        assertFalse(g.overlaps(e));
+
+        assertTrue(f.overlaps(g));
+        assertTrue(g.overlaps(f));
+
+        assertTrue(all.overlaps(g));
+        assertTrue(g.overlaps(all));
+
+
+        assertTrue(a.overlaps(a));
+    }
+
+    public void testContains() {
+        //qwertyuiopasdfghjklzxcvbnm
+        //012345678901234567890123456789
+        Span a = new Span(0, 6);  //qwerty
+        Span b = new Span(6, 6);  //uiopas
+        Span all = new Span(0, 25);
+
+        assertFalse(a.contains(all));
+        assertTrue(all.contains(a));
+
+        assertFalse(a.contains(b));
+        assertFalse(b.contains(a));
+
+        assertTrue(a.contains(a));
+
+        Span c = new Span(0, 7);  //qwertyu
+
+        assertFalse(a.contains(c));
+        assertTrue(c.contains(a));
+    }
+
+
+    public void testSpanTree() {
+        final String text = "Hallo er et ord fra Norge";
+        SpanList sentence = new SpanList();
+        SpanTree tree = new SpanTree("sentence", sentence);
+
+        //Locate words and whitespace:
+        SpanNode hallo = new Span(0, 5);
+        SpanNode spc1 = new Span(5, 1);
+        SpanNode er = new Span(6, 2);
+        SpanNode spc2 = new Span(8, 1);
+        SpanNode et = new Span(9, 2);
+        SpanNode spc3 = new Span(11, 1);
+        SpanNode ord = new Span(12, 3);
+        SpanNode spc4 = new Span(15, 1);
+        SpanNode fra = new Span(16, 3);
+        SpanNode spc5 = new Span(19, 1);
+        SpanNode norge = new Span(20, 5);
+
+        AnnotationType noun = new AnnotationType("noun");
+        AnnotationType verb = new AnnotationType("verb");
+        AnnotationType prep = new AnnotationType("preposition");
+        AnnotationType det = new AnnotationType("determiner");
+        AnnotationType noun_phrase = new AnnotationType("noun_phrase");
+        AnnotationType verb_phrase = new AnnotationType("verb_phrase");
+        AnnotationType prep_phrase = new AnnotationType("preposition_phrase");
+        AnnotationType separator = new AnnotationType("separator");
+        AnnotationType sentenceType = new AnnotationType("sentence");
+
+        //Determine word classes and add annotation for them:
+        tree.annotate(hallo, noun);
+        tree.annotate(spc1, separator);
+        tree.annotate(er, verb);
+        tree.annotate(spc2, separator);
+        tree.annotate(et, det);
+        tree.annotate(spc3, separator);
+        tree.annotate(ord, noun);
+        tree.annotate(spc4, separator);
+        tree.annotate(fra, prep);
+        tree.annotate(spc5, separator);
+        tree.annotate(norge, noun);
+
+
+        //Identify phrases and build natural language parse tree, and annotate as we go:
+        tree.annotate(hallo, noun_phrase);
+
+        SpanList np2 = new SpanList();
+        np2.children().add(et);
+        np2.children().add(spc3);
+        np2.children().add(ord);
+        tree.annotate(np2, noun_phrase);
+
+        tree.annotate(norge, noun_phrase);
+
+        SpanList pp = new SpanList();
+        pp.children().add(fra);
+        pp.children().add(spc5);
+        pp.children().add(norge);
+        tree.annotate(pp, prep_phrase);
+
+        SpanList np3 = new SpanList();
+        np3.children().add(np2);
+        np3.children().add(spc4);
+        np3.children().add(pp);
+        tree.annotate(np3, noun_phrase);
+
+        SpanList vp = new SpanList();
+        vp.children().add(er);
+        vp.children().add(spc2);
+        vp.children().add(np3);
+        tree.annotate(vp, verb_phrase);
+
+        sentence.children().add(hallo);
+        sentence.children().add(spc1);
+        sentence.children().add(vp);
+        tree.annotate(sentence, sentenceType);
+
+
+        //assert that extracted text is correct:
+        assertEquals("Hallo", hallo.getText(text));
+        assertEquals(" ", spc1.getText(text));
+        assertEquals("er", er.getText(text));
+        assertEquals(" ", spc2.getText(text));
+        assertEquals("et", et.getText(text));
+        assertEquals(" ", spc3.getText(text));
+        assertEquals("ord", ord.getText(text));
+        assertEquals(" ", spc4.getText(text));
+        assertEquals("fra", fra.getText(text));
+        assertEquals(" ", spc5.getText(text));
+        assertEquals("Norge", norge.getText(text));
+
+        assertEquals("et ord", np2.getText(text).toString());
+        assertEquals("fra Norge", pp.getText(text).toString());
+        assertEquals("et ord fra Norge", np3.getText(text).toString());
+        assertEquals("er et ord fra Norge", vp.getText(text).toString());
+        assertEquals("Hallo er et ord fra Norge", sentence.getText(text).toString());
+
+        //coming up: assert that children(Annotation) works...
+    }
+
+    public void testOrder() {
+        {
+            String text = "08/20/1999";
+                         //012345678901
+            Span d = new Span(3, 2);
+            Span m = new Span(0, 2);
+            Span y = new Span(6, 4);
+
+            SpanList date = new SpanList();
+            date.children().add(d);
+            date.children().add(m);
+            date.children().add(y);
+
+            assertEquals("20081999", date.getText(text).toString());
+
+            assertEquals(0, date.getFrom());
+            assertEquals(10, date.getTo());
+            assertEquals(10, date.getLength());
+        }
+        {
+            String text = "20/08/2000";
+            //012345678901
+            Span d = new Span(0, 2);
+            Span m = new Span(3, 2);
+            Span y = new Span(6, 4);
+
+            SpanList date = new SpanList();
+            date.children().add(d);
+            date.children().add(m);
+            date.children().add(y);
+
+            assertEquals("20082000", date.getText(text).toString());
+
+            assertEquals(0, date.getFrom());
+            assertEquals(10, date.getTo());
+            assertEquals(10, date.getLength());
+        }
+    }
+
+    public void testNonRecursiveAnnotationIterator() {
+        AnnotationType nounType = new AnnotationType("noun");
+        AnnotationType detType = new AnnotationType("determiner");
+        AnnotationType wordType = new AnnotationType("word");
+        AnnotationType begTagType = new AnnotationType("begin_tag");
+        AnnotationType cmpWordType = new AnnotationType("compound_word");
+
+        SpanNode span = new Span(0,2);
+        SpanTree tree = new SpanTree("span", span);
+
+        Annotation word = new Annotation(wordType);
+        Annotation bgtg = new Annotation(begTagType);
+        Annotation cpwd = new Annotation(cmpWordType);
+        Annotation detr = new Annotation(detType);
+        Annotation noun = new Annotation(nounType);
+
+        tree.annotate(span, word);
+        tree.annotate(span, bgtg);
+        tree.annotate(span, cpwd);
+        tree.annotate(span, detr);
+        tree.annotate(span, noun);
+
+
+        {
+            Iterator<Annotation> it = tree.iterator();
+            assertSame(word, it.next());
+            assertSame(bgtg, it.next());
+            assertSame(cpwd, it.next());
+            assertSame(detr, it.next());
+            assertSame(noun, it.next());
+            try {
+                it.next();
+                fail("Should have gotten NoSuchElementException");
+            } catch (NoSuchElementException e) {
+                //ignore
+            }
+        }
+        {
+            Iterator<Annotation> it = tree.iterator();
+            for (int i = 0; i < 100; i++) {
+                assertTrue(it.hasNext());
+            }
+        }
+        {
+            Iterator<Annotation> it = tree.iterator();
+            assertTrue(it.hasNext());
+            assertSame(word, it.next());
+            assertTrue(it.hasNext());
+            assertSame(bgtg, it.next());
+            assertTrue(it.hasNext());
+            assertSame(cpwd, it.next());
+            assertTrue(it.hasNext());
+            assertSame(detr, it.next());
+            assertTrue(it.hasNext());
+            assertSame(noun, it.next());
+            assertFalse(it.hasNext());
+        }
+
+
+
+        {
+            Iterator<Annotation> it = tree.iterator();
+            assertSame(word, it.next());
+            assertSame(bgtg, it.next());
+            assertSame(cpwd, it.next());
+            assertSame(detr, it.next());
+            assertSame(noun, it.next());
+            try {
+                it.next();
+                fail("Should have gotten NoSuchElementException");
+            } catch (NoSuchElementException e) {
+                //ignore
+            }
+        }
+        {
+            Iterator<Annotation> it = tree.iterator();
+            for (int i = 0; i < 100; i++) {
+                assertTrue(it.hasNext());
+            }
+        }
+        {
+            Iterator<Annotation> it = tree.iterator();
+            assertTrue(it.hasNext());
+            assertSame(word, it.next());
+            assertTrue(it.hasNext());
+            assertSame(bgtg, it.next());
+            assertTrue(it.hasNext());
+            assertSame(cpwd, it.next());
+            assertTrue(it.hasNext());
+            assertSame(detr, it.next());
+            assertTrue(it.hasNext());
+            assertSame(noun, it.next());
+            assertFalse(it.hasNext());
+        }
+
+    }
+
+    public void testRecursion() {
+        AnnotationType noun = new AnnotationType("noun");
+        AnnotationType verb = new AnnotationType("verb");
+        AnnotationType sentenceType = new AnnotationType("sentence");
+        AnnotationType word = new AnnotationType("word");
+        AnnotationType phraseType = new AnnotationType("phrase");
+
+        String text = "There is no bizniz like showbizniz";
+                     //0123456789012345678901234567890123456789
+        SpanList sentence = new SpanList();
+        SpanTree tree = new SpanTree("sentence", sentence);
+
+        Span there = new Span(0, 5);
+        Span is = new Span(6, 2);
+        Span no = new Span(9, 2);
+        Span bizniz = new Span(12, 6);
+        Span like = new Span(19, 4);
+        Span showbizniz = new Span(24, 10);
+
+        tree.annotate(there, word);
+        tree.annotate(is, word);
+        tree.annotate(is, verb);
+        tree.annotate(no, word);
+        tree.annotate(bizniz, word);
+        tree.annotate(bizniz, noun);
+        tree.annotate(like, word);
+        tree.annotate(showbizniz, word);
+        tree.annotate(showbizniz, noun);
+
+        SpanList endPhrase = new SpanList();
+        endPhrase.add(like);
+        endPhrase.add(showbizniz);
+        tree.annotate(endPhrase, phraseType);
+
+        SpanList phrase = new SpanList();
+        phrase.children().add(no);
+        phrase.children().add(bizniz);
+        phrase.children().add(endPhrase);
+        tree.annotate(phrase, phraseType);
+
+        sentence.children().add(there);
+        sentence.children().add(is);
+        sentence.children().add(phrase);
+        tree.annotate(sentence, sentenceType);
+
+        Iterator<SpanNode> children;
+
+        children = sentence.childIteratorRecursive();
+
+        {
+            assertTrue(children.hasNext());
+            SpanNode next = children.next();
+            assertEquals("There", next.getText(text).toString());
+            List<Annotation> a = annotations(tree, next);
+            assertEquals(1, a.size());
+            assertEquals(a.get(0).getType(), word);
+        }
+        {
+            assertTrue(children.hasNext());
+            SpanNode next = children.next();
+            assertEquals("is", next.getText(text).toString());
+            List<Annotation> a = annotations(tree, next);
+            assertEquals(2, a.size());
+            assertEquals(a.get(0).getType(), word);
+            assertEquals(a.get(1).getType(), verb);
+        }
+        {
+            assertTrue(children.hasNext());
+            SpanNode next = children.next();
+            assertEquals("no", next.getText(text).toString());
+            List<Annotation> a = annotations(tree, next);
+            assertEquals(1, a.size());
+            assertEquals(a.get(0).getType(), word);
+        }
+        {
+            assertTrue(children.hasNext());
+            SpanNode next = children.next();
+            assertEquals("bizniz", next.getText(text).toString());
+            List<Annotation> a = annotations(tree, next);
+            assertEquals(2, a.size());
+            assertEquals(a.get(0).getType(), word);
+            assertEquals(a.get(1).getType(), noun);
+        }
+        {
+            assertTrue(children.hasNext());
+            SpanNode next = children.next();
+            assertEquals("like", next.getText(text).toString());
+            List<Annotation> a = annotations(tree, next);
+            assertEquals(1, a.size());
+            assertEquals(a.get(0).getType(), word);
+        }
+        {
+            assertTrue(children.hasNext());
+            SpanNode next = children.next();
+            assertEquals("showbizniz", next.getText(text).toString());
+            List<Annotation> a = annotations(tree, next);
+            assertEquals(2, a.size());
+            assertEquals(a.get(0).getType(), word);
+            assertEquals(a.get(1).getType(), noun);
+        }
+        {
+            assertTrue(children.hasNext());
+            SpanNode next = children.next();
+            assertEquals("likeshowbizniz", next.getText(text).toString());
+            List<Annotation> a = annotations(tree, next);
+            assertEquals(1, a.size());
+            assertEquals(a.get(0).getType(), phraseType);
+        }
+        {
+            assertTrue(children.hasNext());
+            SpanNode next = children.next();
+            assertEquals("nobiznizlikeshowbizniz", next.getText(text).toString());
+            List<Annotation> a = annotations(tree, next);
+            assertEquals(1, a.size());
+            assertEquals(a.get(0).getType(), phraseType);
+        }
+        {
+            assertFalse(children.hasNext());
+        }
+
+        List<Annotation> annotationList = new LinkedList<Annotation>();
+        for (Annotation a : tree) {
+            annotationList.add(a);
+        }
+        Collections.sort(annotationList);
+
+        Iterator<Annotation> annotations = annotationList.iterator();
+
+        {
+        assertTrue(annotations.hasNext());
+        Annotation annotation = annotations.next();
+        assertEquals("There", annotation.getSpanNode().getText(text));
+        assertEquals(word, annotation.getType());  //there: word
+        }
+        {
+        assertTrue(annotations.hasNext());
+        Annotation annotation = annotations.next();
+        assertEquals("Thereisnobiznizlikeshowbizniz", annotation.getSpanNode().getText(text).toString());
+        assertEquals(sentenceType, annotation.getType());  //sentence
+        }
+        {
+        assertTrue(annotations.hasNext());
+        Annotation annotation = annotations.next();
+        assertEquals("is", annotation.getSpanNode().getText(text));
+        assertEquals(word, annotation.getType());  //is: word
+        }
+        {
+        assertTrue(annotations.hasNext());
+        Annotation annotation = annotations.next();
+        assertEquals("is", annotation.getSpanNode().getText(text));
+        assertEquals(verb, annotation.getType());  //is: verb
+        }
+        {
+        assertTrue(annotations.hasNext());
+        Annotation annotation = annotations.next();
+        assertEquals("no", annotation.getSpanNode().getText(text));
+        assertEquals(word, annotation.getType());  //no: word
+        }
+        {
+        assertTrue(annotations.hasNext());
+        Annotation annotation = annotations.next();
+        assertEquals("nobiznizlikeshowbizniz", annotation.getSpanNode().getText(text).toString());
+        assertEquals(phraseType, annotation.getType());  //no bizniz like showbizniz: phrase
+        }
+        {
+        assertTrue(annotations.hasNext());
+        Annotation annotation = annotations.next();
+        assertEquals("bizniz", annotation.getSpanNode().getText(text));
+        assertEquals(word, annotation.getType());  //bizniz: word
+        }
+        {
+        assertTrue(annotations.hasNext());
+        Annotation annotation = annotations.next();
+        assertEquals("bizniz", annotation.getSpanNode().getText(text));
+        assertEquals(noun, annotation.getType());  //bizniz: noun
+        }
+        {
+        assertTrue(annotations.hasNext());
+        Annotation annotation = annotations.next();
+        assertEquals("like", annotation.getSpanNode().getText(text));
+        assertEquals(word, annotation.getType());  //like: word
+        }
+        {
+        assertTrue(annotations.hasNext());
+        Annotation annotation = annotations.next();
+        assertEquals("likeshowbizniz", annotation.getSpanNode().getText(text).toString());
+        assertEquals(phraseType, annotation.getType());  //like showbizniz: phrase
+        }
+        {
+        assertTrue(annotations.hasNext());
+        Annotation annotation = annotations.next();
+        assertEquals("showbizniz", annotation.getSpanNode().getText(text));
+        assertEquals(word, annotation.getType());  //showbizniz: word
+        }
+        {
+        assertTrue(annotations.hasNext());
+        Annotation annotation = annotations.next();
+        assertEquals("showbizniz", annotation.getSpanNode().getText(text));
+        assertEquals(noun, annotation.getType());  //showbizniz: noun
+        }
+
+        assertFalse(annotations.hasNext());
+    }
+
+    private static List<Annotation> annotations(SpanTree tree, SpanNode node) {
+        List<Annotation> list = new ArrayList<Annotation>();
+        Iterator<Annotation> it = tree.iterator(node);
+        while (it.hasNext()) {
+            list.add(it.next());
+        }
+        return list;
+    }
+
+	public void testMultilevelRecursion() {
+		//			   01234567890123
+		String text = "Hello!Goodbye!";
+        AnnotationType block = new AnnotationType("block");
+		SpanList root = new SpanList();
+        SpanTree tree = new SpanTree("root", root);
+
+		SpanList block1 = new SpanList();
+		SpanNode hello = new Span(0,6);
+        tree.annotate(hello, block);
+		block1.add(hello);
+
+		SpanList block2 = new SpanList();
+		SpanNode goodbye = new Span(6,8);
+        tree.annotate(goodbye, block);
+		block2.add(goodbye);
+
+		root.add(block1).add(block2);
+
+		Iterator<SpanNode> nodeIterator = root.childIteratorRecursive();
+		assertTrue(nodeIterator.hasNext());
+		assertTrue(nodeIterator.next().equals(hello));
+		assertTrue(nodeIterator.hasNext());
+		assertTrue(nodeIterator.next().equals(block1));
+		assertTrue(nodeIterator.hasNext());
+		assertTrue(nodeIterator.next().equals(goodbye));
+		assertTrue(nodeIterator.hasNext());
+		assertTrue(nodeIterator.next().equals(block2));
+		assertFalse(nodeIterator.hasNext());
+		assertTrue(root.getText(text).toString().equals(text));
+	}
+
+    public void testRecursiveIteratorDeterministicBehavior() {
+        SpanList root = new SpanList();
+        SpanTree tree = new SpanTree("tree", root);
+
+        SpanList a = new SpanList();
+        Span b = new Span(0,1);
+        Span c = new Span(1,1);
+        a.add(b).add(c);
+        root.add(a);
+        Span d = new Span(2,1);
+        root.add(d);
+
+        Span newC = new Span(3,1);
+
+        ListIterator<SpanNode> children = root.childIteratorRecursive();
+        assertTrue(children.hasNext());
+        assertSame(b, children.next());
+        assertTrue(children.hasNext());
+        assertSame(c, children.next());
+        assertTrue(children.hasNext());
+        assertTrue(children.hasNext());
+        assertTrue(children.hasNext());
+
+        children.set(newC);
+        assertTrue(children.hasNext());
+        assertSame(a, children.next());
+        assertTrue(children.hasNext());
+        assertSame(d, children.next());
+        assertFalse(children.hasNext());
+
+
+        assertSame(a, root.children().get(0));
+        assertSame(d, root.children().get(1));
+        assertEquals(2, root.children().size());
+
+        assertSame(b, a.children().get(0));
+        assertSame(newC, a.children().get(1));
+        assertEquals(2, a.children().size());
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/annotation/SpanTestCase.java b/document/src/test/java/com/yahoo/document/annotation/SpanTestCase.java
new file mode 100755
index 00000000000..47df445eff1
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/annotation/SpanTestCase.java
@@ -0,0 +1,100 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.document.serialization.*;
+import com.yahoo.io.GrowableByteBuffer;
+import org.junit.Test;
+
+import java.util.ListIterator;
+import java.util.NoSuchElementException;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.fail;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class SpanTestCase extends AbstractTypesTest {
+
+    @Test
+    public void testIteration() {
+        Span span = new Span(1, 2);
+        ListIterator<SpanNode> b = span.childIterator();
+        assertFalse(b.hasNext());
+        assertFalse(b.hasPrevious());
+        assertEquals(0, b.nextIndex());
+        assertEquals(0, b.previousIndex());
+        try {
+            b.next();
+            fail();
+        } catch (NoSuchElementException nsee) {
+            //ok
+        }
+        try {
+            b.previous();
+            fail();
+        } catch (NoSuchElementException nsee) {
+            //ok
+        }
+        try {
+            b.remove();
+            fail();
+        } catch (UnsupportedOperationException uoe) {
+            //ok
+        }
+        try {
+            b.set(new Span(1, 1));
+            fail();
+        } catch (UnsupportedOperationException uoe) {
+            //ok
+        }
+        try {
+            b.add(new Span(1, 1));
+            fail();
+        } catch (UnsupportedOperationException uoe) {
+            //ok
+        }
+    }
+
+    @Test
+    public void testSerializeDeserialize() {
+        {
+            Span span = new Span(1, 2);
+            serializeAndAssert(span);
+        }
+        {
+            Span span = new Span(4, 15);
+            serializeAndAssert(span);
+        }
+        {
+            Span span = new Span(1, 19);
+            serializeAndAssert(span);
+        }
+    }
+
+    private void serializeAndAssert(Span span) {
+        GrowableByteBuffer buffer;
+        {
+            buffer = new GrowableByteBuffer(1024);
+            DocumentSerializer serializer = DocumentSerializerFactory.create42(buffer);
+            StringFieldValue value = new StringFieldValue("lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj lk");
+            SpanTree tree = new SpanTree("bababa", span);
+            value.setSpanTree(tree);
+            serializer.write(null, value);
+            buffer.flip();
+        }
+        Span span2;
+        {
+            DocumentDeserializer deserializer = DocumentDeserializerFactory.create42(man, buffer);
+            StringFieldValue value = new StringFieldValue();
+            deserializer.read(null, value);
+            span2 = (Span)value.getSpanTree("bababa").getRoot();
+        }
+
+        assertEquals(span, span2);
+        assertNotSame(span, span2);
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/annotation/SpanTreeAdvTest.java b/document/src/test/java/com/yahoo/document/annotation/SpanTreeAdvTest.java
new file mode 100644
index 00000000000..cfb87ee997d
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/annotation/SpanTreeAdvTest.java
@@ -0,0 +1,321 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.datatypes.StringFieldValue;
+import org.junit.After;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.ListIterator;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+
+/**
+ * @author <a href="mailto:mpraveen@yahoo-inc.com">Praveen Mohan</a>
+ *
+ * This test covers all possible scenarios in SpanTree.
+ *
+ */
+
+
+
+public class SpanTreeAdvTest {
+
+    private boolean debug = false;
+
+    private AnnotationType at1 = new AnnotationType("person", DataType.STRING);
+    private AnnotationType at2 = new AnnotationType("street", DataType.STRING);
+    private AnnotationType at3 = new AnnotationType("city", DataType.STRING);
+
+    public SpanTree tree = null;
+
+    private SpanNode span1, span2, span3;
+    private SpanNode span11, span22, span33;
+    private SpanNode span111, span222, span333;
+
+    private AlternateSpanList root, alternate1, alternate2;
+    private SpanList branch, branch1;
+    private List<SpanNode> subtreeList1, subtreeList2, subtreeList3, subtreeList4;
+    private Annotation an111;
+
+
+    public void populateSpanTree() {
+
+        root = new AlternateSpanList();
+        tree = new SpanTree("test", root);
+
+        span1 = new Span(10, 3);
+        span2 = new Span(4, 6);
+        span3 = new Span(13, 10);
+
+        span11 = new Span(0, 2);
+        span22 = new Span(2, 10);
+        span33 = new Span(12, 10);
+
+        span111 = new Span(5, 10);
+        span222 = new Span(15, 5);
+        span333 = new Span(20, 10);
+
+        an111 = new Annotation(at1);
+
+        tree.annotate(span1, at1).annotate(span2, at2).annotate(span3, at3);
+        tree.annotate(span11, at1).annotate(span22, at2).annotate(span33, at3);
+        tree.annotate(span111, an111).annotate(span222, at2).annotate(span333, at3).annotate(span333, at1);
+        tree.annotate(span222, at2);
+
+        root.add(span3);
+        root.add(span2);
+        root.add(span1);
+
+        alternate1 = new AlternateSpanList();
+        alternate1.add(span11);
+        branch = new SpanList();
+        branch.add(span22);
+        subtreeList1 = new ArrayList<SpanNode>();
+        subtreeList1.add(branch);
+        alternate1.addChildren(1, subtreeList1, 50.0d);
+
+        alternate2 = new AlternateSpanList();
+        alternate2.add(span33);
+        subtreeList2 = new ArrayList<SpanNode>();
+        subtreeList2.add(span111);
+        subtreeList2.add(span222);
+        alternate2.addChildren(1, subtreeList2, 70.0d);
+
+        subtreeList3 = new ArrayList<SpanNode>();
+        branch1 = new SpanList();
+        branch1.add(span333);
+        subtreeList3.add(branch1);
+        alternate2.addChildren(2, subtreeList3, 90.0d);
+        branch.add(alternate2);
+
+        subtreeList4 = new ArrayList<SpanNode>();
+        subtreeList4.add(alternate1);
+        root.addChildren(1, subtreeList4, 10.0d);
+    }
+
+    @Test (expected = IllegalStateException.class)
+    public void assertSharingAnnotationInstance() {
+        populateSpanTree();
+        tree.annotate(span333, an111);
+    }
+
+
+
+    @Test
+    public void assertRemoveChildrenCleanupTest() {
+        populateSpanTree();
+        root.removeChildren();
+        tree.cleanup();
+        Iterator<Annotation> iter =tree.iterator();
+        assertFalse(iter.hasNext());
+        assertEquals(0, tree.numAnnotations());
+    }
+
+
+    @Test
+    public void assertClearChildrenCleanupTest() {
+        populateSpanTree();
+        root.clearChildren();
+        tree.cleanup();
+        Iterator<Annotation> iter =tree.iterator();
+        assertFalse(iter.hasNext());
+        assertEquals(0, tree.numAnnotations());
+    }
+
+    @Test
+    public void assertRemoveChildrenIndexCleanupTest() {
+        populateSpanTree();
+        int no = tree.numAnnotations();
+        root.removeChildren(1);
+        tree.cleanup();
+        int postNo = tree.numAnnotations();
+        assertEquals((no-8), postNo);
+    }
+
+    @Test
+    public void assertClearChildrenIndexCleanupTest() {
+        populateSpanTree();
+        int no = tree.numAnnotations();
+        root.clearChildren(1);
+        tree.cleanup();
+        int postNo = tree.numAnnotations();
+        assertEquals(3, postNo);
+    }
+
+
+    @Test
+    public void assertASPLRemoveCleanupTest() {
+        populateSpanTree();
+        int no = tree.numAnnotations();
+        alternate2.removeChildren(2);
+        alternate2.removeChildren(1);
+        alternate2.remove(span33);
+        tree.cleanup();
+        int postNo = tree.numAnnotations();
+        assertEquals((no-6), postNo);
+    }
+
+
+    @Test
+    public void assertSPLRemoveCleanupTest() {
+        populateSpanTree();
+        int no = tree.numAnnotations();
+        branch.remove(alternate2);
+        tree.cleanup();
+        int postNo = tree.numAnnotations();
+        assertEquals((no-6), postNo);
+    }
+
+    @Test
+    public void assertIteratorRecursiveASPLTest() {
+        populateSpanTree();
+        int no = 0;
+        Iterator<Annotation> it = tree.iteratorRecursive(alternate2);
+        while (it.hasNext()) {
+            it.next();
+            no ++;
+        }
+        assertEquals(6, no);
+    }
+
+    @Test
+    public void assertClearAnnotationsRecursiveASPLTest() {
+        populateSpanTree();
+        int no = tree.numAnnotations();
+        tree.clearAnnotationsRecursive(alternate2);
+        int postNo = tree.numAnnotations();
+        assertEquals((no-6), postNo);
+    }
+
+
+
+    @Test (expected = IllegalStateException.class)
+    public void assertReuseRemovedNode() {
+        populateSpanTree();
+        root.remove(span3);
+        tree.annotate(span3, new Annotation(at1));
+    }
+
+    @Test (expected = IllegalStateException.class)
+    public void assertNeedForCleanup() {
+        populateSpanTree();
+        root.remove(span3);
+        Iterator<Annotation> iter =tree.iterator();
+        while (iter.hasNext()) {
+            SpanNode sn = iter.next().getSpanNode();
+        }
+    }
+
+    @Test
+    public void validateSpanTree() {
+        populateSpanTree();
+        int no = tree.numAnnotations();
+        root.remove(span3);
+        tree.cleanup();
+        int noAfter = tree.numAnnotations();
+        assertEquals((no-1), noAfter);
+    }
+
+    @After
+    public void tearDown() {
+        tree = null;
+    }
+
+
+    public void consumeAnnotations(SpanList root) {
+        if (root instanceof AlternateSpanList) {
+            parseAlternateLists((AlternateSpanList)root);
+            if (debug) System.out.println("\nGetting annotations for the SpanList itself : [" + root.getFrom() + ", " + root.getTo() + "] ");
+            getAnnotationsForNode(root);
+            return;
+        }
+        if (debug) System.out.println("\n\nSpanList: [" + root.getFrom() + ", " + root.getTo() + "] num Children: " + root.numChildren());
+        if (debug) System.out.println("-------------------");
+        Iterator<SpanNode> childIterator = root.childIterator();
+		while (childIterator.hasNext()) {
+			SpanNode node = childIterator.next();
+            //System.out.println("Span Node: " + node); // + " Span Text: " + node.getText(fieldValStr));
+            if (debug) System.out.println("\n\nSpan Node: [" + node.getFrom() + ", " + node.getTo() + "] ");
+            if (node instanceof AlternateSpanList) {
+                parseAlternateLists((AlternateSpanList)node);
+                if (debug) System.out.println("---- Alternate SpanList complete ---");
+            } else if (node instanceof SpanList) {
+                if (debug) System.out.println("Encountered another span list");
+                SpanList spl = (SpanList) node;
+                ListIterator<SpanNode> lli = spl.childIterator();
+                while (lli.hasNext()) System.out.print(" " + lli.next() + " ");
+                consumeAnnotations((SpanList) node);
+            } else {
+       			if (debug) System.out.println("\nGetting annotations for this span node: [" + node.getFrom() + ", " + node.getTo() + "] ");
+   	    	    getAnnotationsForNode(node);
+            }
+		}
+        if (debug) System.out.println("\nGetting annotations for the SpanList itself : [" + root.getFrom() + ", " + root.getTo() + "] ");
+        getAnnotationsForNode(root);
+    }
+
+    public void parseAlternateLists(AlternateSpanList aspl) {
+        int no = aspl.getNumSubTrees();
+        if (debug) System.out.println("Parsing Alternate span list. No of subtrees: " + no);
+        int ctr = 0;
+        while (ctr < no) {
+            if (debug) System.out.println("\nSubTree: " + ctr + " probability: " + aspl.getProbability(ctr));
+            ListIterator<SpanNode> lIter = aspl.childIterator(ctr);
+            while (lIter.hasNext()) {
+                SpanNode spnNode = lIter.next();
+                if (debug) System.out.println("Parsing span node: [" + spnNode.getFrom() + ", " + spnNode.getTo() + "] ");
+                if (spnNode instanceof AlternateSpanList) {
+                    if (debug) System.out.println("A child alternate span list found. Recursing");
+                    parseAlternateLists((AlternateSpanList)spnNode);
+                } else if (spnNode instanceof SpanList) {
+                    if (debug) System.out.println("A child span list found. Recursing");
+                    consumeAnnotations((SpanList)spnNode);
+                } else {
+                    //System.out.println("Span Node (from alternate spanlist): " + spnNode);
+                    getAnnotationsForNode(spnNode);
+                }
+            }
+            ctr ++;
+        }
+    }
+
+
+    public void parseFieldForAnnotations(StringFieldValue sfv) {
+        Collection<SpanTree> c = sfv.getSpanTrees();
+        Iterator<SpanTree> iiter = c.iterator();
+        while (iiter.hasNext()) {
+            if (debug) System.out.println(sfv + " has annotations");
+            tree = iiter.next();
+            SpanList root = (SpanList) tree.getRoot();
+            consumeAnnotations(root);
+        }
+    }
+
+
+    public void getAnnotationsForNode(SpanNode node) {
+        Iterator<Annotation> iter = tree.iterator(node);
+        boolean annotationPresent = false;
+		while (iter.hasNext()) {
+            annotationPresent = true;
+			Annotation xx = iter.next();
+			AnnotationType t = xx.getType();
+			StringFieldValue fValue = (StringFieldValue) xx.getFieldValue();
+			if (debug) System.out.println("Annotation: " + xx);
+            if (fValue == null) {
+                if (debug) System.out.println("Field Value is null");
+                return;
+            } else {
+                if (debug) System.out.println("Field Value: " + fValue.getString());
+            }
+		}
+        if (!annotationPresent) {
+            if (debug) System.out.println("****No annotations found for the span node: [" + node.getFrom() + ", " + node.getTo() + "] ");
+        }
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/annotation/SpanTreeTestCase.java b/document/src/test/java/com/yahoo/document/annotation/SpanTreeTestCase.java
new file mode 100755
index 00000000000..e61d8d10d8a
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/annotation/SpanTreeTestCase.java
@@ -0,0 +1,880 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.DocumentTypeManager;
+import com.yahoo.document.Field;
+import com.yahoo.document.StructDataType;
+import com.yahoo.document.datatypes.IntegerFieldValue;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.document.datatypes.Struct;
+import com.yahoo.document.serialization.*;
+import com.yahoo.io.GrowableByteBuffer;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.*;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class SpanTreeTestCase extends AbstractTypesTest {
+    SpanTree tree;
+    SpanList root;
+    SpanNode a;
+    SpanNode b;
+    SpanList c;
+    SpanNode d;
+    SpanNode e;
+    Annotation dummyA;
+    Annotation numberB;
+    Annotation bananaC;
+    Annotation appleC;
+    Annotation grapeC;
+    Annotation dummyD;
+    Annotation dummyE;
+
+    /**
+     * Sets up a simple tree:
+     * root:
+     * a (dummyA), b (numberB), c (bananaC, appleC, grapeC)
+     * c:
+     * d (dummyD), e (dummyE)
+     *
+     */
+    @Before
+    public void setUp() {
+        tree = new SpanTree("ballooo");
+        root = (SpanList) tree.getRoot();
+
+        a = new Span(1,1);
+        b = new Span(2,1);
+        c = new SpanList();
+        d = new Span(3,1);
+        e = new Span(4,1);
+
+        c.add(d).add(e);
+        root.add(a).add(b).add(c);
+
+        dummyA = new Annotation(dummy);
+        numberB = new Annotation(number);
+        bananaC = new Annotation(banana);
+        appleC = new Annotation(apple);
+        grapeC = new Annotation(grape);
+        dummyD = new Annotation(dummy);
+        dummyE = new Annotation(dummy);
+
+        tree.annotate(a, dummyA)
+                .annotate(b, numberB)
+                .annotate(c, bananaC)
+                .annotate(c, appleC)
+                .annotate(c, grapeC)
+                .annotate(d, dummyD)
+                .annotate(e, dummyE);
+    }
+
+    /**
+     * Tests iterating through all annotations in a tree.
+     */
+    @Test
+    public void testAnnotationIteration() {
+        List<Annotation> allAnnotationsList = new ArrayList<>();
+        for (Annotation a : tree) {
+            allAnnotationsList.add(a);
+        }
+        Collections.sort(allAnnotationsList);
+
+        Iterator<Annotation> allAnnotations = allAnnotationsList.iterator();
+
+        assertTrue(allAnnotations.hasNext());
+        assertTrue(allAnnotations.hasNext());
+        assertTrue(allAnnotations.hasNext());
+        assertTrue(allAnnotations.hasNext());
+
+        assertSame(dummyA, allAnnotations.next());
+        assertSame(a, dummyA.getSpanNode());
+
+        assertTrue(allAnnotations.hasNext());
+        assertTrue(allAnnotations.hasNext());
+        assertTrue(allAnnotations.hasNext());
+        assertTrue(allAnnotations.hasNext());
+
+        assertSame(numberB, allAnnotations.next());
+        assertSame(b, numberB.getSpanNode());
+
+        assertTrue(allAnnotations.hasNext());
+        assertTrue(allAnnotations.hasNext());
+        assertTrue(allAnnotations.hasNext());
+        assertTrue(allAnnotations.hasNext());
+
+        assertSame(dummyD, allAnnotations.next());
+        assertSame(d, dummyD.getSpanNode());
+
+        assertTrue(allAnnotations.hasNext());
+        assertTrue(allAnnotations.hasNext());
+        assertTrue(allAnnotations.hasNext());
+        assertTrue(allAnnotations.hasNext());
+
+        assertSame(grapeC, allAnnotations.next());
+        assertSame(c, grapeC.getSpanNode());
+
+        assertTrue(allAnnotations.hasNext());
+        assertTrue(allAnnotations.hasNext());
+        assertTrue(allAnnotations.hasNext());
+        assertTrue(allAnnotations.hasNext());
+
+        assertSame(bananaC, allAnnotations.next());
+        assertSame(c, bananaC.getSpanNode());
+
+        assertTrue(allAnnotations.hasNext());
+        assertTrue(allAnnotations.hasNext());
+        assertTrue(allAnnotations.hasNext());
+        assertTrue(allAnnotations.hasNext());
+
+        assertSame(appleC, allAnnotations.next());
+        assertSame(c, appleC.getSpanNode());
+
+        assertTrue(allAnnotations.hasNext());
+        assertTrue(allAnnotations.hasNext());
+        assertTrue(allAnnotations.hasNext());
+        assertTrue(allAnnotations.hasNext());
+
+
+        assertSame(dummyE, allAnnotations.next());
+        assertSame(e, dummyE.getSpanNode());
+
+        assertFalse(allAnnotations.hasNext());
+        assertFalse(allAnnotations.hasNext());
+        assertFalse(allAnnotations.hasNext());
+        assertFalse(allAnnotations.hasNext());
+    }
+
+    /**
+     * Tests iterating through A's annotations in a tree.
+     */
+    @Test
+    public void testAnnotationIterationForA() {
+        //A has no child nodes, should have equal results for recursive and non-recursive iterator
+        aIteration(tree.iterator(a));
+        aIteration(tree.iteratorRecursive(a));
+    }
+
+    private void aIteration(Iterator<Annotation> aAnnotations) {
+        assertTrue(aAnnotations.hasNext());
+        assertTrue(aAnnotations.hasNext());
+        assertTrue(aAnnotations.hasNext());
+        assertTrue(aAnnotations.hasNext());
+
+        assertSame(dummyA, aAnnotations.next());
+        assertSame(a, dummyA.getSpanNode());
+
+        assertFalse(aAnnotations.hasNext());
+        assertFalse(aAnnotations.hasNext());
+        assertFalse(aAnnotations.hasNext());
+        assertFalse(aAnnotations.hasNext());
+    }
+
+    /**
+     * Tests iterating through B's annotations in a tree.
+     */
+    @Test
+    public void testAnnotationIterationForB() {
+        //B has no child nodes, should have equal results for recursive and non-recursive iterator
+        bIteration(tree.iterator(b));
+        bIteration(tree.iteratorRecursive(b));
+    }
+
+    private void bIteration(Iterator<Annotation> bAnnotations) {
+        assertTrue(bAnnotations.hasNext());
+        assertTrue(bAnnotations.hasNext());
+        assertTrue(bAnnotations.hasNext());
+        assertTrue(bAnnotations.hasNext());
+
+        assertSame(numberB, bAnnotations.next());
+        assertSame(b, numberB.getSpanNode());
+
+        assertFalse(bAnnotations.hasNext());
+        assertFalse(bAnnotations.hasNext());
+        assertFalse(bAnnotations.hasNext());
+        assertFalse(bAnnotations.hasNext());
+    }
+
+    /**
+     * Tests iterating through C's annotations in a tree.
+     */
+    @Test
+    public void testAnnotationIterationForC() {
+        List<Annotation> cAnnotationsList = new ArrayList<>();
+        for (Iterator<Annotation> iteratorc = tree.iterator(c); iteratorc.hasNext(); ) {
+            cAnnotationsList.add(iteratorc.next());
+        }
+        Collections.sort(cAnnotationsList);
+
+        Iterator<Annotation> cAnnotations = cAnnotationsList.iterator();
+
+        assertTrue(cAnnotations.hasNext());
+        assertTrue(cAnnotations.hasNext());
+        assertTrue(cAnnotations.hasNext());
+        assertTrue(cAnnotations.hasNext());
+
+        assertSame(grapeC, cAnnotations.next());
+        assertSame(c, grapeC.getSpanNode());
+
+        assertTrue(cAnnotations.hasNext());
+        assertTrue(cAnnotations.hasNext());
+        assertTrue(cAnnotations.hasNext());
+        assertTrue(cAnnotations.hasNext());
+
+        assertSame(bananaC, cAnnotations.next());
+        assertSame(c, bananaC.getSpanNode());
+
+        assertTrue(cAnnotations.hasNext());
+        assertTrue(cAnnotations.hasNext());
+        assertTrue(cAnnotations.hasNext());
+        assertTrue(cAnnotations.hasNext());
+
+        assertSame(appleC, cAnnotations.next());
+        assertSame(c, appleC.getSpanNode());
+
+        assertFalse(cAnnotations.hasNext());
+        assertFalse(cAnnotations.hasNext());
+        assertFalse(cAnnotations.hasNext());
+        assertFalse(cAnnotations.hasNext());
+    }
+
+    @Test
+    public void testRecursiveAnnotationIterationForC() {
+        List<Annotation> cAnnotationsList = new ArrayList<>();
+        for (Iterator<Annotation> iteratorc = tree.iteratorRecursive(c); iteratorc.hasNext(); ) {
+            cAnnotationsList.add(iteratorc.next());
+        }
+        Collections.sort(cAnnotationsList);
+
+        Iterator<Annotation> cAnnotations = cAnnotationsList.iterator();
+
+        assertTrue(cAnnotations.hasNext());
+        assertTrue(cAnnotations.hasNext());
+        assertTrue(cAnnotations.hasNext());
+        assertTrue(cAnnotations.hasNext());
+
+        assertSame(dummyD, cAnnotations.next());
+        assertSame(d, dummyD.getSpanNode());
+
+        assertTrue(cAnnotations.hasNext());
+        assertTrue(cAnnotations.hasNext());
+        assertTrue(cAnnotations.hasNext());
+        assertTrue(cAnnotations.hasNext());
+
+        assertSame(grapeC, cAnnotations.next());
+        assertSame(c, grapeC.getSpanNode());
+
+        assertTrue(cAnnotations.hasNext());
+        assertTrue(cAnnotations.hasNext());
+        assertTrue(cAnnotations.hasNext());
+        assertTrue(cAnnotations.hasNext());
+
+        assertSame(bananaC, cAnnotations.next());
+        assertSame(c, bananaC.getSpanNode());
+
+        assertTrue(cAnnotations.hasNext());
+        assertTrue(cAnnotations.hasNext());
+        assertTrue(cAnnotations.hasNext());
+        assertTrue(cAnnotations.hasNext());
+
+        assertSame(appleC, cAnnotations.next());
+        assertSame(c, appleC.getSpanNode());
+
+        assertTrue(cAnnotations.hasNext());
+        assertTrue(cAnnotations.hasNext());
+        assertTrue(cAnnotations.hasNext());
+        assertTrue(cAnnotations.hasNext());
+
+        assertSame(dummyE, cAnnotations.next());
+        assertSame(e, dummyE.getSpanNode());
+
+        assertFalse(cAnnotations.hasNext());
+        assertFalse(cAnnotations.hasNext());
+        assertFalse(cAnnotations.hasNext());
+        assertFalse(cAnnotations.hasNext());
+    }
+
+
+    /**
+     * Tests iterating through D's annotations in a tree.
+     */
+    @Test
+    public void testAnnotationIterationForD() {
+        //D has no child nodes, should have equal results for recursive and non-recursive iterator
+        dIteration(tree.iterator(d));
+        dIteration(tree.iteratorRecursive(d));
+    }
+
+    private void dIteration(Iterator<Annotation> dAnnotations) {
+        assertTrue(dAnnotations.hasNext());
+        assertTrue(dAnnotations.hasNext());
+        assertTrue(dAnnotations.hasNext());
+        assertTrue(dAnnotations.hasNext());
+
+        assertSame(dummyD, dAnnotations.next());
+        assertSame(d, dummyD.getSpanNode());
+
+        assertFalse(dAnnotations.hasNext());
+        assertFalse(dAnnotations.hasNext());
+        assertFalse(dAnnotations.hasNext());
+        assertFalse(dAnnotations.hasNext());
+    }
+
+
+    /**
+     * Tests iterating through E's annotations in a tree.
+     */
+    @Test
+    public void testAnnotationIterationForE() {
+        //E has no child nodes, should have equal results for recursive and non-recursive iterator
+        eIteration(tree.iterator(e));
+        eIteration(tree.iteratorRecursive(e));
+    }
+
+    private void eIteration(Iterator<Annotation> eAnnotations) {
+        assertTrue(eAnnotations.hasNext());
+        assertTrue(eAnnotations.hasNext());
+        assertTrue(eAnnotations.hasNext());
+        assertTrue(eAnnotations.hasNext());
+
+        assertSame(dummyE, eAnnotations.next());
+        assertSame(e, dummyE.getSpanNode());
+
+        assertFalse(eAnnotations.hasNext());
+        assertFalse(eAnnotations.hasNext());
+        assertFalse(eAnnotations.hasNext());
+        assertFalse(eAnnotations.hasNext());
+    }
+
+    @Test
+    public void testCleanup() {
+        SpanList a = new SpanList();
+        SpanTree tree = new SpanTree("ballooO", a);
+        Span b = new Span(0,1);
+        Span c = new Span(1,1);
+        a.add(b).add(c);
+
+        AnnotationReferenceDataType refTypeToDummy = new AnnotationReferenceDataType(dummy);
+        AnnotationType annotationTypeWithRefToDummy = new AnnotationType("reftodummy", refTypeToDummy);
+
+        AnnotationReferenceDataType refTypeToRefTypeToDummy = new AnnotationReferenceDataType(annotationTypeWithRefToDummy);
+        AnnotationType annotationTypeWithRefToRefToDummy = new AnnotationType("reftoreftodummy", refTypeToRefTypeToDummy);
+
+        StructDataType structType = new StructDataType("str");
+        Field refToDummyField = new Field("refToDummy", refTypeToDummy);
+        structType.addField(refToDummyField);
+        AnnotationType annotationTypeWithStruct = new AnnotationType("structwithref", structType);
+
+
+        Annotation dummyAnnotationForB = new Annotation(dummy);
+        tree.annotate(b, dummyAnnotationForB);
+
+        AnnotationReference referenceToDummyB = new AnnotationReference(refTypeToDummy, dummyAnnotationForB);
+        Annotation annotationWithRefToDummyB = new Annotation(annotationTypeWithRefToDummy, referenceToDummyB);
+        tree.annotate(annotationWithRefToDummyB);
+
+        AnnotationReference referenceToRefToDummyB = new AnnotationReference(refTypeToRefTypeToDummy, annotationWithRefToDummyB);
+        Annotation annotationWithRefToRefToDummyB = new Annotation(annotationTypeWithRefToRefToDummy, referenceToRefToDummyB);
+        tree.annotate(annotationWithRefToRefToDummyB);
+
+        Struct struct = new Struct(structType);
+        struct.setFieldValue(refToDummyField, new AnnotationReference(refTypeToDummy, dummyAnnotationForB));
+        Annotation annotationWithStruct = new Annotation(annotationTypeWithStruct, struct);
+        tree.annotate(annotationWithStruct);
+
+
+        Iterator<Annotation> annotationIt;
+
+        assertEquals(4, tree.numAnnotations());
+        assertTrue(a.isValid());
+        assertTrue(b.isValid());
+        assertTrue(c.isValid());
+        assertEquals(2, a.numChildren());
+
+        {
+            List<Annotation> allAnnotationsList = new ArrayList<>();
+            for (Annotation an : tree) {
+                allAnnotationsList.add(an);
+            }
+            Collections.sort(allAnnotationsList);
+
+            annotationIt = allAnnotationsList.iterator();
+        }
+        // NOTE: ordering of annotations is derived from their name
+
+        assertSame(annotationWithRefToDummyB, annotationIt.next());
+        assertFalse(annotationWithRefToDummyB.hasSpanNode());
+        assertFalse(annotationWithRefToDummyB.isSpanNodeValid());
+        assertTrue(annotationWithRefToDummyB.hasFieldValue());
+        assertSame(dummyAnnotationForB, ((AnnotationReference) annotationWithRefToDummyB.getFieldValue()).getReference());
+
+        assertSame(annotationWithStruct, annotationIt.next());
+        assertFalse(annotationWithStruct.hasSpanNode());
+        assertFalse(annotationWithStruct.isSpanNodeValid());
+        assertTrue(annotationWithStruct.hasFieldValue());
+        assertSame(struct, annotationWithStruct.getFieldValue());
+        assertSame(dummyAnnotationForB, ((AnnotationReference) struct.getFieldValue(refToDummyField)).getReference());
+
+        assertSame(annotationWithRefToRefToDummyB, annotationIt.next());
+        assertFalse(annotationWithRefToRefToDummyB.hasSpanNode());
+        assertFalse(annotationWithRefToRefToDummyB.isSpanNodeValid());
+        assertTrue(annotationWithRefToRefToDummyB.hasFieldValue());
+        assertSame(annotationWithRefToDummyB, ((AnnotationReference) annotationWithRefToRefToDummyB.getFieldValue()).getReference());
+
+        assertSame(dummyAnnotationForB, annotationIt.next());
+        assertTrue(dummyAnnotationForB.hasSpanNode());
+        assertTrue(dummyAnnotationForB.isSpanNodeValid());
+        assertFalse(dummyAnnotationForB.hasFieldValue());
+
+        assertFalse(annotationIt.hasNext());
+
+        //removing b!!
+        a.remove(b);
+
+
+        assertEquals(4, tree.numAnnotations());
+        assertTrue(a.isValid());
+        assertFalse(b.isValid());
+        assertTrue(c.isValid());
+        assertEquals(1, a.numChildren());
+
+        {
+            List<Annotation> allAnnotationsList = new ArrayList<>();
+            for (Annotation an : tree) {
+                allAnnotationsList.add(an);
+            }
+            Collections.sort(allAnnotationsList);
+
+            annotationIt = allAnnotationsList.iterator();
+        }
+
+        assertSame(annotationWithRefToDummyB, annotationIt.next());
+        assertFalse(annotationWithRefToDummyB.hasSpanNode());
+        assertFalse(annotationWithRefToDummyB.isSpanNodeValid());
+        assertTrue(annotationWithRefToDummyB.hasFieldValue());
+        assertSame(dummyAnnotationForB, ((AnnotationReference) annotationWithRefToDummyB.getFieldValue()).getReference());
+
+        assertSame(annotationWithStruct, annotationIt.next());
+        assertFalse(annotationWithStruct.hasSpanNode());
+        assertFalse(annotationWithStruct.isSpanNodeValid());
+        assertTrue(annotationWithStruct.hasFieldValue());
+        assertSame(struct, annotationWithStruct.getFieldValue());
+        assertSame(dummyAnnotationForB, ((AnnotationReference) struct.getFieldValue(refToDummyField)).getReference());
+
+        assertSame(annotationWithRefToRefToDummyB, annotationIt.next());
+        assertFalse(annotationWithRefToRefToDummyB.hasSpanNode());
+        assertFalse(annotationWithRefToRefToDummyB.isSpanNodeValid());
+        assertTrue(annotationWithRefToRefToDummyB.hasFieldValue());
+        assertSame(annotationWithRefToDummyB, ((AnnotationReference) annotationWithRefToRefToDummyB.getFieldValue()).getReference());
+
+        assertSame(dummyAnnotationForB, annotationIt.next());
+        assertTrue(dummyAnnotationForB.hasSpanNode());
+        assertFalse(dummyAnnotationForB.isSpanNodeValid());
+        assertFalse(dummyAnnotationForB.hasFieldValue());
+
+        assertFalse(annotationIt.hasNext());
+
+
+        //cleaning up tree!!
+        tree.cleanup();
+
+        assertEquals(1, tree.numAnnotations());
+        assertTrue(a.isValid());
+        assertFalse(b.isValid());
+        assertTrue(c.isValid());
+        assertEquals(1, a.numChildren());
+
+        assertFalse(dummyAnnotationForB.hasSpanNode());
+        assertFalse(dummyAnnotationForB.isSpanNodeValid());
+        assertFalse(dummyAnnotationForB.hasFieldValue());
+
+        assertFalse(annotationWithRefToDummyB.hasSpanNode());
+        assertFalse(annotationWithRefToDummyB.isSpanNodeValid());
+        assertFalse(annotationWithRefToDummyB.hasFieldValue());
+
+        assertFalse(annotationWithRefToRefToDummyB.hasSpanNode());
+        assertFalse(annotationWithRefToRefToDummyB.isSpanNodeValid());
+        assertFalse(annotationWithRefToRefToDummyB.hasFieldValue());
+
+        annotationIt = tree.iterator();
+        assertSame(annotationWithStruct, annotationIt.next());
+        assertFalse(annotationWithStruct.hasSpanNode());
+        assertFalse(annotationWithStruct.isSpanNodeValid());
+        assertTrue(annotationWithStruct.hasFieldValue());
+        assertSame(struct, annotationWithStruct.getFieldValue());
+        assertNull(struct.getFieldValue(refToDummyField));
+
+        assertFalse(annotationIt.hasNext());
+    }
+
+    @Test
+    public void testSimpleCopy() {
+        StringFieldValue string = new StringFieldValue("yahoooo");
+
+        SpanList list = new SpanList();
+        Span span1 = new Span(0,1);
+        Span span2 = new Span(1,1);
+        list.add(span1).add(span2);
+        SpanTree tree = new SpanTree("foo", list);
+        string.setSpanTree(tree);
+
+        Annotation a1 = new Annotation(grape);
+        tree.annotate(span1, a1);
+        Annotation a2 = new Annotation(apple);
+        tree.annotate(span2, a2);
+
+        StringFieldValue stringCopy = string.clone();
+
+
+
+        assertEquals(string, stringCopy);
+        assertNotSame(string, stringCopy);
+
+        SpanTree treeCopy = stringCopy.getSpanTree("foo");
+        assertEquals(tree, treeCopy);
+        assertNotSame(tree, treeCopy);
+
+        SpanList listCopy = (SpanList) treeCopy.getRoot();
+        assertEquals(list, listCopy);
+        assertNotSame(list, listCopy);
+
+        Span span1Copy = (Span) listCopy.children().get(0);
+        assertEquals(span1, span1Copy);
+        assertNotSame(span1, span1Copy);
+
+        Span span2Copy = (Span) listCopy.children().get(1);
+        assertEquals(span2, span2Copy);
+        assertNotSame(span2, span2Copy);
+
+        Iterator<Annotation> annotationIt;
+        {
+            List<Annotation> allAnnotationsList = new ArrayList<>();
+            for (Annotation an : treeCopy) {
+                allAnnotationsList.add(an);
+            }
+            Collections.sort(allAnnotationsList);
+
+            annotationIt = allAnnotationsList.iterator();
+        }
+
+        Annotation a1Copy = annotationIt.next();
+        assertEquals(a1, a1Copy);
+        assertNotSame(a1, a1Copy);
+        assertSame(span1Copy, a1Copy.getSpanNode());
+        assertNotSame(span1, a1Copy.getSpanNode());
+
+        Annotation a2Copy = annotationIt.next();
+        assertEquals(a2, a2Copy);
+        assertNotSame(a2, a2Copy);
+        assertSame(span2Copy, a2Copy.getSpanNode());
+        assertNotSame(span2, a2Copy.getSpanNode());
+    }
+
+    @Test
+    public void testSimpleCopy2() {
+        StringFieldValue string = new StringFieldValue("yahoooo");
+
+        SpanList list = new SpanList();
+        Span span1 = new Span(0,1);
+        Span span2 = new Span(1,1);
+        list.add(span1).add(span2);
+        SpanTree tree = new SpanTree("foo", list);
+        string.setSpanTree(tree);
+
+        Annotation a1 = new Annotation(grape);
+        tree.annotate(span1, a1);
+        Annotation a2 = new Annotation(apple);
+        tree.annotate(span2, a2);
+
+        Struct donald = new Struct(person);
+        donald.setFieldValue("firstname", "donald");
+        donald.setFieldValue("lastname", "duck");
+        donald.setFieldValue("birthyear", 1929);
+        Annotation donaldAnn = new Annotation(personA, donald);
+        tree.annotate(list, donaldAnn);
+
+
+        StringFieldValue stringCopy = string.clone();
+
+
+
+        assertEquals(string, stringCopy);
+        assertNotSame(string, stringCopy);
+
+        SpanTree treeCopy = stringCopy.getSpanTree("foo");
+        assertEquals(tree, treeCopy);
+        assertNotSame(tree, treeCopy);
+
+        SpanList listCopy = (SpanList) treeCopy.getRoot();
+        assertEquals(list, listCopy);
+        assertNotSame(list, listCopy);
+
+        Span span1Copy = (Span) listCopy.children().get(0);
+        assertEquals(span1, span1Copy);
+        assertNotSame(span1, span1Copy);
+
+        Span span2Copy = (Span) listCopy.children().get(1);
+        assertEquals(span2, span2Copy);
+        assertNotSame(span2, span2Copy);
+
+        Iterator<Annotation> annotationIt;
+        {
+            List<Annotation> allAnnotationsList = new ArrayList<>();
+            for (Annotation an : treeCopy) {
+                allAnnotationsList.add(an);
+            }
+            Collections.sort(allAnnotationsList);
+
+            annotationIt = allAnnotationsList.iterator();
+        }
+
+        Annotation a1Copy = annotationIt.next();
+        assertEquals(a1, a1Copy);
+        assertNotSame(a1, a1Copy);
+        assertSame(span1Copy, a1Copy.getSpanNode());
+        assertNotSame(span1, a1Copy.getSpanNode());
+
+        Annotation donaldAnnCopy = annotationIt.next();
+        assertEquals(donaldAnn, donaldAnnCopy);
+        assertNotSame(donaldAnn, donaldAnnCopy);
+        assertSame(listCopy, donaldAnnCopy.getSpanNode());
+        assertNotSame(list, donaldAnnCopy.getSpanNode());
+
+        Struct donaldCopy = (Struct) donaldAnnCopy.getFieldValue();
+        assertEquals(donald, donaldCopy);
+        assertNotSame(donald, donaldCopy);
+
+        Annotation a2Copy = annotationIt.next();
+        assertEquals(a2, a2Copy);
+        assertNotSame(a2, a2Copy);
+        assertSame(span2Copy, a2Copy.getSpanNode());
+        assertNotSame(span2, a2Copy.getSpanNode());
+    }
+
+    @Test
+    public void testCopyAnnotatedString() {
+        StringFieldValue str = getAnnotatedString();
+        StringFieldValue strCopy = str.clone();
+
+        SpanTree balloooTree = str.getSpanTree("ballooo");
+        AlternateSpanList root = (AlternateSpanList) balloooTree.getRoot();
+        Span s1 = (Span) root.children(0).get(0);
+        Span s2 = (Span) root.children(0).get(1);
+        Span s3 = (Span) root.children(0).get(2);
+        AlternateSpanList s4 = (AlternateSpanList) root.children(0).get(3);
+        Span s5 = (Span) s4.children(0).get(0);
+        Span s6 = (Span) s4.children(0).get(1);
+        Span s7 = (Span) root.children(1).get(0);
+        Span s8 = (Span) root.children(1).get(1);
+        Span s9 = (Span) root.children(1).get(2);
+
+        SpanTree balloooTreeCopy = strCopy.getSpanTree("ballooo");
+        assertEquals(balloooTree, balloooTreeCopy);
+        assertNotSame(balloooTree, balloooTreeCopy);
+        AlternateSpanList rootCopy = (AlternateSpanList) balloooTreeCopy.getRoot();
+        assertEquals(root, rootCopy);
+        assertNotSame(root, rootCopy);
+        Span s1Copy = (Span) rootCopy.children(0).get(0);
+        assertEquals(s1, s1Copy);
+        assertNotSame(s1, s1Copy);
+        Span s2Copy = (Span) rootCopy.children(0).get(1);
+        assertEquals(s2, s2Copy);
+        assertNotSame(s2, s2Copy);
+        Span s3Copy = (Span) rootCopy.children(0).get(2);
+        assertEquals(s3, s3Copy);
+        assertNotSame(s3, s3Copy);
+        AlternateSpanList s4Copy = (AlternateSpanList) rootCopy.children(0).get(3);
+        assertEquals(s4, s4Copy);
+        assertNotSame(s4, s4Copy);
+        Span s5Copy = (Span) s4Copy.children(0).get(0);
+        assertEquals(s5, s5Copy);
+        assertNotSame(s5, s5Copy);
+        Span s6Copy = (Span) s4Copy.children(0).get(1);
+        assertEquals(s6, s6Copy);
+        assertNotSame(s6, s6Copy);
+        Span s7Copy = (Span) rootCopy.children(1).get(0);
+        assertEquals(s7, s7Copy);
+        assertNotSame(s7, s7Copy);
+        Span s8Copy = (Span) rootCopy.children(1).get(1);
+        assertEquals(s8, s8Copy);
+        assertNotSame(s8, s8Copy);
+        Span s9Copy = (Span) rootCopy.children(1).get(2);
+        assertEquals(s9, s9Copy);
+        assertNotSame(s9, s9Copy);
+
+
+        Iterator<Annotation> annotationsBalloooTree;
+        {
+            List<Annotation> allAnnotationsList = new ArrayList<>();
+            for (Annotation an : balloooTree) {
+                allAnnotationsList.add(an);
+            }
+            Collections.sort(allAnnotationsList);
+
+            annotationsBalloooTree = allAnnotationsList.iterator();
+        }
+
+        Annotation dummyAnnForS1 = annotationsBalloooTree.next();
+
+        Annotation dummyAnnForS2 = annotationsBalloooTree.next();
+
+        Annotation numberAnnForS2 = annotationsBalloooTree.next();
+        IntegerFieldValue integerValForS2 = (IntegerFieldValue) numberAnnForS2.getFieldValue();
+
+        Annotation motherAnnForS2 = annotationsBalloooTree.next();
+        Struct motherValForS2 = (Struct) motherAnnForS2.getFieldValue();
+
+        Annotation dummyAnnForS3 = annotationsBalloooTree.next();
+
+        Annotation numberAnnForS3 = annotationsBalloooTree.next();
+        IntegerFieldValue integerValForS3 = (IntegerFieldValue) numberAnnForS3.getFieldValue();
+
+        Annotation dummyAnnForS5 = annotationsBalloooTree.next();
+
+        Annotation daughterAnnForS6 = annotationsBalloooTree.next();
+        Struct daughterValForS6 = (Struct) daughterAnnForS6.getFieldValue();
+        AnnotationReference refFromS6ToMotherAnn = (AnnotationReference) daughterValForS6.getFieldValue("related");
+
+
+        Iterator<Annotation> annotationsBalloooTreeCopy;
+        {
+            List<Annotation> allAnnotationsList = new ArrayList<>();
+            for (Annotation an : balloooTreeCopy) {
+                allAnnotationsList.add(an);
+            }
+            Collections.sort(allAnnotationsList);
+
+            annotationsBalloooTreeCopy = allAnnotationsList.iterator();
+        }
+
+        Annotation dummyAnnForS1Copy = annotationsBalloooTreeCopy.next();
+        assertEquals(dummyAnnForS1, dummyAnnForS1Copy);
+        assertNotSame(dummyAnnForS1, dummyAnnForS1Copy);
+
+        Annotation dummyAnnForS2Copy = annotationsBalloooTreeCopy.next();
+        assertEquals(dummyAnnForS2, dummyAnnForS2Copy);
+        assertNotSame(dummyAnnForS2, dummyAnnForS2Copy);
+
+        Annotation numberAnnForS2Copy = annotationsBalloooTreeCopy.next();
+        assertEquals(numberAnnForS2, numberAnnForS2Copy);
+        assertNotSame(numberAnnForS2, numberAnnForS2Copy);
+        IntegerFieldValue integerValForS2Copy = (IntegerFieldValue) numberAnnForS2Copy.getFieldValue();
+        assertEquals(integerValForS2, integerValForS2Copy);
+        assertNotSame(integerValForS2, integerValForS2Copy);
+
+        Annotation motherAnnForS2Copy = annotationsBalloooTreeCopy.next();
+        assertEquals(motherAnnForS2, motherAnnForS2Copy);
+        assertNotSame(motherAnnForS2, motherAnnForS2Copy);
+        Struct motherValForS2Copy = (Struct) motherAnnForS2Copy.getFieldValue();
+        assertEquals(motherValForS2, motherValForS2Copy);
+        assertNotSame(motherValForS2, motherValForS2Copy);
+
+        Annotation dummyAnnForS3Copy = annotationsBalloooTreeCopy.next();
+        assertEquals(dummyAnnForS3, dummyAnnForS3Copy);
+        assertNotSame(dummyAnnForS3, dummyAnnForS3Copy);
+
+        Annotation numberAnnForS3Copy = annotationsBalloooTreeCopy.next();
+        assertEquals(numberAnnForS3, numberAnnForS3Copy);
+        assertNotSame(numberAnnForS3, numberAnnForS3Copy);
+        IntegerFieldValue integerValForS3Copy = (IntegerFieldValue) numberAnnForS3Copy.getFieldValue();
+        assertEquals(integerValForS3, integerValForS3Copy);
+        assertNotSame(integerValForS3, integerValForS3Copy);
+
+        Annotation dummyAnnForS5Copy = annotationsBalloooTreeCopy.next();
+        assertEquals(dummyAnnForS5, dummyAnnForS5Copy);
+        assertNotSame(dummyAnnForS5, dummyAnnForS5Copy);
+
+        Annotation daughterAnnForS6Copy = annotationsBalloooTreeCopy.next();
+        assertEquals(daughterAnnForS6, daughterAnnForS6Copy);
+        assertNotSame(daughterAnnForS6, daughterAnnForS6Copy);
+        Struct daughterValForS6Copy = (Struct) daughterAnnForS6Copy.getFieldValue();
+        assertEquals(daughterValForS6, daughterValForS6Copy);
+        assertNotSame(daughterValForS6, daughterValForS6Copy);
+        AnnotationReference refFromS6ToMotherAnnCopy = (AnnotationReference) daughterValForS6Copy.getFieldValue("related");
+        assertEquals(refFromS6ToMotherAnn, refFromS6ToMotherAnnCopy);
+        assertNotSame(refFromS6ToMotherAnn, refFromS6ToMotherAnnCopy);
+
+        assertEquals(str, strCopy);
+
+    }
+
+    @Test
+    public void testCyclicReferences() {
+        DocumentTypeManager docMan = new DocumentTypeManager();
+        AnnotationTypeRegistry reg = docMan.getAnnotationTypeRegistry();
+
+        StringFieldValue strfval = new StringFieldValue("hohohoho");
+        SpanList root = new SpanList();
+        SpanTree tree = new SpanTree("habahaba", root);
+        strfval.setSpanTree(tree);
+
+        //set up types:
+        AnnotationType endTagType = new AnnotationType("endtag");
+        AnnotationType beginTagType = new AnnotationType("begintag");
+        AnnotationReferenceDataType refToBeginTagDataType = new AnnotationReferenceDataType(beginTagType);
+        AnnotationReferenceDataType refToEndTagDataType = new AnnotationReferenceDataType(endTagType);
+        endTagType.setDataType(refToBeginTagDataType);
+        beginTagType.setDataType(refToEndTagDataType);
+
+        //register types:
+        reg.register(endTagType);
+        reg.register(beginTagType);
+        docMan.register(refToBeginTagDataType);
+        docMan.register(refToEndTagDataType);
+
+        //set up annotations and their references to each other:
+        AnnotationReference refToBeginTag = new AnnotationReference(refToBeginTagDataType);
+        AnnotationReference refToEndTag = new AnnotationReference(refToEndTagDataType);
+        Annotation beginTag = new Annotation(beginTagType, refToEndTag);
+        Annotation endTag = new Annotation(endTagType, refToBeginTag);
+        refToBeginTag.setReference(beginTag);
+        refToEndTag.setReference(endTag);
+
+        //create spans:
+        Span beginTagSpan = new Span(0, 1);
+        Span endTagSpan = new Span(1, 1);
+        root.add(beginTagSpan);
+        root.add(endTagSpan);
+
+        //annotate spans:
+        tree.annotate(beginTagSpan, beginTag);
+        tree.annotate(endTagSpan, endTag);
+
+
+        //none of the below statements should lead to a StackOverflowError:
+        assertFalse(endTag.toString().equals(beginTag.toString()));
+
+        assertTrue(endTag.hashCode() != beginTag.hashCode());
+
+        assertFalse(endTag.equals(beginTag));
+        assertFalse(beginTag.equals(endTag));
+
+        StringFieldValue copyString = strfval.clone();
+        assertEquals(strfval, copyString);
+
+
+        GrowableByteBuffer buffer = new GrowableByteBuffer(1024);
+        DocumentSerializer serializer = DocumentSerializerFactory.create42(buffer);
+
+        serializer.write(new Field("stringfield", DataType.STRING), strfval);
+        buffer.flip();
+
+        DocumentDeserializer deserializer = DocumentDeserializerFactory.create42(docMan, buffer);
+        StringFieldValue stringFieldValue2 = new StringFieldValue();
+        deserializer.read(new Field("stringfield", DataType.STRING), stringFieldValue2);
+
+        assertEquals(strfval, stringFieldValue2);
+        assertNotSame(strfval, stringFieldValue2);
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/annotation/SystemTestCase.java b/document/src/test/java/com/yahoo/document/annotation/SystemTestCase.java
new file mode 100755
index 00000000000..e8afa472233
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/annotation/SystemTestCase.java
@@ -0,0 +1,130 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+import com.yahoo.document.Document;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.DocumentTypeManager;
+import com.yahoo.document.DocumentTypeManagerConfigurer;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.document.datatypes.Struct;
+import com.yahoo.document.serialization.*;
+import com.yahoo.io.GrowableByteBuffer;
+
+import java.util.Iterator;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class SystemTestCase extends junit.framework.TestCase {
+    DocumentTypeManager manager;
+
+    private void annotate(Document document) {
+        AnnotationTypeRegistry registry = manager.getAnnotationTypeRegistry();
+        AnnotationType personType = registry.getType("person");
+        AnnotationType artistType = registry.getType("artist");
+        AnnotationType dateType = registry.getType("date");
+        AnnotationType placeType = registry.getType("place");
+        AnnotationType eventType = registry.getType("event");
+
+        SpanList root = new SpanList();
+        SpanTree tree = new SpanTree("meaningoflife", root);
+
+        SpanNode personSpan = new Span(0,5);
+        SpanNode artistSpan = new Span(5,10);
+        SpanNode dateSpan = new Span(10,15);
+        SpanNode placeSpan = new Span(15,20);
+
+        root.add(personSpan);
+        root.add(artistSpan);
+        root.add(dateSpan);
+        root.add(placeSpan);
+
+        Struct personValue = new Struct(manager.getDataType("annotation.person"));
+        personValue.setFieldValue("name", "george washington");
+        Annotation person = new Annotation(personType, personValue);
+        tree.annotate(personSpan, person);
+
+        Struct artistValue = new Struct(manager.getDataType("annotation.artist"));
+        artistValue.setFieldValue("name", "elvis presley");
+        artistValue.setFieldValue("instrument", 20);
+        Annotation artist = new Annotation(artistType, artistValue);
+        tree.annotate(artistSpan, artist);
+
+        Struct dateValue = new Struct(manager.getDataType("annotation.date"));
+        dateValue.setFieldValue("exacttime", 123456789L);
+        Annotation date = new Annotation(dateType, dateValue);
+        tree.annotate(dateSpan, date);
+
+        Struct placeValue = new Struct(manager.getDataType("annotation.place"));
+        placeValue.setFieldValue("lat", 1467L);
+        placeValue.setFieldValue("lon", 789L);
+        Annotation place = new Annotation(placeType, placeValue);
+        tree.annotate(placeSpan, place);
+
+        Struct eventValue = new Struct(manager.getDataType("annotation.event"));
+        eventValue.setFieldValue("description", "Big concert");
+        eventValue.setFieldValue("person", new AnnotationReference((AnnotationReferenceDataType) manager.getDataType("annotationreference<person>"), person));
+        eventValue.setFieldValue("date", new AnnotationReference((AnnotationReferenceDataType) manager.getDataType("annotationreference<date>"), date));
+        eventValue.setFieldValue("place", new AnnotationReference((AnnotationReferenceDataType) manager.getDataType("annotationreference<place>"), place));
+        Annotation event = new Annotation(eventType, eventValue);
+        tree.annotate(root, event);
+
+        StringFieldValue content = new StringFieldValue("This is the story of a big concert by Elvis and a special guest appearance by George Washington");
+        content.setSpanTree(tree);
+
+        document.setFieldValue(document.getDataType().getField("content"), content);
+    }
+
+    private void consume(Document document) {
+        StringFieldValue content = (StringFieldValue) document.getFieldValue(document.getDataType().getField("content"));
+
+        SpanTree tree = content.getSpanTree("meaningoflife");
+        SpanList root = (SpanList) tree.getRoot();
+
+        Iterator<SpanNode> childIterator = root.childIterator();
+        SpanNode personSpan = childIterator.next();
+        SpanNode artistSpan = childIterator.next();
+        SpanNode dateSpan = childIterator.next();
+        SpanNode placeSpan = childIterator.next();
+
+        Annotation person = tree.iterator(personSpan).next();
+        Struct personValue = (Struct) person.getFieldValue();
+        System.err.println("Person is " + personValue.getField("name"));
+
+        Annotation artist = tree.iterator(artistSpan).next();
+        Struct artistValue = (Struct) artist.getFieldValue();
+        System.err.println("Artist is " + artistValue.getFieldValue("name") + " who plays the " + artistValue.getFieldValue("instrument"));
+
+        Annotation date = tree.iterator(dateSpan).next();
+        Struct dateValue = (Struct) date.getFieldValue();
+        System.err.println("Date is " + dateValue.getFieldValue("exacttime"));
+
+        Annotation place = tree.iterator(placeSpan).next();
+        Struct placeValue = (Struct) place.getFieldValue();
+        System.err.println("Place is " + placeValue.getFieldValue("lat") + ";" + placeValue.getFieldValue("lon"));
+
+        Annotation event = tree.iterator(root).next();
+        Struct eventValue = (Struct) event.getFieldValue();
+        System.err.println("Event is " + eventValue.getFieldValue("description") + " with " + eventValue.getFieldValue("person") + " and " + eventValue.getFieldValue("date") + " and " + eventValue.getFieldValue("place"));
+    }
+
+    public void setUp() {
+        manager = new DocumentTypeManager();
+        DocumentTypeManagerConfigurer.configure(manager, "file:src/test/java/com/yahoo/document/annotation/documentmanager.systemtest.cfg");
+    }
+
+    public void testSystemTest() {
+        DocumentType type = manager.getDocumentType("article");
+        Document inDocument = new Document(type, "doc:article:boringarticle:longarticle");
+        annotate(inDocument);
+
+        GrowableByteBuffer buffer = new GrowableByteBuffer();
+        DocumentSerializer serializer = DocumentSerializerFactory.create42(buffer);
+        serializer.write(inDocument);
+        buffer.flip();
+        DocumentDeserializer deserializer = DocumentDeserializerFactory.create42(manager, buffer);
+
+        Document outDocument = new Document(deserializer);
+        consume(outDocument);
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/annotation/documentmanager.6394548.cfg b/document/src/test/java/com/yahoo/document/annotation/documentmanager.6394548.cfg
new file mode 100644
index 00000000000..35717a3e0c1
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/annotation/documentmanager.6394548.cfg
@@ -0,0 +1,189 @@
+enablecompression false
+datatype[0].id 1381038251
+datatype[0].structtype[0].name "position"
+datatype[0].structtype[0].version 0
+datatype[0].structtype[0].compresstype NONE
+datatype[0].structtype[0].compresslevel 0
+datatype[0].structtype[0].compressthreshold 95
+datatype[0].structtype[0].compressminsize 800
+datatype[0].structtype[0].field[0].name "x"
+datatype[0].structtype[0].field[0].datatype 0
+datatype[0].structtype[0].field[1].name "y"
+datatype[0].structtype[0].field[1].datatype 0
+datatype[1].id -1466283082
+datatype[1].structtype[0].name "annotation.person"
+datatype[1].structtype[0].version 0
+datatype[1].structtype[0].compresstype NONE
+datatype[1].structtype[0].compresslevel 0
+datatype[1].structtype[0].compressthreshold 95
+datatype[1].structtype[0].compressminsize 800
+datatype[1].structtype[0].field[0].name "name"
+datatype[1].structtype[0].field[0].datatype 2
+datatype[2].id -1149562679
+datatype[2].annotationreftype[0].annotation "person"
+datatype[3].id -772171888
+datatype[3].annotationreftype[0].annotation "date"
+datatype[4].id -2109350185
+datatype[4].annotationreftype[0].annotation "place"
+datatype[5].id 1194300957
+datatype[5].structtype[0].name "annotation.event"
+datatype[5].structtype[0].version 0
+datatype[5].structtype[0].compresstype NONE
+datatype[5].structtype[0].compresslevel 0
+datatype[5].structtype[0].compressthreshold 95
+datatype[5].structtype[0].compressminsize 800
+datatype[5].structtype[0].field[0].name "description"
+datatype[5].structtype[0].field[0].datatype 2
+datatype[5].structtype[0].field[1].name "person"
+datatype[5].structtype[0].field[1].datatype -1149562679
+datatype[5].structtype[0].field[2].name "date"
+datatype[5].structtype[0].field[2].datatype -772171888
+datatype[5].structtype[0].field[3].name "place"
+datatype[5].structtype[0].field[3].datatype -2109350185
+datatype[6].id 1463704666
+datatype[6].structtype[0].name "annotation.morty.RICK_DOCSTATS"
+datatype[6].structtype[0].version 0
+datatype[6].structtype[0].compresstype NONE
+datatype[6].structtype[0].compresslevel 0
+datatype[6].structtype[0].compressthreshold 95
+datatype[6].structtype[0].compressminsize 800
+datatype[6].structtype[0].field[0].name "bodycount"
+datatype[6].structtype[0].field[0].datatype 0
+datatype[6].structtype[0].field[1].name "anchorcount"
+datatype[6].structtype[0].field[1].datatype 0
+datatype[7].id 1157126952
+datatype[7].structtype[0].name "annotation.artist"
+datatype[7].structtype[0].version 0
+datatype[7].structtype[0].compresstype NONE
+datatype[7].structtype[0].compresslevel 0
+datatype[7].structtype[0].compressthreshold 95
+datatype[7].structtype[0].compressminsize 800
+datatype[7].structtype[0].field[0].name "instrument"
+datatype[7].structtype[0].field[0].datatype 0
+datatype[7].structtype[0].inherits[0].name "annotation.person"
+datatype[7].structtype[0].inherits[0].version 0
+datatype[8].id 2076579146
+datatype[8].structtype[0].name "annotation.place"
+datatype[8].structtype[0].version 0
+datatype[8].structtype[0].compresstype NONE
+datatype[8].structtype[0].compresslevel 0
+datatype[8].structtype[0].compressthreshold 95
+datatype[8].structtype[0].compressminsize 800
+datatype[8].structtype[0].field[0].name "lat"
+datatype[8].structtype[0].field[0].datatype 4
+datatype[8].structtype[0].field[1].name "lon"
+datatype[8].structtype[0].field[1].datatype 4
+datatype[9].id -840345201
+datatype[9].structtype[0].name "annotation.date"
+datatype[9].structtype[0].version 0
+datatype[9].structtype[0].compresstype NONE
+datatype[9].structtype[0].compresslevel 0
+datatype[9].structtype[0].compressthreshold 95
+datatype[9].structtype[0].compressminsize 800
+datatype[9].structtype[0].field[0].name "exacttime"
+datatype[9].structtype[0].field[0].datatype 4
+datatype[10].id -1486737430
+datatype[10].arraytype[0].datatype 2
+datatype[11].id 1021048351
+datatype[11].structtype[0].name "annotation.morty.RICK_FEATURESET"
+datatype[11].structtype[0].version 0
+datatype[11].structtype[0].compresstype NONE
+datatype[11].structtype[0].compresslevel 0
+datatype[11].structtype[0].compressthreshold 95
+datatype[11].structtype[0].compressminsize 800
+datatype[11].structtype[0].field[0].name "foo1"
+datatype[11].structtype[0].field[0].datatype 2
+datatype[11].structtype[0].field[1].name "foo2"
+datatype[11].structtype[0].field[1].datatype 0
+datatype[11].structtype[0].field[2].name "foo3"
+datatype[11].structtype[0].field[2].datatype 0
+datatype[11].structtype[0].field[3].name "foo4"
+datatype[11].structtype[0].field[3].datatype 2
+datatype[11].structtype[0].field[4].name "foo5"
+datatype[11].structtype[0].field[4].datatype 2
+datatype[11].structtype[0].field[5].name "foo6"
+datatype[11].structtype[0].field[5].datatype -1486737430
+datatype[11].structtype[0].field[6].name "foo7"
+datatype[11].structtype[0].field[6].datatype 0
+datatype[11].structtype[0].field[7].name "foo8"
+datatype[11].structtype[0].field[7].datatype 0
+datatype[11].structtype[0].field[8].name "foo9"
+datatype[11].structtype[0].field[8].datatype 1
+datatype[11].structtype[0].field[9].name "foo10"
+datatype[11].structtype[0].field[9].datatype -1486737430
+datatype[11].structtype[0].inherits[0].name "annotation.morty.FEATURESET"
+datatype[11].structtype[0].inherits[0].version 0
+datatype[12].id -228273582
+datatype[12].maptype[0].keytype 2
+datatype[12].maptype[0].valtype 5
+datatype[13].id -1584287606
+datatype[13].maptype[0].keytype 2
+datatype[13].maptype[0].valtype 0
+datatype[14].id 1980242844
+datatype[14].structtype[0].name "annotation.morty.FEATURESET"
+datatype[14].structtype[0].version 0
+datatype[14].structtype[0].compresstype NONE
+datatype[14].structtype[0].compresslevel 0
+datatype[14].structtype[0].compressthreshold 95
+datatype[14].structtype[0].compressminsize 800
+datatype[14].structtype[0].field[0].name "realvaluedfeatures"
+datatype[14].structtype[0].field[0].datatype -228273582
+datatype[14].structtype[0].field[1].name "discretevaluedfeatures"
+datatype[14].structtype[0].field[1].datatype -1584287606
+datatype[14].structtype[0].field[2].name "score"
+datatype[14].structtype[0].field[2].datatype 5
+datatype[15].id 892457735
+datatype[15].structtype[0].name "article.header"
+datatype[15].structtype[0].version 0
+datatype[15].structtype[0].compresstype NONE
+datatype[15].structtype[0].compresslevel 0
+datatype[15].structtype[0].compressthreshold 95
+datatype[15].structtype[0].compressminsize 800
+datatype[15].structtype[0].field[0].name "title"
+datatype[15].structtype[0].field[0].datatype 2
+datatype[15].structtype[0].field[1].name "content"
+datatype[15].structtype[0].field[1].datatype 2
+datatype[15].structtype[0].field[2].name "rankfeatures"
+datatype[15].structtype[0].field[2].datatype 2
+datatype[15].structtype[0].field[3].name "summaryfeatures"
+datatype[15].structtype[0].field[3].datatype 2
+datatype[16].id -1984964900
+datatype[16].structtype[0].name "article.body"
+datatype[16].structtype[0].version 0
+datatype[16].structtype[0].compresstype NONE
+datatype[16].structtype[0].compresslevel 0
+datatype[16].structtype[0].compressthreshold 95
+datatype[16].structtype[0].compressminsize 800
+datatype[17].id 559508792
+datatype[17].documenttype[0].name "article"
+datatype[17].documenttype[0].version 0
+datatype[17].documenttype[0].inherits[0].name "document"
+datatype[17].documenttype[0].inherits[0].version 0
+datatype[17].documenttype[0].headerstruct 892457735
+datatype[17].documenttype[0].bodystruct -1984964900
+annotationtype[0].id 609952424
+annotationtype[0].name "person"
+annotationtype[0].datatype -1466283082
+annotationtype[1].id -455530995
+annotationtype[1].name "event"
+annotationtype[1].datatype 1194300957
+annotationtype[2].id 295631537
+annotationtype[2].name "morty.RICK_DOCSTATS"
+annotationtype[2].datatype 1463704666
+annotationtype[3].id 690330276
+annotationtype[3].name "artist"
+annotationtype[3].datatype 1157126952
+annotationtype[3].inherits[0].id 609952424
+annotationtype[4].id -162455681
+annotationtype[4].name "date"
+annotationtype[4].datatype -840345201
+annotationtype[5].id 1707984040
+annotationtype[5].name "place"
+annotationtype[5].datatype 2076579146
+annotationtype[6].id -62680437
+annotationtype[6].name "morty.FEATURESET"
+annotationtype[6].datatype 1980242844
+annotationtype[7].id 1056322897
+annotationtype[7].name "morty.RICK_FEATURESET"
+annotationtype[7].datatype 1021048351
+annotationtype[7].inherits[0].id -62680437
diff --git a/document/src/test/java/com/yahoo/document/annotation/documentmanager.bug4259784.cfg b/document/src/test/java/com/yahoo/document/annotation/documentmanager.bug4259784.cfg
new file mode 100644
index 00000000000..77d72ed264e
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/annotation/documentmanager.bug4259784.cfg
@@ -0,0 +1,147 @@
+enablecompression false
+annotationtype[4]
+annotationtype[0].datatype -476092672
+annotationtype[0].id 1278713514
+annotationtype[0].name "company"
+annotationtype[0].inherits[1]
+annotationtype[0].inherits[0].id 9765800
+annotationtype[1].datatype 912259135
+annotationtype[1].id 9765800
+annotationtype[1].name "industry"
+annotationtype[1].inherits[0]
+annotationtype[2].datatype 515587158
+annotationtype[2].id -270471211
+annotationtype[2].name "location"
+annotationtype[2].inherits[0]
+annotationtype[3].datatype -1466283082
+annotationtype[3].id 609952424
+annotationtype[3].name "person"
+annotationtype[3].inherits[0]
+datatype[10]
+datatype[0].id -1149562679
+datatype[0].annotationreftype[1]
+datatype[0].annotationreftype[0].annotation "person"
+datatype[0].arraytype[0]
+datatype[0].documenttype[0]
+datatype[0].structtype[0]
+datatype[0].weightedsettype[0]
+datatype[1].id -1386162972
+datatype[1].annotationreftype[0]
+datatype[1].arraytype[0]
+datatype[1].documenttype[1]
+datatype[1].documenttype[0].bodystruct 1387420336
+datatype[1].documenttype[0].headerstruct -945638949
+datatype[1].documenttype[0].name "blog"
+datatype[1].documenttype[0].version 0
+datatype[1].documenttype[0].inherits[0]
+datatype[1].structtype[0]
+datatype[1].weightedsettype[0]
+datatype[2].id -1466283082
+datatype[2].annotationreftype[0]
+datatype[2].arraytype[0]
+datatype[2].documenttype[0]
+datatype[2].structtype[1]
+datatype[2].structtype[0].name "annotation.person"
+datatype[2].structtype[0].version 0
+datatype[2].structtype[0].field[1]
+datatype[2].structtype[0].field[0].datatype 2
+datatype[2].structtype[0].field[0].name "name"
+datatype[2].structtype[0].field[0].id[0]
+datatype[2].structtype[0].inherits[0]
+datatype[2].weightedsettype[0]
+datatype[3].id -476092672
+datatype[3].annotationreftype[0]
+datatype[3].arraytype[0]
+datatype[3].documenttype[0]
+datatype[3].structtype[1]
+datatype[3].structtype[0].name "annotation.company"
+datatype[3].structtype[0].version 0
+datatype[3].structtype[0].field[3]
+datatype[3].structtype[0].field[0].datatype 1184817987
+datatype[3].structtype[0].field[0].name "directors"
+datatype[3].structtype[0].field[0].id[0]
+datatype[3].structtype[0].field[1].datatype 2
+datatype[3].structtype[0].field[1].name "name"
+datatype[3].structtype[0].field[1].id[0]
+datatype[3].structtype[0].field[2].datatype 1321486441
+datatype[3].structtype[0].field[2].name "place"
+datatype[3].structtype[0].field[2].id[0]
+datatype[3].structtype[0].inherits[1]
+datatype[3].structtype[0].inherits[0].name "annotation.industry"
+datatype[3].structtype[0].inherits[0].version 0
+datatype[3].weightedsettype[0]
+datatype[4].id -945638949
+datatype[4].annotationreftype[0]
+datatype[4].arraytype[0]
+datatype[4].documenttype[0]
+datatype[4].structtype[1]
+datatype[4].structtype[0].name "blog.header"
+datatype[4].structtype[0].version 0
+datatype[4].structtype[0].field[4]
+datatype[4].structtype[0].field[0].datatype 2
+datatype[4].structtype[0].field[0].name "author"
+datatype[4].structtype[0].field[0].id[0]
+datatype[4].structtype[0].field[1].datatype 2
+datatype[4].structtype[0].field[1].name "body"
+datatype[4].structtype[0].field[1].id[0]
+datatype[4].structtype[0].field[2].datatype 2
+datatype[4].structtype[0].field[2].name "title"
+datatype[4].structtype[0].field[2].id[0]
+datatype[4].structtype[0].field[3].datatype 10
+datatype[4].structtype[0].field[3].name "url"
+datatype[4].structtype[0].field[3].id[0]
+datatype[4].structtype[0].inherits[0]
+datatype[4].weightedsettype[0]
+datatype[5].id 1184817987
+datatype[5].annotationreftype[0]
+datatype[5].arraytype[1]
+datatype[5].arraytype[0].datatype -1149562679
+datatype[5].documenttype[0]
+datatype[5].structtype[0]
+datatype[5].weightedsettype[0]
+datatype[6].id 1321486441
+datatype[6].annotationreftype[1]
+datatype[6].annotationreftype[0].annotation "location"
+datatype[6].arraytype[0]
+datatype[6].documenttype[0]
+datatype[6].structtype[0]
+datatype[6].weightedsettype[0]
+datatype[7].id 1387420336
+datatype[7].annotationreftype[0]
+datatype[7].arraytype[0]
+datatype[7].documenttype[0]
+datatype[7].structtype[1]
+datatype[7].structtype[0].name "blog.body"
+datatype[7].structtype[0].version 0
+datatype[7].structtype[0].field[0]
+datatype[7].structtype[0].inherits[0]
+datatype[7].weightedsettype[0]
+datatype[8].id 515587158
+datatype[8].annotationreftype[0]
+datatype[8].arraytype[0]
+datatype[8].documenttype[0]
+datatype[8].structtype[1]
+datatype[8].structtype[0].name "annotation.location"
+datatype[8].structtype[0].version 0
+datatype[8].structtype[0].field[2]
+datatype[8].structtype[0].field[0].datatype 1
+datatype[8].structtype[0].field[0].name "lat"
+datatype[8].structtype[0].field[0].id[0]
+datatype[8].structtype[0].field[1].datatype 1
+datatype[8].structtype[0].field[1].name "lon"
+datatype[8].structtype[0].field[1].id[0]
+datatype[8].structtype[0].inherits[0]
+datatype[8].weightedsettype[0]
+datatype[9].id 912259135
+datatype[9].annotationreftype[0]
+datatype[9].arraytype[0]
+datatype[9].documenttype[0]
+datatype[9].structtype[1]
+datatype[9].structtype[0].name "annotation.industry"
+datatype[9].structtype[0].version 0
+datatype[9].structtype[0].field[1]
+datatype[9].structtype[0].field[0].datatype 2
+datatype[9].structtype[0].field[0].name "vertical"
+datatype[9].structtype[0].field[0].id[0]
+datatype[9].structtype[0].inherits[0]
+datatype[9].weightedsettype[0]
diff --git a/document/src/test/java/com/yahoo/document/annotation/documentmanager.bug4261985.cfg b/document/src/test/java/com/yahoo/document/annotation/documentmanager.bug4261985.cfg
new file mode 100644
index 00000000000..bbc67412652
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/annotation/documentmanager.bug4261985.cfg
@@ -0,0 +1,181 @@
+enablecompression false
+annotationtype[5]
+annotationtype[0].datatype 1574163290
+annotationtype[0].id 1883143872
+annotationtype[0].name "bigshots"
+annotationtype[0].inherits[0]
+annotationtype[1].datatype -476092672
+annotationtype[1].id 1278713514
+annotationtype[1].name "company"
+annotationtype[1].inherits[1]
+annotationtype[1].inherits[0].id 9765800
+annotationtype[2].datatype 912259135
+annotationtype[2].id 9765800
+annotationtype[2].name "industry"
+annotationtype[2].inherits[0]
+annotationtype[3].datatype 515587158
+annotationtype[3].id -270471211
+annotationtype[3].name "location"
+annotationtype[3].inherits[0]
+annotationtype[4].datatype -1466283082
+annotationtype[4].id 609952424
+annotationtype[4].name "person"
+annotationtype[4].inherits[0]
+datatype[13]
+datatype[00].id -1149562679
+datatype[00].annotationreftype[1]
+datatype[00].annotationreftype[0].annotation "person"
+datatype[00].arraytype[0]
+datatype[00].documenttype[0]
+datatype[00].structtype[0]
+datatype[00].weightedsettype[0]
+datatype[01].id -1386162972
+datatype[01].annotationreftype[0]
+datatype[01].arraytype[0]
+datatype[01].documenttype[1]
+datatype[01].documenttype[0].bodystruct 1387420336
+datatype[01].documenttype[0].headerstruct -945638949
+datatype[01].documenttype[0].name "blog"
+datatype[01].documenttype[0].version 0
+datatype[01].documenttype[0].inherits[0]
+datatype[01].structtype[0]
+datatype[01].weightedsettype[0]
+datatype[02].id -1466283082
+datatype[02].annotationreftype[0]
+datatype[02].arraytype[0]
+datatype[02].documenttype[0]
+datatype[02].structtype[1]
+datatype[02].structtype[0].name "annotation.person"
+datatype[02].structtype[0].version 0
+datatype[02].structtype[0].field[1]
+datatype[02].structtype[0].field[0].datatype 2
+datatype[02].structtype[0].field[0].name "name"
+datatype[02].structtype[0].field[0].id[0]
+datatype[02].structtype[0].inherits[0]
+datatype[02].weightedsettype[0]
+datatype[03].id -476092672
+datatype[03].annotationreftype[0]
+datatype[03].arraytype[0]
+datatype[03].documenttype[0]
+datatype[03].structtype[1]
+datatype[03].structtype[0].name "annotation.company"
+datatype[03].structtype[0].version 0
+datatype[03].structtype[0].field[2]
+datatype[03].structtype[0].field[0].datatype 1184817987
+datatype[03].structtype[0].field[0].name "directors"
+datatype[03].structtype[0].field[0].id[0]
+datatype[03].structtype[0].field[1].datatype 2
+datatype[03].structtype[0].field[1].name "name"
+datatype[03].structtype[0].field[1].id[0]
+datatype[03].structtype[0].inherits[1]
+datatype[03].structtype[0].inherits[0].name "annotation.industry"
+datatype[03].structtype[0].inherits[0].version 0
+datatype[03].weightedsettype[0]
+datatype[04].id -945638949
+datatype[04].annotationreftype[0]
+datatype[04].arraytype[0]
+datatype[04].documenttype[0]
+datatype[04].structtype[1]
+datatype[04].structtype[0].name "blog.header"
+datatype[04].structtype[0].version 0
+datatype[04].structtype[0].field[4]
+datatype[04].structtype[0].field[0].datatype 2
+datatype[04].structtype[0].field[0].name "author"
+datatype[04].structtype[0].field[0].id[0]
+datatype[04].structtype[0].field[1].datatype 2
+datatype[04].structtype[0].field[1].name "body"
+datatype[04].structtype[0].field[1].id[0]
+datatype[04].structtype[0].field[2].datatype 2
+datatype[04].structtype[0].field[2].name "title"
+datatype[04].structtype[0].field[2].id[0]
+datatype[04].structtype[0].field[3].datatype 10
+datatype[04].structtype[0].field[3].name "url"
+datatype[04].structtype[0].field[3].id[0]
+datatype[04].structtype[0].inherits[0]
+datatype[04].weightedsettype[0]
+datatype[05].id 108708069
+datatype[05].annotationreftype[1]
+datatype[05].annotationreftype[0].annotation "bigshots"
+datatype[05].arraytype[0]
+datatype[05].documenttype[0]
+datatype[05].structtype[0]
+datatype[05].weightedsettype[0]
+datatype[06].id 1184817987
+datatype[06].annotationreftype[0]
+datatype[06].arraytype[1]
+datatype[06].arraytype[0].datatype -1149562679
+datatype[06].documenttype[0]
+datatype[06].structtype[0]
+datatype[06].weightedsettype[0]
+datatype[07].id 1321486441
+datatype[07].annotationreftype[1]
+datatype[07].annotationreftype[0].annotation "location"
+datatype[07].arraytype[0]
+datatype[07].documenttype[0]
+datatype[07].structtype[0]
+datatype[07].weightedsettype[0]
+datatype[08].id 1387420336
+datatype[08].annotationreftype[0]
+datatype[08].arraytype[0]
+datatype[08].documenttype[0]
+datatype[08].structtype[1]
+datatype[08].structtype[0].name "blog.body"
+datatype[08].structtype[0].version 0
+datatype[08].structtype[0].field[0]
+datatype[08].structtype[0].inherits[0]
+datatype[08].weightedsettype[0]
+datatype[09].id 1574163290
+datatype[09].annotationreftype[0]
+datatype[09].arraytype[0]
+datatype[09].documenttype[0]
+datatype[09].structtype[1]
+datatype[09].structtype[0].name "annotation.bigshots"
+datatype[09].structtype[0].version 0
+datatype[09].structtype[0].field[2]
+datatype[09].structtype[0].field[0].datatype 1975335457
+datatype[09].structtype[0].field[0].name "ceos"
+datatype[09].structtype[0].field[0].id[0]
+datatype[09].structtype[0].field[1].datatype 108708069
+datatype[09].structtype[0].field[1].name "self"
+datatype[09].structtype[0].field[1].id[0]
+datatype[09].structtype[0].inherits[0]
+datatype[09].weightedsettype[0]
+datatype[10].id 1975335457
+datatype[10].annotationreftype[1]
+datatype[10].annotationreftype[0].annotation "company"
+datatype[10].arraytype[0]
+datatype[10].documenttype[0]
+datatype[10].structtype[0]
+datatype[10].weightedsettype[0]
+datatype[11].id 515587158
+datatype[11].annotationreftype[0]
+datatype[11].arraytype[0]
+datatype[11].documenttype[0]
+datatype[11].structtype[1]
+datatype[11].structtype[0].name "annotation.location"
+datatype[11].structtype[0].version 0
+datatype[11].structtype[0].field[2]
+datatype[11].structtype[0].field[0].datatype 1
+datatype[11].structtype[0].field[0].name "lat"
+datatype[11].structtype[0].field[0].id[0]
+datatype[11].structtype[0].field[1].datatype 1
+datatype[11].structtype[0].field[1].name "lon"
+datatype[11].structtype[0].field[1].id[0]
+datatype[11].structtype[0].inherits[0]
+datatype[11].weightedsettype[0]
+datatype[12].id 912259135
+datatype[12].annotationreftype[0]
+datatype[12].arraytype[0]
+datatype[12].documenttype[0]
+datatype[12].structtype[1]
+datatype[12].structtype[0].name "annotation.industry"
+datatype[12].structtype[0].version 0
+datatype[12].structtype[0].field[2]
+datatype[12].structtype[0].field[0].datatype 1321486441
+datatype[12].structtype[0].field[0].name "place"
+datatype[12].structtype[0].field[0].id[0]
+datatype[12].structtype[0].field[1].datatype 2
+datatype[12].structtype[0].field[1].name "vertical"
+datatype[12].structtype[0].field[1].id[0]
+datatype[12].structtype[0].inherits[0]
+datatype[12].weightedsettype[0]
diff --git a/document/src/test/java/com/yahoo/document/annotation/documentmanager.bug4475379.cfg b/document/src/test/java/com/yahoo/document/annotation/documentmanager.bug4475379.cfg
new file mode 100644
index 00000000000..ae299ab81fa
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/annotation/documentmanager.bug4475379.cfg
@@ -0,0 +1,129 @@
+enablecompression false
+annotationtype[4]
+annotationtype[0].datatype -476092672
+annotationtype[0].id 1278713514
+annotationtype[0].name "company"
+annotationtype[0].inherits[1]
+annotationtype[0].inherits[0].id 9765800
+annotationtype[1].datatype 912259135
+annotationtype[1].id 9765800
+annotationtype[1].name "industry"
+annotationtype[1].inherits[0]
+annotationtype[2].datatype 515587158
+annotationtype[2].id -270471211
+annotationtype[2].name "location"
+annotationtype[2].inherits[0]
+annotationtype[3].datatype -1466283082
+annotationtype[3].id 609952424
+annotationtype[3].name "person"
+annotationtype[3].inherits[0]
+datatype[7]
+datatype[0].id -1386162972
+datatype[0].annotationreftype[0]
+datatype[0].arraytype[0]
+datatype[0].documenttype[1]
+datatype[0].documenttype[0].bodystruct 1387420336
+datatype[0].documenttype[0].headerstruct -945638949
+datatype[0].documenttype[0].name "blog"
+datatype[0].documenttype[0].version 0
+datatype[0].documenttype[0].inherits[0]
+datatype[0].structtype[0]
+datatype[0].weightedsettype[0]
+datatype[1].id -1466283082
+datatype[1].annotationreftype[0]
+datatype[1].arraytype[0]
+datatype[1].documenttype[0]
+datatype[1].structtype[1]
+datatype[1].structtype[0].name "annotation.person"
+datatype[1].structtype[0].version 0
+datatype[1].structtype[0].field[1]
+datatype[1].structtype[0].field[0].datatype 2
+datatype[1].structtype[0].field[0].name "name"
+datatype[1].structtype[0].field[0].id[0]
+datatype[1].structtype[0].inherits[0]
+datatype[1].weightedsettype[0]
+datatype[2].id -476092672
+datatype[2].annotationreftype[0]
+datatype[2].arraytype[0]
+datatype[2].documenttype[0]
+datatype[2].structtype[1]
+datatype[2].structtype[0].name "annotation.company"
+datatype[2].structtype[0].version 0
+datatype[2].structtype[0].field[5]
+datatype[2].structtype[0].field[0].datatype 4
+datatype[2].structtype[0].field[0].name "alt"
+datatype[2].structtype[0].field[0].id[0]
+datatype[2].structtype[0].field[1].datatype 2
+datatype[2].structtype[0].field[1].name "ceo"
+datatype[2].structtype[0].field[1].id[0]
+datatype[2].structtype[0].field[2].datatype 1
+datatype[2].structtype[0].field[2].name "lat"
+datatype[2].structtype[0].field[2].id[0]
+datatype[2].structtype[0].field[3].datatype 1
+datatype[2].structtype[0].field[3].name "lon"
+datatype[2].structtype[0].field[3].id[0]
+datatype[2].structtype[0].field[4].datatype 2
+datatype[2].structtype[0].field[4].name "name"
+datatype[2].structtype[0].field[4].id[0]
+datatype[2].structtype[0].inherits[1]
+datatype[2].structtype[0].inherits[0].name "annotation.industry"
+datatype[2].structtype[0].inherits[0].version 0
+datatype[2].weightedsettype[0]
+datatype[3].id -945638949
+datatype[3].annotationreftype[0]
+datatype[3].arraytype[0]
+datatype[3].documenttype[0]
+datatype[3].structtype[1]
+datatype[3].structtype[0].name "blog.header"
+datatype[3].structtype[0].version 0
+datatype[3].structtype[0].field[4]
+datatype[3].structtype[0].field[0].datatype 2
+datatype[3].structtype[0].field[0].name "author"
+datatype[3].structtype[0].field[0].id[0]
+datatype[3].structtype[0].field[1].datatype 2
+datatype[3].structtype[0].field[1].name "body"
+datatype[3].structtype[0].field[1].id[0]
+datatype[3].structtype[0].field[2].datatype 2
+datatype[3].structtype[0].field[2].name "title"
+datatype[3].structtype[0].field[2].id[0]
+datatype[3].structtype[0].field[3].datatype 10
+datatype[3].structtype[0].field[3].name "url"
+datatype[3].structtype[0].field[3].id[0]
+datatype[3].structtype[0].inherits[0]
+datatype[3].weightedsettype[0]
+datatype[4].id 1387420336
+datatype[4].annotationreftype[0]
+datatype[4].arraytype[0]
+datatype[4].documenttype[0]
+datatype[4].structtype[1]
+datatype[4].structtype[0].name "blog.body"
+datatype[4].structtype[0].version 0
+datatype[4].structtype[0].field[0]
+datatype[4].structtype[0].inherits[0]
+datatype[4].weightedsettype[0]
+datatype[5].id 515587158
+datatype[5].annotationreftype[0]
+datatype[5].arraytype[0]
+datatype[5].documenttype[0]
+datatype[5].structtype[1]
+datatype[5].structtype[0].name "annotation.location"
+datatype[5].structtype[0].version 0
+datatype[5].structtype[0].field[1]
+datatype[5].structtype[0].field[0].datatype 2
+datatype[5].structtype[0].field[0].name "name"
+datatype[5].structtype[0].field[0].id[0]
+datatype[5].structtype[0].inherits[0]
+datatype[5].weightedsettype[0]
+datatype[6].id 912259135
+datatype[6].annotationreftype[0]
+datatype[6].arraytype[0]
+datatype[6].documenttype[0]
+datatype[6].structtype[1]
+datatype[6].structtype[0].name "annotation.industry"
+datatype[6].structtype[0].version 0
+datatype[6].structtype[0].field[1]
+datatype[6].structtype[0].field[0].datatype 2
+datatype[6].structtype[0].field[0].name "vertical"
+datatype[6].structtype[0].field[0].id[0]
+datatype[6].structtype[0].inherits[0]
+datatype[6].weightedsettype[0]
diff --git a/document/src/test/java/com/yahoo/document/annotation/documentmanager.systemtest.cfg b/document/src/test/java/com/yahoo/document/annotation/documentmanager.systemtest.cfg
new file mode 100644
index 00000000000..ce8be4410a1
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/annotation/documentmanager.systemtest.cfg
@@ -0,0 +1,155 @@
+enablecompression false
+datatype[11]
+datatype[0].id -198681903
+datatype[0].arraytype[0]
+datatype[0].weightedsettype[0]
+datatype[0].structtype[0]
+datatype[0].documenttype[0]
+datatype[0].annotationreftype[1]
+datatype[0].annotationreftype[0].annotation "person"
+datatype[1].id 1812054936
+datatype[1].arraytype[0]
+datatype[1].weightedsettype[0]
+datatype[1].structtype[0]
+datatype[1].documenttype[0]
+datatype[1].annotationreftype[1]
+datatype[1].annotationreftype[0].annotation "date"
+datatype[2].id 692270031
+datatype[2].arraytype[0]
+datatype[2].weightedsettype[0]
+datatype[2].structtype[0]
+datatype[2].documenttype[0]
+datatype[2].annotationreftype[1]
+datatype[2].annotationreftype[0].annotation "place"
+datatype[3].id 892457735
+datatype[3].arraytype[0]
+datatype[3].weightedsettype[0]
+datatype[3].structtype[1]
+datatype[3].structtype[0].name "article.header"
+datatype[3].structtype[0].version 0
+datatype[3].structtype[0].field[2]
+datatype[3].structtype[0].field[0].name "title"
+datatype[3].structtype[0].field[0].datatype 2
+datatype[3].structtype[0].field[0].id[0]
+datatype[3].structtype[0].field[1].name "content"
+datatype[3].structtype[0].field[1].datatype 2
+datatype[3].structtype[0].field[1].id[0]
+datatype[3].structtype[0].inherits[0]
+datatype[3].documenttype[0]
+datatype[3].annotationreftype[0]
+datatype[4].id -1984964900
+datatype[4].arraytype[0]
+datatype[4].weightedsettype[0]
+datatype[4].structtype[1]
+datatype[4].structtype[0].name "article.body"
+datatype[4].structtype[0].version 0
+datatype[4].structtype[0].field[0]
+datatype[4].structtype[0].inherits[0]
+datatype[4].documenttype[0]
+datatype[4].annotationreftype[0]
+datatype[5].id 559508792
+datatype[5].arraytype[0]
+datatype[5].weightedsettype[0]
+datatype[5].structtype[0]
+datatype[5].documenttype[1]
+datatype[5].documenttype[0].name "article"
+datatype[5].documenttype[0].version 0
+datatype[5].documenttype[0].headerstruct 892457735
+datatype[5].documenttype[0].bodystruct -1984964900
+datatype[5].documenttype[0].inherits[0]
+datatype[5].annotationreftype[0]
+datatype[6].id -1466283082
+datatype[6].arraytype[0]
+datatype[6].weightedsettype[0]
+datatype[6].structtype[1]
+datatype[6].structtype[0].name "annotation.person"
+datatype[6].structtype[0].version 0
+datatype[6].structtype[0].field[1]
+datatype[6].structtype[0].field[0].name "name"
+datatype[6].structtype[0].field[0].datatype 2
+datatype[6].structtype[0].field[0].id[0]
+datatype[6].structtype[0].inherits[0]
+datatype[6].documenttype[0]
+datatype[6].annotationreftype[0]
+datatype[7].id 1157126952
+datatype[7].arraytype[0]
+datatype[7].weightedsettype[0]
+datatype[7].structtype[1]
+datatype[7].structtype[0].name "annotation.artist"
+datatype[7].structtype[0].version 0
+datatype[7].structtype[0].field[1]
+datatype[7].structtype[0].field[0].name "instrument"
+datatype[7].structtype[0].field[0].datatype 0
+datatype[7].structtype[0].field[0].id[0]
+datatype[7].structtype[0].inherits[1]
+datatype[7].structtype[0].inherits[0].name "annotation.person"
+datatype[7].structtype[0].inherits[0].version 0
+datatype[7].documenttype[0]
+datatype[7].annotationreftype[0]
+datatype[8].id -840345201
+datatype[8].arraytype[0]
+datatype[8].weightedsettype[0]
+datatype[8].structtype[1]
+datatype[8].structtype[0].name "annotation.date"
+datatype[8].structtype[0].version 0
+datatype[8].structtype[0].field[1]
+datatype[8].structtype[0].field[0].name "exacttime"
+datatype[8].structtype[0].field[0].datatype 4
+datatype[8].structtype[0].field[0].id[0]
+datatype[8].structtype[0].inherits[0]
+datatype[8].documenttype[0]
+datatype[8].annotationreftype[0]
+datatype[9].id 2076579146
+datatype[9].arraytype[0]
+datatype[9].weightedsettype[0]
+datatype[9].structtype[1]
+datatype[9].structtype[0].name "annotation.place"
+datatype[9].structtype[0].version 0
+datatype[9].structtype[0].field[2]
+datatype[9].structtype[0].field[0].name "lat"
+datatype[9].structtype[0].field[0].datatype 4
+datatype[9].structtype[0].field[0].id[0]
+datatype[9].structtype[0].field[1].name "lon"
+datatype[9].structtype[0].field[1].datatype 4
+datatype[9].structtype[0].field[1].id[0]
+datatype[9].structtype[0].inherits[0]
+datatype[9].documenttype[0]
+datatype[9].annotationreftype[0]
+datatype[10].id 1194300957
+datatype[10].arraytype[0]
+datatype[10].weightedsettype[0]
+datatype[10].structtype[1]
+datatype[10].structtype[0].name "annotation.event"
+datatype[10].structtype[0].version 0
+datatype[10].structtype[0].field[4]
+datatype[10].structtype[0].field[0].name "description"
+datatype[10].structtype[0].field[0].datatype 2
+datatype[10].structtype[0].field[0].id[0]
+datatype[10].structtype[0].field[1].name "person"
+datatype[10].structtype[0].field[1].datatype -198681903
+datatype[10].structtype[0].field[1].id[0]
+datatype[10].structtype[0].field[2].name "date"
+datatype[10].structtype[0].field[2].datatype 1812054936
+datatype[10].structtype[0].field[2].id[0]
+datatype[10].structtype[0].field[3].name "place"
+datatype[10].structtype[0].field[3].datatype 692270031
+datatype[10].structtype[0].field[3].id[0]
+datatype[10].structtype[0].inherits[0]
+datatype[10].documenttype[0]
+datatype[10].annotationreftype[0]
+annotationtype[5]
+annotationtype[0].name "person"
+annotationtype[0].id 609952424
+annotationtype[0].datatype -1466283082
+annotationtype[1].name "event"
+annotationtype[1].id -455530995
+annotationtype[1].datatype 1194300957
+annotationtype[2].name "artist"
+annotationtype[2].id 690330276
+annotationtype[2].datatype 1157126952
+annotationtype[3].name "date"
+annotationtype[3].id -162455681
+annotationtype[3].datatype -840345201
+annotationtype[4].name "place"
+annotationtype[4].id 1707984040
+annotationtype[4].datatype 2076579146
diff --git a/document/src/test/java/com/yahoo/document/datatypes/ArrayTestCase.java b/document/src/test/java/com/yahoo/document/datatypes/ArrayTestCase.java
new file mode 100755
index 00000000000..a2f0581f003
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/datatypes/ArrayTestCase.java
@@ -0,0 +1,258 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.datatypes;
+
+import com.yahoo.document.ArrayDataType;
+import com.yahoo.document.DataType;
+
+import java.util.*;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class ArrayTestCase extends junit.framework.TestCase {
+    public void testToArray() {
+        ArrayDataType dt = new ArrayDataType(DataType.STRING);
+        Array<StringFieldValue> arr = new Array<>(dt);
+        arr.add(new StringFieldValue("a"));
+        arr.add(new StringFieldValue("b"));
+        arr.add(new StringFieldValue("c"));
+        StringFieldValue[] tooSmall = new StringFieldValue[0];
+        StringFieldValue[] bigEnough = new StringFieldValue[3];
+        StringFieldValue[] a = arr.toArray(tooSmall);
+        assertNotSame(tooSmall, a);
+        assertEquals(new StringFieldValue("a"), a[0]);
+        assertEquals(new StringFieldValue("b"), a[1]);
+        assertEquals(new StringFieldValue("c"), a[2]);
+        StringFieldValue[] b = arr.toArray(bigEnough);
+        assertSame(bigEnough, b);
+        assertEquals(new StringFieldValue("a"), b[0]);
+        assertEquals(new StringFieldValue("b"), b[1]);
+        assertEquals(new StringFieldValue("c"), b[2]);
+    }
+
+    public void testCreateIllegalArray() {
+        ArrayList<FieldValue> arrayList = new ArrayList<>();
+        arrayList.add(new StringFieldValue("foo"));
+        arrayList.add(new IntegerFieldValue(1000));
+        DataType stringType = new ArrayDataType(DataType.STRING);
+        try {
+            Array<FieldValue> illegalArray = new Array<>(stringType, arrayList);
+            fail("Expected an exception");
+        } catch (IllegalArgumentException e) {
+            assertEquals("FieldValue 1000 is not compatible with datatype "
+                         + "Array<string> (code: -1486737430).",
+                         e.getMessage());
+        }
+
+        DataType intType = new ArrayDataType(DataType.INT);
+        Array<IntegerFieldValue> intArray = new Array<>(intType);
+        intArray.add(new IntegerFieldValue(42));
+        Array<StringFieldValue> stringArray = new Array<>(stringType);
+        try {
+            stringArray.assign(intArray);
+            fail("Expected an exception");
+        } catch (IllegalArgumentException e) {
+            assertEquals("Incompatible data types. Got datatype int (code: 0),"
+                         + " expected datatype string (code: 2)",
+                         e.getMessage());
+        }
+    }
+
+    public void testWrappedList() {
+        Array<StringFieldValue> array = new Array<StringFieldValue>(DataType.getArray(DataType.STRING));
+        List<String> list = new ArrayList<>();
+        list.add("foo");
+        list.add("bar");
+        list.add("baz");
+
+        array.assign(list);
+
+        assertEquals(3, array.size());
+        assertEquals(3, list.size());
+        assertFalse(array.isEmpty());
+        assertFalse(list.isEmpty());
+        assertEquals("foo", list.get(0));
+        assertEquals("bar", list.get(1));
+        assertEquals("baz", list.get(2));
+        assertEquals(new StringFieldValue("foo"), array.get(0));
+        assertEquals(new StringFieldValue("bar"), array.get(1));
+        assertEquals(new StringFieldValue("baz"), array.get(2));
+
+        array.remove(2);
+
+        assertEquals(2, array.size());
+        assertEquals(2, list.size());
+        assertEquals("foo", list.get(0));
+        assertEquals("bar", list.get(1));
+        assertEquals(new StringFieldValue("foo"), array.get(0));
+        assertEquals(new StringFieldValue("bar"), array.get(1));
+
+        list.remove(1);
+
+        assertEquals(1, array.size());
+        assertEquals(1, list.size());
+        assertEquals("foo", list.get(0));
+        assertEquals(new StringFieldValue("foo"), array.get(0));
+
+
+        list.add("bar");
+
+        assertEquals(2, array.size());
+        assertEquals(2, list.size());
+        assertEquals("foo", list.get(0));
+        assertEquals("bar", list.get(1));
+        assertEquals(new StringFieldValue("foo"), array.get(0));
+        assertEquals(new StringFieldValue("bar"), array.get(1));
+
+        array.add(new StringFieldValue("baz"));
+
+        assertEquals(3, array.size());
+        assertEquals(3, list.size());
+        assertEquals("foo", list.get(0));
+        assertEquals("bar", list.get(1));
+        assertEquals("baz", list.get(2));
+        assertEquals(new StringFieldValue("foo"), array.get(0));
+        assertEquals(new StringFieldValue("bar"), array.get(1));
+        assertEquals(new StringFieldValue("baz"), array.get(2));
+
+        assertTrue(array.contains(new StringFieldValue("foo")));
+        assertTrue(list.contains("foo"));
+
+        list.add("foo");
+
+        assertEquals(0, list.indexOf("foo"));
+        assertEquals(0, array.indexOf(new StringFieldValue("foo")));
+        assertEquals(3, list.lastIndexOf("foo"));
+        assertEquals(3, array.lastIndexOf(new StringFieldValue("foo")));
+
+        list.set(3, "banana");
+
+        assertEquals(4, array.size());
+        assertEquals(4, list.size());
+        assertEquals("foo", list.get(0));
+        assertEquals("bar", list.get(1));
+        assertEquals("baz", list.get(2));
+        assertEquals("banana", list.get(3));
+        assertEquals(new StringFieldValue("foo"), array.get(0));
+        assertEquals(new StringFieldValue("bar"), array.get(1));
+        assertEquals(new StringFieldValue("baz"), array.get(2));
+        assertEquals(new StringFieldValue("banana"), array.get(3));
+
+        array.set(3, new StringFieldValue("apple"));
+
+        assertEquals(4, array.size());
+        assertEquals(4, list.size());
+        assertEquals("foo", list.get(0));
+        assertEquals("bar", list.get(1));
+        assertEquals("baz", list.get(2));
+        assertEquals("apple", list.get(3));
+        assertEquals(new StringFieldValue("foo"), array.get(0));
+        assertEquals(new StringFieldValue("bar"), array.get(1));
+        assertEquals(new StringFieldValue("baz"), array.get(2));
+        assertEquals(new StringFieldValue("apple"), array.get(3));
+
+        list.remove("bar");
+
+        assertEquals(3, array.size());
+        assertEquals(3, list.size());
+        assertEquals("foo", list.get(0));
+        assertEquals("baz", list.get(1));
+        assertEquals("apple", list.get(2));
+        assertEquals(new StringFieldValue("foo"), array.get(0));
+        assertEquals(new StringFieldValue("baz"), array.get(1));
+        assertEquals(new StringFieldValue("apple"), array.get(2));
+
+        array.remove(new StringFieldValue("baz"));
+
+        assertEquals(2, array.size());
+        assertEquals(2, list.size());
+        assertEquals("foo", list.get(0));
+        assertEquals("apple", list.get(1));
+        assertEquals(new StringFieldValue("foo"), array.get(0));
+        assertEquals(new StringFieldValue("apple"), array.get(1));
+
+        assertNotNull(array.toArray(new StringFieldValue[5]));
+
+        try {
+            array.retainAll(new ArrayList<StringFieldValue>());
+            fail("Not implemented yet.");
+        } catch (UnsupportedOperationException uoe) {
+            //ok!
+        }
+
+        array.add(1, new StringFieldValue("boo"));
+
+        assertEquals(3, array.size());
+        assertEquals(3, list.size());
+        assertEquals("foo", list.get(0));
+        assertEquals("boo", list.get(1));
+        assertEquals("apple", list.get(2));
+        assertEquals(new StringFieldValue("foo"), array.get(0));
+        assertEquals(new StringFieldValue("boo"), array.get(1));
+        assertEquals(new StringFieldValue("apple"), array.get(2));
+
+        array.toString();
+
+        List<StringFieldValue> subArray = array.subList(1, 3);
+        assertEquals(2, subArray.size());
+        assertEquals(new StringFieldValue("boo"), subArray.get(0));
+        assertEquals(new StringFieldValue("apple"), subArray.get(1));
+
+
+        assertEquals(false, array.containsAll(Arrays.<StringFieldValue>asList(new StringFieldValue("bob"))));
+        assertEquals(true, array.containsAll(Arrays.<StringFieldValue>asList(new StringFieldValue("foo"), new StringFieldValue("boo"), new StringFieldValue("apple"))));
+
+        array.removeAll(Arrays.<StringFieldValue>asList(new StringFieldValue("foo"), new StringFieldValue("boo")));
+
+        assertEquals(1, array.size());
+        assertEquals(1, list.size());
+        assertEquals("apple", list.get(0));
+        assertEquals(new StringFieldValue("apple"), array.get(0));
+
+        array.add(new StringFieldValue("ibm"));
+
+        assertEquals(2, array.size());
+        assertEquals(2, list.size());
+
+        {
+            Iterator<StringFieldValue> it = array.iterator();
+            assertTrue(it.hasNext());
+            assertEquals(new StringFieldValue("apple"), it.next());
+            assertTrue(it.hasNext());
+            assertEquals(new StringFieldValue("ibm"), it.next());
+            assertFalse(it.hasNext());
+        }
+        {
+            ListIterator<StringFieldValue> it = array.listIterator();
+            assertTrue(it.hasNext());
+            assertEquals(new StringFieldValue("apple"), it.next());
+            assertTrue(it.hasNext());
+            assertEquals(new StringFieldValue("ibm"), it.next());
+            assertFalse(it.hasNext());
+        }
+
+        array.addAll(Arrays.<StringFieldValue>asList(new StringFieldValue("microsoft"), new StringFieldValue("google")));
+
+        assertEquals(4, array.size());
+        assertEquals(4, list.size());
+
+        array.clear();
+
+        assertEquals(0, array.size());
+        assertEquals(0, list.size());
+
+    }
+
+    public void testListWrapperToArray() {
+        Array<StringFieldValue> array = new Array<>(new ArrayDataType(DataType.STRING));
+        List<StringFieldValue> assignFrom = new ArrayList<>(3);
+        assignFrom.add(new StringFieldValue("a"));
+        assignFrom.add(new StringFieldValue("b"));
+        assignFrom.add(new StringFieldValue("c"));
+        array.assign(assignFrom);
+        final StringFieldValue[] expected = new StringFieldValue[] { new StringFieldValue("a"), new StringFieldValue("b"),
+                new StringFieldValue("c") };
+        assertTrue(Arrays.equals(expected, array.toArray(new StringFieldValue[0])));
+    }
+
+}
diff --git a/document/src/test/java/com/yahoo/document/datatypes/MapTestCase.java b/document/src/test/java/com/yahoo/document/datatypes/MapTestCase.java
new file mode 100644
index 00000000000..0b15ccf9854
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/datatypes/MapTestCase.java
@@ -0,0 +1,166 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.datatypes;
+
+import java.util.Map;
+
+import com.yahoo.document.ArrayDataType;
+import com.yahoo.document.DataType;
+import com.yahoo.document.DocumentTypeManager;
+import com.yahoo.document.Field;
+import com.yahoo.document.MapDataType;
+import com.yahoo.document.StructDataType;
+import com.yahoo.document.serialization.*;
+import com.yahoo.io.GrowableByteBuffer;
+
+
+public class MapTestCase extends junit.framework.TestCase {
+
+    public void testStringMap() {
+        MapDataType mapType = new MapDataType(DataType.STRING, DataType.STRING);
+        MapFieldValue<StringFieldValue, StringFieldValue> map = mapType.createFieldValue();
+        StringFieldValue sfvk1=new StringFieldValue("k1");
+        StringFieldValue sfvk2=new StringFieldValue("k2");
+        StringFieldValue sfvk3=new StringFieldValue("k3");
+        StringFieldValue sfvv1=new StringFieldValue("v1");
+        StringFieldValue sfvv2=new StringFieldValue("v2");
+        StringFieldValue sfvv3=new StringFieldValue("v3");
+        map.put(sfvk1, sfvv1);
+        map.put(sfvk2, sfvv2);
+        map.put(sfvk3, sfvv3);
+        assertEquals(map.get(sfvk1), sfvv1);
+        assertEquals(map.get(sfvk2), sfvv2);
+        assertEquals(map.get(sfvk3), sfvv3);
+        assertEquals(map.get(new StringFieldValue("k1")).getWrappedValue(), "v1");
+        assertEquals(map.get(new StringFieldValue("k2")).getWrappedValue(), "v2");
+        assertEquals(map.get(new StringFieldValue("k3")).getWrappedValue(), "v3");
+    }
+
+    public void testAdvancedMap() {
+        MapDataType stringMapType1 = new MapDataType(DataType.STRING, DataType.STRING);
+        MapDataType stringMapType2 = new MapDataType(DataType.STRING, DataType.STRING);
+        MapFieldValue sm1 = stringMapType1.createFieldValue();
+        MapFieldValue sm2 = stringMapType2.createFieldValue();
+        StringFieldValue e = new StringFieldValue("e");
+        StringFieldValue g = new StringFieldValue("g");
+        sm1.put(new StringFieldValue("a"), new StringFieldValue("b"));
+        sm1.put(new StringFieldValue("c"), new StringFieldValue("d"));
+        sm2.put(e, new StringFieldValue("f"));
+        sm2.put(g, new StringFieldValue("h"));
+
+        StructDataType structType= new StructDataType("teststr");
+        structType.addField(new Field("int", DataType.INT));
+        structType.addField(new Field("flt", DataType.FLOAT));
+        Struct s = structType.createFieldValue();
+        s.setFieldValue("int", 99);
+        s.setFieldValue("flt", -89.345);
+
+        ArrayDataType twoDimArray = DataType.getArray(DataType.getArray(DataType.FLOAT));
+        Array tda = twoDimArray.createFieldValue();
+
+        MapDataType floatToTwoDimArray = new MapDataType(DataType.FLOAT, twoDimArray);
+        MapDataType stringToStruct = new MapDataType(DataType.STRING, structType);
+        MapDataType stringMapToStringMap = new MapDataType(stringMapType1, stringMapType2);
+
+        MapFieldValue f2tda = floatToTwoDimArray.createFieldValue();
+        f2tda.put(new FloatFieldValue(3.4f), tda);
+
+        MapFieldValue s2sct = stringToStruct.createFieldValue();
+        s2sct.put(new StringFieldValue("s1"), s);
+        MapFieldValue sm2sm = stringMapToStringMap.createFieldValue();
+        sm2sm.put(sm1, sm2);
+
+        assertEquals(f2tda.get(new FloatFieldValue(3.4f)), tda);
+        assertEquals(new IntegerFieldValue(99), ((Struct) (s2sct.get(new StringFieldValue("s1")))).getFieldValue("int"));
+        assertEquals(new StringFieldValue("f"), ((MapFieldValue)(sm2sm.get(sm1))).get(e));
+        assertEquals(new StringFieldValue("h"), ((MapFieldValue)(sm2sm.get(sm1))).get(g));
+
+        // Look up using different map w same contents
+        // TODO it works even if sm1_2 is empty, something with class id?
+        MapFieldValue sm1_2 = stringMapType1.createFieldValue();
+        sm1_2.put(new StringFieldValue("a"), new StringFieldValue("b"));
+        sm1_2.put(new StringFieldValue("c"), new StringFieldValue("d"));
+        assertEquals(new StringFieldValue ("f"), ((MapFieldValue)(sm2sm.get(sm1_2))).get(e));
+        assertEquals(new StringFieldValue("h"), ((MapFieldValue)(sm2sm.get(sm1_2))).get(g));
+
+    }
+
+    public void testSerializationStringMap() {
+        MapDataType mapType = new MapDataType(DataType.STRING, DataType.STRING);
+        MapFieldValue<StringFieldValue, StringFieldValue> map = mapType.createFieldValue();
+        //Field f = new Field("stringmap",mapType);
+        StringFieldValue sfvk1=new StringFieldValue("k1");
+        StringFieldValue sfvk2=new StringFieldValue("k2");
+        StringFieldValue sfvk3=new StringFieldValue("k3");
+        StringFieldValue sfvv1=new StringFieldValue("v1");
+        StringFieldValue sfvv2=new StringFieldValue("v2");
+        StringFieldValue sfvv3=new StringFieldValue("v3");
+        map.put(sfvk1, sfvv1);
+        map.put(sfvk2, sfvv2);
+        map.put(sfvk3, sfvv3);
+        assertCorrectSerialization(mapType, map);
+    }
+
+    public void testSerializationComplex() {
+        ArrayDataType twoDimArray = DataType.getArray(DataType.getArray(DataType.FLOAT));
+        MapDataType floatToTwoDimArray = new MapDataType(DataType.FLOAT, twoDimArray);
+        MapFieldValue<FloatFieldValue, Array<Array<FloatFieldValue>>> map = floatToTwoDimArray.createFieldValue();
+
+        Array<FloatFieldValue> af1 = new Array(DataType.getArray(DataType.FLOAT));
+        af1.add(new FloatFieldValue(11f));
+        af1.add(new FloatFieldValue(111f));
+        Array<FloatFieldValue> af2 = new Array(DataType.getArray(DataType.FLOAT));
+        af2.add(new FloatFieldValue(22f));
+        af2.add(new FloatFieldValue(222f));
+        Array<Array<FloatFieldValue>> aaf1 = new Array(twoDimArray);
+        aaf1.add(af1);
+        aaf1.add(af2);
+
+        Array<FloatFieldValue> af3 = new Array(DataType.getArray(DataType.FLOAT));
+        af3.add(new FloatFieldValue(33f));
+        af3.add(new FloatFieldValue(333f));
+        Array<FloatFieldValue> af4 = new Array(DataType.getArray(DataType.FLOAT));
+        af4.add(new FloatFieldValue(44f));
+        af4.add(new FloatFieldValue(444f));
+        Array<Array<FloatFieldValue>> aaf2 = new Array(twoDimArray);
+        aaf2.add(af3);
+        aaf2.add(af4);
+
+        map.put(new FloatFieldValue(1.1f), aaf1);
+        map.put(new FloatFieldValue(2.2f), aaf2);
+        assertCorrectSerialization(floatToTwoDimArray, map);
+    }
+
+    private void assertCorrectSerialization(MapDataType mapType, MapFieldValue<? extends FieldValue, ? extends FieldValue> map) {
+        Field f = new Field("", mapType);
+        DocumentTypeManager man = new DocumentTypeManager();
+        man.register(mapType);
+        GrowableByteBuffer buffer = new GrowableByteBuffer(1024);
+        DocumentSerializer serializer = DocumentSerializerFactory.create42(buffer);
+        serializer.write(f, map);
+        buffer.flip();
+        DocumentDeserializer deserializer = DocumentDeserializerFactory.create42(man, buffer);
+        MapFieldValue<FieldValue, FieldValue> map2 = new MapFieldValue<FieldValue, FieldValue>(mapType);
+        deserializer.read(f, map2);
+        assertNotSame(map, map2);
+        for (Map.Entry<?,?> e : map.entrySet()) {
+            assertEquals(e.getValue(), map2.get(e.getKey()));
+        }
+    }
+
+    public void testIllegalMapAssignment() {
+        MapDataType type1 = new MapDataType(DataType.INT, DataType.INT);
+        MapDataType type2 = new MapDataType(DataType.STRING, DataType.STRING);
+        MapFieldValue map1 = new MapFieldValue(type1);
+        map1.put(new IntegerFieldValue(42), new IntegerFieldValue(84));
+        MapFieldValue map2 = new MapFieldValue(type2);
+        try {
+            map2.assign(map1);
+            fail("Expected an exception");
+        } catch (IllegalArgumentException e) {
+            assertEquals("Incompatible data types. Got datatype int (code: 0),"
+                         + " expected datatype string (code: 2)",
+                         e.getMessage());
+        }
+    }
+
+}
diff --git a/document/src/test/java/com/yahoo/document/datatypes/NumericFieldValueTestCase.java b/document/src/test/java/com/yahoo/document/datatypes/NumericFieldValueTestCase.java
new file mode 100644
index 00000000000..c7f88ac64e9
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/datatypes/NumericFieldValueTestCase.java
@@ -0,0 +1,19 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.datatypes;
+
+import org.junit.Test;
+import static org.junit.Assert.*;
+
+/**
+ * Created by balder on 8/6/15.
+ */
+public class NumericFieldValueTestCase {
+    @Test
+    public void requireThatConstructionFromStringWorks() {
+        assertEquals(67, new IntegerFieldValue("67").getInteger());
+        assertEquals(67, new LongFieldValue("67").getLong());
+        assertEquals(67, new ByteFieldValue("67").getByte());
+        assertEquals(67.45, new FloatFieldValue("67.45").getFloat(),    0.00001);
+        assertEquals(67.45, new DoubleFieldValue("67.45").getDouble(), 0.000001);
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/datatypes/PredicateFieldValueTest.java b/document/src/test/java/com/yahoo/document/datatypes/PredicateFieldValueTest.java
new file mode 100644
index 00000000000..b2b1fde0126
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/datatypes/PredicateFieldValueTest.java
@@ -0,0 +1,193 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.datatypes;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.Field;
+import com.yahoo.document.predicate.FeatureSet;
+import com.yahoo.document.predicate.SimplePredicates;
+import com.yahoo.document.predicate.Predicate;
+import com.yahoo.document.serialization.FieldReader;
+import com.yahoo.document.serialization.FieldWriter;
+import com.yahoo.document.serialization.XmlStream;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a>
+ */
+public class PredicateFieldValueTest {
+
+    @Test
+    public void requireThatFactoryProducesPredicate() {
+        assertEquals(PredicateFieldValue.class, PredicateFieldValue.getFactory().create().getClass());
+    }
+
+    @Test
+    public void requireThatAccessorsWork() {
+        PredicateFieldValue value = new PredicateFieldValue();
+        Predicate predicate = SimplePredicates.newPredicate();
+        value.setPredicate(predicate);
+        assertSame(predicate, value.getPredicate());
+    }
+
+    @Test
+    public void requireThatConstructorsWork() {
+        assertNull(new PredicateFieldValue().getPredicate());
+
+        Predicate predicate = SimplePredicates.newPredicate();
+        assertEquals(predicate, new PredicateFieldValue(predicate).getPredicate());
+    }
+
+    @Test
+    public void requireThatDataTypeIsPredicate() {
+        assertEquals(DataType.PREDICATE,
+                     new PredicateFieldValue().getDataType());
+    }
+
+    @Test
+    public void requireThatXmlOutputIsEmptyForNullPredicate() {
+        XmlStream expected = new XmlStream();
+        expected.beginTag("tag");
+        expected.endTag();
+        assertEquals(expected.toString(), printXml("tag", new PredicateFieldValue()));
+    }
+
+    @Test
+    public void requireThatXmlOutputIsPredicateLanguage() {
+        Predicate predicate = new FeatureSet("key", "valueA", "valueB");
+        XmlStream expected = new XmlStream();
+        expected.beginTag("tag");
+        expected.addContent(predicate.toString());
+        expected.endTag();
+        assertEquals(expected.toString(), printXml("tag", new PredicateFieldValue(predicate)));
+    }
+
+    @Test
+    public void requireThatClearNullsPredicate() {
+        PredicateFieldValue value = new PredicateFieldValue(SimplePredicates.newPredicate());
+        value.clear();
+        assertNull(value.getPredicate());
+    }
+
+    @Test
+    public void requireThatNullCanBeAssigned() {
+        PredicateFieldValue value = new PredicateFieldValue(SimplePredicates.newPredicate());
+        value.assign(null);
+        assertNull(value.getPredicate());
+    }
+
+    @Test
+    public void requireThatPredicateCanBeAssigned() {
+        Predicate predicate = SimplePredicates.newPredicate();
+        PredicateFieldValue value = new PredicateFieldValue();
+        value.assign(predicate);
+        assertSame(predicate, value.getPredicate());
+    }
+
+    @Test
+    public void requireThatPredicateFieldValueCanBeAssigned() {
+        Predicate predicate = SimplePredicates.newPredicate();
+        PredicateFieldValue value = new PredicateFieldValue();
+        value.assign(new PredicateFieldValue(predicate));
+        assertSame(predicate, value.getPredicate());
+    }
+
+    @Test
+    public void requireThatBadAssignThrows() {
+        try {
+            new PredicateFieldValue().assign(new Object());
+            fail();
+        } catch (IllegalArgumentException e) {
+            assertEquals("Expected com.yahoo.document.datatypes.PredicateFieldValue, got java.lang.Object.",
+                         e.getMessage());
+        }
+    }
+
+    @Test
+    public void requireThatSerializeCallsBackToWriter() {
+        Field field = Mockito.mock(Field.class);
+        FieldWriter writer = Mockito.mock(FieldWriter.class);
+        PredicateFieldValue value = new PredicateFieldValue(SimplePredicates.newPredicate());
+        value.serialize(field, writer);
+        Mockito.verify(writer).write(field, value);
+    }
+
+    @Test
+    public void requireThatDeserializeCallsBackToReader() {
+        Field field = Mockito.mock(Field.class);
+        FieldReader reader = Mockito.mock(FieldReader.class);
+        PredicateFieldValue value = new PredicateFieldValue(SimplePredicates.newPredicate());
+        value.deserialize(field, reader);
+        Mockito.verify(reader).read(field, value);
+    }
+
+    @Test
+    public void requireThatWrappedValueIsPredicate() {
+        Predicate predicate = SimplePredicates.newPredicate();
+        PredicateFieldValue value = new PredicateFieldValue(predicate);
+        assertSame(predicate, value.getWrappedValue());
+    }
+
+    @Test
+    public void requireThatCloneIsImplemented() {
+        PredicateFieldValue value1 = new PredicateFieldValue();
+        PredicateFieldValue value2 = value1.clone();
+        assertEquals(value1, value2);
+        assertNotSame(value1, value2);
+
+        value1 = new PredicateFieldValue(SimplePredicates.newString("foo"));
+        value2 = value1.clone();
+        assertEquals(value1, value2);
+        assertNotSame(value1, value2);
+        assertNotSame(value1.getPredicate(), value2.getPredicate());
+    }
+
+    @Test
+    public void requireThatHashCodeIsImplemented() {
+        assertEquals(new PredicateFieldValue().hashCode(),
+                     new PredicateFieldValue().hashCode());
+    }
+
+    @Test
+    public void requireThatEqualsIsImplemented() {
+        // null predicate in lhs
+        PredicateFieldValue lhs = new PredicateFieldValue();
+        assertTrue(lhs.equals(lhs));
+        assertFalse(lhs.equals(new Object()));
+
+        PredicateFieldValue rhs = new PredicateFieldValue(SimplePredicates.newString("bar"));
+        assertFalse(lhs.equals(rhs));
+        rhs.setPredicate(null);
+        assertTrue(lhs.equals(rhs));
+
+        // non-null predicate in lhs
+        lhs = new PredicateFieldValue(SimplePredicates.newString("foo"));
+        assertTrue(lhs.equals(lhs));
+        assertFalse(lhs.equals(new Object()));
+
+        rhs = new PredicateFieldValue();
+        assertFalse(lhs.equals(rhs));
+        rhs.setPredicate(SimplePredicates.newString("bar"));
+        assertFalse(lhs.equals(rhs));
+        rhs.setPredicate(SimplePredicates.newString("foo"));
+        assertTrue(lhs.equals(rhs));
+    }
+
+    private static String printXml(String tag, FieldValue value) {
+        XmlStream out = new XmlStream();
+        out.beginTag(tag);
+        if (value != null) {
+            value.printXml(out);
+        }
+        out.endTag();
+        return out.toString();
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/datatypes/RawTestCase.java b/document/src/test/java/com/yahoo/document/datatypes/RawTestCase.java
new file mode 100644
index 00000000000..2c76d5ec547
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/datatypes/RawTestCase.java
@@ -0,0 +1,32 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.datatypes;
+
+import org.junit.Test;
+
+import java.nio.ByteBuffer;
+
+import static org.junit.Assert.*;
+
+/**
+ * Created by balder on 10/21/14.
+ */
+public class RawTestCase {
+    @Test
+    public void requireThatAssignWorks() {
+        byte [] buf = {0,1,2,3};
+        byte [] empty = {};
+        ByteBuffer bb = ByteBuffer.wrap(buf);
+        Raw a = new Raw();
+        a.assign(buf);
+        assertEquals(bb, a.getWrappedValue());
+        a.assign(empty);
+        assertNotEquals(bb, a.getWrappedValue());
+        a.assign(bb);
+        assertEquals(bb, a.getWrappedValue());
+        a.assign(empty);
+        assertNotEquals(bb, a.getWrappedValue());
+        a.assign(new Raw(bb));
+        assertEquals(bb, a.getWrappedValue());
+    }
+
+}
diff --git a/document/src/test/java/com/yahoo/document/datatypes/StringFieldValueTestCase.java b/document/src/test/java/com/yahoo/document/datatypes/StringFieldValueTestCase.java
new file mode 100644
index 00000000000..8a0bfe68e72
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/datatypes/StringFieldValueTestCase.java
@@ -0,0 +1,393 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.datatypes;
+
+import org.junit.Test;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ * @since 5.1.14
+ */
+public class StringFieldValueTestCase {
+
+    @Test
+    public void requireThatCharWorks() {
+        new StringFieldValue("\t");
+        new StringFieldValue("\r");
+        new StringFieldValue("\n");
+        for (int c = 0x20; c < 0xFDD0; c++) {
+            new StringFieldValue("" + Character.toChars(c)[0]);
+        }
+        for (int c = 0xFDE0; c < 0xFFFF; c++) {
+            new StringFieldValue("" + Character.toChars(c)[0]);
+        }
+        for (int c = 0x10000; c < 0x1FFFE; c++) {
+            char[] chars = Character.toChars(c);
+            new StringFieldValue("" + chars[0] + chars[1]);
+        }
+        for (int c = 0x20000; c < 0x2FFFE; c++) {
+            char[] chars = Character.toChars(c);
+            new StringFieldValue("" + chars[0] + chars[1]);
+        }
+        for (int c = 0x30000; c < 0x3FFFE; c++) {
+            char[] chars = Character.toChars(c);
+            new StringFieldValue("" + chars[0] + chars[1]);
+        }
+        for (int c = 0x40000; c < 0x4FFFE; c++) {
+            char[] chars = Character.toChars(c);
+            new StringFieldValue("" + chars[0] + chars[1]);
+        }
+        for (int c = 0x50000; c < 0x5FFFE; c++) {
+            char[] chars = Character.toChars(c);
+            new StringFieldValue("" + chars[0] + chars[1]);
+        }
+        for (int c = 0x60000; c < 0x6FFFE; c++) {
+            char[] chars = Character.toChars(c);
+            new StringFieldValue("" + chars[0] + chars[1]);
+        }
+        for (int c = 0x70000; c < 0x7FFFE; c++) {
+            char[] chars = Character.toChars(c);
+            new StringFieldValue("" + chars[0] + chars[1]);
+        }
+        for (int c = 0x80000; c < 0x8FFFE; c++) {
+            char[] chars = Character.toChars(c);
+            new StringFieldValue("" + chars[0] + chars[1]);
+        }
+        for (int c = 0x90000; c < 0x9FFFE; c++) {
+            char[] chars = Character.toChars(c);
+            new StringFieldValue("" + chars[0] + chars[1]);
+        }
+        for (int c = 0xA0000; c < 0xAFFFE; c++) {
+            char[] chars = Character.toChars(c);
+            new StringFieldValue("" + chars[0] + chars[1]);
+        }
+        for (int c = 0xB0000; c < 0xBFFFE; c++) {
+            char[] chars = Character.toChars(c);
+            new StringFieldValue("" + chars[0] + chars[1]);
+        }
+        for (int c = 0xC0000; c < 0xCFFFE; c++) {
+            char[] chars = Character.toChars(c);
+            new StringFieldValue("" + chars[0] + chars[1]);
+        }
+        for (int c = 0xD0000; c < 0xDFFFE; c++) {
+            char[] chars = Character.toChars(c);
+            new StringFieldValue("" + chars[0] + chars[1]);
+        }
+        for (int c = 0xE0000; c < 0xEFFFE; c++) {
+            char[] chars = Character.toChars(c);
+            new StringFieldValue("" + chars[0] + chars[1]);
+        }
+        for (int c = 0xF0000; c < 0xFFFFE; c++) {
+            char[] chars = Character.toChars(c);
+            new StringFieldValue("" + chars[0] + chars[1]);
+        }
+        for (int c = 0x100000; c < 0x10FFFE; c++) {
+            char[] chars = Character.toChars(c);
+            new StringFieldValue("" + chars[0] + chars[1]);
+        }
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails0() {
+        new StringFieldValue("\u0000");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails1() {
+        new StringFieldValue("\u0001");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails2() {
+        new StringFieldValue("\u0002");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails3() {
+        new StringFieldValue("\u0003");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails4() {
+        new StringFieldValue("\u0004");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails5() {
+        new StringFieldValue("\u0005");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails6() {
+        new StringFieldValue("\u0006");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails7() {
+        new StringFieldValue("\u0007");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFailsB() {
+        new StringFieldValue("\u000B");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFailsC() {
+        new StringFieldValue("\u000C");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFailsE() {
+        new StringFieldValue("\u000E");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFailsF() {
+        new StringFieldValue("\u000F");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails10() {
+        new StringFieldValue("\u0010");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails11() {
+        new StringFieldValue("\u0011");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails12() {
+        new StringFieldValue("\u0012");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails13() {
+        new StringFieldValue("\u0013");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails14() {
+        new StringFieldValue("\u0014");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails15() {
+        new StringFieldValue("\u0015");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails16() {
+        new StringFieldValue("\u0016");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails17() {
+        new StringFieldValue("\u0017");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails18() {
+        new StringFieldValue("\u0018");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails19() {
+        new StringFieldValue("\u0019");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails1A() {
+        new StringFieldValue("\u001A");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails1B() {
+        new StringFieldValue("\u001B");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails1C() {
+        new StringFieldValue("\u001C");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails1D() {
+        new StringFieldValue("\u001D");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails1E() {
+        new StringFieldValue("\u001E");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails1F() {
+        new StringFieldValue("\u001F");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFailsFDD0() {
+        new StringFieldValue("\uFDD0");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFailsFDD1() {
+        new StringFieldValue("\uFDD1");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFailsFDD2() {
+        new StringFieldValue("\uFDD2");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFailsFDD3() {
+        new StringFieldValue("\uFDD3");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFailsFDD4() {
+        new StringFieldValue("\uFDD4");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFailsFDD5() {
+        new StringFieldValue("\uFDD5");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFailsFDD6() {
+        new StringFieldValue("\uFDD6");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFailsFDD7() {
+        new StringFieldValue("\uFDD7");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFailsFDD8() {
+        new StringFieldValue("\uFDD8");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFailsFDD9() {
+        new StringFieldValue("\uFDD9");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFailsFDDA() {
+        new StringFieldValue("\uFDDA");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFailsFDDB() {
+        new StringFieldValue("\uFDDB");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFailsFDDC() {
+        new StringFieldValue("\uFDDC");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFailsFDDD() {
+        new StringFieldValue("\uFDDD");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFailsFDDE() {
+        new StringFieldValue("\uFDDE");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFailsFDDF() {
+        new StringFieldValue("\uFDDF");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails1FFFE() {
+        new StringFieldValue("\uD83F\uDFFE");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails1FFFF() {
+        new StringFieldValue("\uD83F\uDFFF");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails2FFFE() {
+        new StringFieldValue("\uD87F\uDFFE");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails2FFFF() {
+        new StringFieldValue("\uD87F\uDFFF");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails3FFFE() {
+        new StringFieldValue("\uD8BF\uDFFE");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails3FFFF() {
+        new StringFieldValue("\uD8BF\uDFFF");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails4FFFE() {
+        new StringFieldValue("\uD8FF\uDFFE");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails4FFFF() {
+        new StringFieldValue("\uD8FF\uDFFF");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails5FFFE() {
+        new StringFieldValue("\uD93F\uDFFE");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails5FFFF() {
+        new StringFieldValue("\uD93F\uDFFF");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails6FFFE() {
+        new StringFieldValue("\uD97F\uDFFE");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails6FFFF() {
+        new StringFieldValue("\uD97F\uDFFF");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails7FFFE() {
+        new StringFieldValue("\uD9BF\uDFFE");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails7FFFF() {
+        new StringFieldValue("\uD9BF\uDFFF");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails8FFFE() {
+        new StringFieldValue("\uD9FF\uDFFE");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails8FFFF() {
+        new StringFieldValue("\uD9FF\uDFFF");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails9FFFE() {
+        new StringFieldValue("\uDA3F\uDFFE");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails9FFFF() {
+        new StringFieldValue("\uDA3F\uDFFF");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFailsAFFFE() {
+        new StringFieldValue("\uDA7F\uDFFE");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFailsAFFFF() {
+        new StringFieldValue("\uDA7F\uDFFF");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFailsBFFFE() {
+        new StringFieldValue("\uDABF\uDFFE");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFailsBFFFF() {
+        new StringFieldValue("\uDABF\uDFFF");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFailsCFFFE() {
+        new StringFieldValue("\uDAFF\uDFFE");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFailsCFFFF() {
+        new StringFieldValue("\uDAFF\uDFFF");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFailsDFFFE() {
+        new StringFieldValue("\uDB3F\uDFFE");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFailsDFFFF() {
+        new StringFieldValue("\uDB3F\uDFFF");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFailsEFFFE() {
+        new StringFieldValue("\uDB7F\uDFFE");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFailsEFFFF() {
+        new StringFieldValue("\uDB7F\uDFFF");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFailsFFFFE() {
+        new StringFieldValue("\uDBBF\uDFFE");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFailsFFFFF() {
+        new StringFieldValue("\uDBBF\uDFFF");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails10FFFE() {
+        new StringFieldValue("\uDBFF\uDFFE");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails10FFFF() {
+        new StringFieldValue("\uDBFF\uDFFF");
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/datatypes/StringTestCase.java b/document/src/test/java/com/yahoo/document/datatypes/StringTestCase.java
new file mode 100644
index 00000000000..cca39ee5bc4
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/datatypes/StringTestCase.java
@@ -0,0 +1,434 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.datatypes;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.Document;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.DocumentTypeManager;
+import com.yahoo.document.DocumentTypeManagerConfigurer;
+import com.yahoo.document.Field;
+import com.yahoo.document.annotation.AbstractTypesTest;
+import com.yahoo.document.annotation.Annotation;
+import com.yahoo.document.annotation.AnnotationType;
+import com.yahoo.document.annotation.AnnotationTypeRegistry;
+import com.yahoo.document.annotation.Span;
+import com.yahoo.document.annotation.SpanList;
+import com.yahoo.document.annotation.SpanNode;
+import com.yahoo.document.annotation.SpanTree;
+import com.yahoo.document.serialization.*;
+import com.yahoo.io.GrowableByteBuffer;
+import com.yahoo.vespa.objects.BufferSerializer;
+import org.junit.Test;
+
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.ListIterator;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class StringTestCase extends AbstractTypesTest {
+
+    @Test
+    public void testDeserialize() throws Exception {
+        byte[] buf = new byte[1500];
+        GrowableByteBuffer data = GrowableByteBuffer.wrap(buf);
+        //short string
+        java.lang.String foo = "foo";
+
+        data.put((byte)0);
+        data.put((byte)(foo.length() + 1));
+
+        data.put(foo.getBytes());
+        data.put((byte)0);
+
+        int positionAfterPut = data.position();
+
+        data.position(0);
+
+        StringFieldValue tmp = new StringFieldValue();
+        DocumentDeserializer deser = DocumentDeserializerFactory.create42(null, data);
+        tmp.deserialize(deser);
+        java.lang.String foo2 = tmp.getString();
+
+        assertTrue(foo.equals(foo2));
+        assertEquals(data.position(), positionAfterPut);
+
+        //=====================
+
+        buf = new byte[1500];
+        data = GrowableByteBuffer.wrap(buf);
+
+        //long string
+        java.lang.String blah = "blahblahblahblahblahblahblahblahblahblahblah" +
+                                "blahblahblahblahblahblahblahblahblahblahblah" +
+                                "blahblahblahblahblahblahblahblahblahblahblah" +
+                                "blahblahblahblahblahblahblahblahblahblahblah" +
+                                "blahblahblahblahblahblahblahblahblahblahblah" +
+                                "blahblahblahblahblahblahblahblahblahblahblah" +
+                                "blahblahblahblahblahblahblahblahblahblahblah" +
+                                "blahblahblahblahblahblahblahblahblahblahblah" +
+                                "blahblahblahblahblahblahblahblahblahblahblah" +
+                                "blahblahblahblahblahblahblahblahblahblahblah" +
+                                "blahblahblahblahblahblahblahblahblahblahblah" +
+                                "blahblahblahblahblahblahblahblahblahblahblah" +
+                                "blahblahblahblahblahblahblahblahblahblahblah" +
+                                "blahblahblahblahblahblahblahblahblahblahblah" +
+                                "blahblahblahblahblahblahblahblahblahblahblah" +
+                                "blahblahblahblahblahblahblahblahblahblahblah" +
+                                "blahblahblahblahblahblahblahblahblahblahblah" +
+                                "blahblahblahblahblahblahblahblahblahblahblah" +
+                                "blahblahblahblahblahblahblahblahblahblahblah" +
+                                "blahblahblahblahblahblahblahblahblahblahblah" +
+                                "blahblahblahblahblahblahblahblahblahblahblah" +
+                                "blahblahblahblahblahblahblahblahblahblahblah" +
+                                "blahblahblahblahblahblahblahblahblahblahblah" +
+                                "blahblahblahblahblahblahblahblahblahblahblah" +
+                                "blahblahblahblahblahblahblahblahblahblahblah" +
+                                "blahblahblahblahblahblahblahblahblahblahblah" +
+                                "blahblahblahblahblahblahblahblahblahblahblah" +
+                                "blahblahblahblahblahblahblahblahblahblahblah" +
+                                "blahblahblahblahblahblahblahblahblahblahblah" +
+                                "blahblahblahblahblahblahblahblahblahblahblah" +
+                                "blahblahblahblahblahblahblahblahblahblahblah" +
+                                "blahblahblahblahblahblahblahblahblahblahblah" +
+                                "blahblahblahblahblahblahblahblahblahblahblah";
+
+        int length = blah.length() + 1;
+
+        data.put((byte)0);
+        data.putInt(length | 0x80000000);
+
+        data.put(blah.getBytes());
+        data.put((byte)0);
+
+        positionAfterPut = data.position();
+
+        data.position(0);
+
+        tmp = new StringFieldValue();
+
+        deser = DocumentDeserializerFactory.create42(null, data);
+        tmp.deserialize(deser);
+        java.lang.String blah2 = tmp.getString();
+
+        assertEquals(data.position(), positionAfterPut);
+        assertTrue(blah.equals(blah2));
+    }
+
+/*    public void testSpanTree() {
+        StringFieldValue annotatedText = new StringFieldValue("banana airlines");
+        SpanList lingTree = new SpanList();
+        annotatedText.setSpanTree("linguistics", lingTree);
+        for (Annotation anAnnotation : annotatedText.getSpanTree("linguistics")) {
+            System.err.println(anAnnotation);
+        }
+    }*/
+
+    @Test
+    public void testSerializeDeserialize() throws Exception {
+        java.lang.String test = "Hello hello";
+        BufferSerializer data = new BufferSerializer(new GrowableByteBuffer(100, 2.0f));
+        StringFieldValue value = new StringFieldValue(test);
+        value.serialize(data);
+
+        data.getBuf().position(0);
+
+        StringFieldValue tmp = new StringFieldValue();
+        DocumentDeserializer deser = DocumentDeserializerFactory.create42(null, data.getBuf());
+        tmp.deserialize(deser);
+        java.lang.String test2 = tmp.getString();
+        assertEquals(test, test2);
+    }
+
+    @Test
+    public void testSerializationWithTree() {
+        StringFieldValue text = getAnnotatedString();
+
+        serializeAndAssert(text);
+    }
+
+    private void serializeAndAssert(StringFieldValue stringFieldValue) {
+        Field f = new Field("text", DataType.STRING);
+
+        GrowableByteBuffer buffer = new GrowableByteBuffer(1024);
+        DocumentSerializer serializer = DocumentSerializerFactory.create42(buffer);
+        serializer.write(f, stringFieldValue);
+        buffer.flip();
+
+        DocumentDeserializer deserializer = DocumentDeserializerFactory.create42(man, buffer);
+        StringFieldValue stringFieldValue2 = new StringFieldValue();
+        deserializer.read(f, stringFieldValue2);
+
+        assertEquals(stringFieldValue, stringFieldValue2);
+        assertNotSame(stringFieldValue, stringFieldValue2);
+    }
+
+    @Test
+    public void testNestedSpanTreeBug4187377() {
+        AnnotationType type = new AnnotationType("ann", DataType.STRING);
+
+        StringFieldValue outerString = new StringFieldValue("Ballooo");
+        SpanTree outerTree = new SpanTree("outer");
+        outerString.setSpanTree(outerTree);
+
+        SpanList outerRoot = (SpanList)outerTree.getRoot();
+        Span outerSpan = new Span(0, 1);
+        outerRoot.add(outerSpan);
+
+        StringFieldValue innerString = new StringFieldValue("innerBalloooo");
+
+        outerTree.annotate(outerSpan, new Annotation(type, innerString));
+
+        SpanTree innerTree = new SpanTree("inner");
+        innerString.setSpanTree(innerTree);
+
+        SpanList innerRoot = (SpanList)innerTree.getRoot();
+        Span innerSpan = new Span(0, 1);
+        innerRoot.add(innerSpan);
+        innerTree.annotate(innerSpan, new Annotation(type));
+
+        GrowableByteBuffer buffer = new GrowableByteBuffer(1024);
+        DocumentSerializer serializer = DocumentSerializerFactory.create42(buffer);
+
+        try {
+            serializer.write(null, outerString);
+            fail("Should have failed, nested span trees are not supported.");
+        } catch (SerializationException se) {
+            //OK!
+        }
+    }
+
+    /**
+     * Test for bug 4066566. No assertions, but works if it runs without exceptions.
+     */
+    @Test
+    public void testAnnotatorConsumer() {
+        DocumentTypeManager manager = new DocumentTypeManager();
+        DocumentTypeManagerConfigurer
+                .configure(manager, "file:src/test/java/com/yahoo/document/datatypes/documentmanager.blog.sd");
+
+        DocumentType blogType = manager.getDocumentType("blog");
+        Document doc = new Document(blogType, "doc:blog:http://blogs.sun.com/praveenm");
+        doc.setFieldValue("url", new StringFieldValue("http://blogs.sun.com/praveenm"));
+        doc.setFieldValue("title", new StringFieldValue("Beginning JavaFX"));
+        doc.setFieldValue("author", new StringFieldValue("Praveen Mohan"));
+        doc.setFieldValue("body", new StringFieldValue(
+                "JavaFX can expand its wings across different domains such as manufacturing, logistics, retail, etc. Many companies have adopted it - IBM, Oracle, Yahoo, Honeywell. Even the non-IT industries such as GE, WIPRO, Ford etc. So it is a success for Christopher Oliver and Richard Bair. Scott Mcnealy is happy"));
+
+        doc = annotate(doc, manager);
+        doc = serializeAndDeserialize(doc, manager);
+        doc = consume(doc, manager);
+        System.err.println(doc);
+    }
+
+    private Document serializeAndDeserialize(Document doc, DocumentTypeManager manager) {
+        GrowableByteBuffer buffer = new GrowableByteBuffer(1024);
+        DocumentSerializer serializer = DocumentSerializerFactory.create42(buffer);
+        serializer.write(doc);
+        buffer.flip();
+
+        DocumentDeserializer deserializer = DocumentDeserializerFactory.create42(manager, buffer);
+        return new Document(deserializer);
+    }
+
+    public Document annotate(Document document, DocumentTypeManager manager) {
+        AnnotationTypeRegistry registry = manager.getAnnotationTypeRegistry();
+
+        AnnotationType company = registry.getType("company");
+        AnnotationType industry = registry.getType("industry");
+        AnnotationType person = registry.getType("person");
+        AnnotationType location = registry.getType("location");
+
+        Map<String, AnnotationType> m = registry.getTypes();
+        for (String key : m.keySet()) {
+            System.out.println("Key: " + key);
+            AnnotationType val = m.get(key);
+            parseAnnotationType(val);
+        }
+
+        SpanTree tree = new SpanTree("testannotations");
+        SpanList root = (SpanList)tree.getRoot();
+
+        SpanNode companySpan = new Span(0, 5);
+        SpanNode industrySpan = new Span(5, 10);
+        SpanNode personSpan = new Span(10, 15);
+        SpanNode locationSpan = new Span(15, 20);
+
+        root.add(companySpan);
+        root.add(industrySpan);
+        root.add(personSpan);
+        root.add(locationSpan);
+
+        Struct companyValue = (Struct)company.getDataType().createFieldValue();
+        companyValue.setFieldValue("name", new StringFieldValue("Sun"));
+        companyValue.setFieldValue("ceo", new StringFieldValue("Scott Mcnealy"));
+        companyValue.setFieldValue("lat", new DoubleFieldValue(37.7));
+        companyValue.setFieldValue("lon", new DoubleFieldValue(-122.44));
+        companyValue.setFieldValue("vertical", new StringFieldValue("software"));
+        Annotation compAn = new Annotation(company, companyValue);
+        tree.annotate(companySpan, compAn);
+
+        Struct personValue = new Struct(manager.getDataType("annotation.person"));
+        personValue.setFieldValue("name", new StringFieldValue("Richard Bair"));
+        Annotation personAn = new Annotation(person, personValue);
+        tree.annotate(personSpan, personAn);
+
+        Struct locValue = new Struct(manager.getDataType("annotation.location"));
+        locValue.setFieldValue("name", new StringFieldValue("Prinsens Gate"));
+        Annotation loc = new Annotation(location, locValue);
+        tree.annotate(locationSpan, loc);
+
+        Struct locValue2 = new Struct(manager.getDataType("annotation.location"));
+        locValue2.setFieldValue("name", new StringFieldValue("Kongens Gate"));
+        Annotation locAn = new Annotation(location, locValue2);
+        tree.annotate(locationSpan, locAn);
+
+        SpanList branch = new SpanList();
+
+        SpanNode span1 = new Span(0, 3);
+        SpanNode span2 = new Span(1, 9);
+        SpanNode span3 = new Span(12, 10);
+
+        branch.add(span1);
+        branch.add(span3);
+        branch.add(span2);
+
+        Struct industryValue = new Struct(manager.getDataType("annotation.industry"));
+        industryValue.setFieldValue("vertical", new StringFieldValue("Manufacturing"));
+        Annotation ind = new Annotation(industry, industryValue);
+        tree.annotate(span1, ind);
+
+        Struct pValue = new Struct(manager.getDataType("annotation.person"));
+        pValue.setFieldValue("name", new StringFieldValue("Praveen Mohan"));
+        Annotation pAn = new Annotation(person, pValue);
+        tree.annotate(span2, pAn);
+
+        Struct lValue = new Struct(manager.getDataType("annotation.location"));
+        lValue.setFieldValue("name", new StringFieldValue("Embassy Golf Links"));
+        Annotation locn = new Annotation(location, lValue);
+        tree.annotate(span3, locn);
+
+        Struct cValue = (Struct)company.getDataType().createFieldValue();
+        cValue.setFieldValue("name", new StringFieldValue("Yahoo"));
+        cValue.setFieldValue("ceo", new StringFieldValue("Carol Bartz"));
+        cValue.setFieldValue("lat", new DoubleFieldValue(127.7));
+        cValue.setFieldValue("lon", new DoubleFieldValue(-42.44));
+        cValue.setFieldValue("vertical", new StringFieldValue("search"));
+        Annotation cAn = new Annotation(company, cValue);
+        tree.annotate(branch, cAn);
+
+        Struct pVal = new Struct(manager.getDataType("annotation.person"));
+        pVal.setFieldValue("name", new StringFieldValue("Kim Omar"));
+        Annotation an = new Annotation(person, pVal);
+        tree.annotate(root, an);
+        root.add(branch);
+
+        StringFieldValue body = (StringFieldValue)document.getFieldValue(document.getDataType().getField("body"));
+
+        root.remove(branch);
+        tree.cleanup();
+
+        System.out.println("No. Of Annotations: " + tree.numAnnotations());
+        body.setSpanTree(tree);
+
+        document.setFieldValue(document.getField("body"), body);
+
+        return document;
+    }
+
+    public Document consume(Document document, DocumentTypeManager docTypeMgr) {
+        DocumentType type = docTypeMgr.getDocumentType("blog");
+        Collection<Field> fc = type.getFields();
+        for (Field f : fc) {
+            System.out.println("\n\nField Name: " + f.getName());
+            System.out.println("DataType: " + f.getDataType());
+            System.out.println("isHeader? " + f.isHeader());
+
+            FieldValue val = document.getFieldValue(f);
+            if (val instanceof StringFieldValue) {
+                StringFieldValue sfv = (StringFieldValue)val;
+                System.out.println(f.getName() + " is a StringField. Field Value: " + sfv.getString());
+                Collection<SpanTree> c = sfv.getSpanTrees();
+                for (SpanTree tree : c) {
+                    System.out.println(f.getName() + " has annotations");
+                    consumeAnnotations(tree, (SpanList)tree.getRoot());
+                }
+            }
+        }
+
+        return document;
+    }
+
+    public void consumeAnnotations(SpanTree tree, SpanList root) {
+        System.out.println("\n\nSpanList: " + root + " num Children: " + root.numChildren());
+        System.out.println("-------------------");
+        Iterator<SpanNode> childIterator = root.childIterator();
+        while (childIterator.hasNext()) {
+            SpanNode node = childIterator.next();
+            System.out.println("Span Node: " + node); // + " Span Text: " + node.getText(fieldValStr));
+            if (node instanceof SpanList) {
+                System.out.println("Encountered another span list");
+                SpanList spl = (SpanList)node;
+                ListIterator<SpanNode> lli = spl.childIterator();
+                while (lli.hasNext()) {
+                    System.out.print(" " + lli.next() + " ");
+                }
+                consumeAnnotations(tree, (SpanList)node);
+            } else {
+                System.out.println("\nGetting annotations for this span node: " + node);
+                getAnnotationsForNode(tree, node);
+            }
+        }
+        System.out.println("\nGetting annotations for the SpanList itself : " + root);
+        getAnnotationsForNode(tree, root);
+    }
+
+    public void getAnnotationsForNode(SpanTree tree, SpanNode node) {
+        Iterator<Annotation> iter = tree.iterator(node);
+        boolean annotationPresent = false;
+        while (iter.hasNext()) {
+            annotationPresent = true;
+            Annotation xx = iter.next();
+            Struct fValue = (Struct)xx.getFieldValue();
+            System.out.println("Annotation: " + xx);
+            if (fValue == null) {
+                System.out.println("Field Value is null");
+                return;
+            }
+            Iterator fieldIter = fValue.iterator();
+            while (fieldIter.hasNext()) {
+                Map.Entry m = (Map.Entry)fieldIter.next();
+                Field f = (Field)m.getKey();
+                FieldValue val = (FieldValue)m.getValue();
+                System.out.println("Field : " + f + " Value: " + val);
+            }
+        }
+        if (!annotationPresent) {
+            System.out.println("****No annotations found for the span node: " + node);
+        }
+    }
+
+    public void parseAnnotationType(AnnotationType t) {
+        System.out.println("Type Name: " + t.getName());
+        System.out.println("Type ID: " + t.getId());
+        DataType dt = t.getDataType();
+        String dataTypeStr;
+        if (dt == DataType.STRING) {
+            dataTypeStr = "String";
+        } else if (dt == DataType.INT) {
+            dataTypeStr = "Integer";
+        } else if (dt == DataType.URI) {
+            dataTypeStr = "URL";
+        } else {
+            dataTypeStr = "UNKNOWN";
+        }
+        System.out.println("Type DataType: " + dataTypeStr);
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/datatypes/StructTestCase.java b/document/src/test/java/com/yahoo/document/datatypes/StructTestCase.java
new file mode 100644
index 00000000000..76e1588b2a1
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/datatypes/StructTestCase.java
@@ -0,0 +1,356 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.datatypes;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.Field;
+import com.yahoo.document.StructDataType;
+
+import java.nio.ByteBuffer;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.junit.Test;
+import static org.junit.Assert.*;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class StructTestCase {
+    @Test
+    public void testBasicStuff() throws Exception {
+        StructDataType type = new StructDataType("teststr");
+        type.addField(new Field("int", 0, DataType.INT, true));
+        type.addField(new Field("flt", 1, DataType.FLOAT, true));
+        type.addField(new Field("str", 2, DataType.STRING, true));
+        type.addField(new Field("raw", 3, DataType.RAW, true));
+        type.addField(new Field("lng", 4, DataType.LONG, true));
+        type.addField(new Field("dbl", 5, DataType.DOUBLE, true));
+        type.addField(new Field("uri", 6, DataType.URI, true));
+        type.addField(new Field("byt", 8, DataType.BYTE, true));
+
+        Struct struct = new Struct(type);
+        {
+            //add and remove again:
+            assertEquals(0, struct.getFields().size());
+            IntegerFieldValue ifv = new IntegerFieldValue(5);
+            struct.setFieldValue("int", ifv);
+            assertEquals(1, struct.getFields().size());
+            struct.removeFieldValue("int");
+            assertEquals(0, struct.getFields().size());
+        }
+
+        {
+            //add three elements and remove one of them, and replace the last one:
+            assertEquals(0, struct.getFields().size());
+
+            IntegerFieldValue ifv = new IntegerFieldValue(5);
+            struct.setFieldValue("int", ifv);
+            assertEquals(1, struct.getFields().size());
+
+            FloatFieldValue ffv = new FloatFieldValue(5.0f);
+            struct.setFieldValue("flt", ffv);
+            assertEquals(2, struct.getFields().size());
+
+            DoubleFieldValue dfv = new DoubleFieldValue(6.0d);
+            struct.setFieldValue("dbl", dfv);
+            assertEquals(3, struct.getFields().size());
+
+            Iterator<Map.Entry<Field, FieldValue>> it = struct.iterator();
+            assertSame(ifv, it.next().getValue());
+            assertSame(ffv, it.next().getValue());
+            assertSame(dfv, it.next().getValue());
+            assertFalse(it.hasNext());
+
+            struct.removeFieldValue("flt");
+            assertEquals(2, struct.getFields().size());
+
+            it = struct.iterator();
+            assertSame(ifv, it.next().getValue());
+            assertSame(dfv, it.next().getValue());
+            assertFalse(it.hasNext());
+
+            DoubleFieldValue dfv2 = new DoubleFieldValue(9.0d);
+            struct.setFieldValue("dbl", dfv2);
+            assertEquals(2, struct.getFields().size());
+
+            it = struct.iterator();
+            assertSame(ifv, it.next().getValue());
+            assertSame(dfv2, it.next().getValue());
+            assertFalse(it.hasNext());
+        }
+    }
+
+    @Test
+    public void testSetGetPrimitiveTypes() throws Exception {
+        StructDataType type = new StructDataType("teststr");
+        type.addField(new Field("int", DataType.INT));
+        type.addField(new Field("flt", DataType.FLOAT));
+        type.addField(new Field("str", DataType.STRING));
+        type.addField(new Field("raw", DataType.RAW));
+        type.addField(new Field("lng", DataType.LONG));
+        type.addField(new Field("dbl", DataType.DOUBLE));
+        type.addField(new Field("uri", DataType.URI));
+        type.addField(new Field("byt", DataType.BYTE));
+
+        Struct struct = new Struct(type);
+
+        {
+            IntegerFieldValue nt = new IntegerFieldValue(544);
+            Object o = struct.setFieldValue("int", nt);
+            assertNull(o);
+            assertEquals(new IntegerFieldValue(544), struct.getFieldValue("int"));
+            o = struct.setFieldValue("int", 500);
+            assertEquals(nt, o);
+            assertFalse(nt.equals(struct.getFieldValue("int")));
+        }
+        {
+            FloatFieldValue flt = new FloatFieldValue(5.44f);
+            Object o = struct.setFieldValue("flt", flt);
+            assertNull(o);
+            assertEquals(flt, struct.getFieldValue("flt"));
+            o = struct.setFieldValue("flt", new FloatFieldValue(5.00f));
+            assertEquals(flt, o);
+            assertFalse(flt.equals(struct.getFieldValue("flt")));
+        }
+        {
+            StringFieldValue string = new StringFieldValue("this is a string");
+            Object o = struct.setFieldValue("str", string);
+            assertNull(o);
+            assertEquals(string, struct.getFieldValue("str"));
+            o = struct.setFieldValue("str", "another string");
+            assertEquals(string, o);
+            assertSame(string, o);
+            assertFalse(string.equals(struct.getFieldValue("str")));
+        }
+        {
+            Raw buf = new Raw(ByteBuffer.wrap(new byte[100]));
+            Object o = struct.setFieldValue("raw", buf);
+            assertNull(o);
+            assertEquals(buf, struct.getFieldValue("raw"));
+            o = struct.setFieldValue("raw", new Raw(ByteBuffer.wrap(new byte[50])));
+            assertEquals(buf, o);
+            assertSame(buf, o);
+            assertFalse(buf.equals(struct.getFieldValue("raw")));
+        }
+        {
+            LongFieldValue lng = new LongFieldValue(59879879879079L);
+            Object o = struct.setFieldValue("lng", lng);
+            assertEquals(lng, struct.getFieldValue("lng"));
+            o = struct.setFieldValue("lng", new LongFieldValue(23418798734243L));
+            assertEquals(lng, o);
+            assertFalse(lng.equals(struct.getFieldValue("lng")));
+        }
+        {
+            DoubleFieldValue dbl = new DoubleFieldValue(5.44d);
+            Object o = struct.setFieldValue("dbl", dbl);
+            assertNull(o);
+            assertEquals(dbl, struct.getFieldValue("dbl"));
+            o = struct.setFieldValue("dbl", new DoubleFieldValue(5.00d));
+            assertEquals(dbl, o);
+            assertFalse(dbl.equals(struct.getFieldValue("dbl")));
+        }
+        {
+            UriFieldValue uri = new UriFieldValue("this is a uri");
+            Object o = struct.setFieldValue("uri", uri);
+            assertNull(o);
+            assertEquals(uri, struct.getFieldValue("uri"));
+            o = struct.setFieldValue("uri", "another uri");
+            assertEquals(uri, o);
+            assertSame(uri, o);
+            assertFalse(uri.equals(struct.getFieldValue("uri")));
+        }
+        {
+            ByteFieldValue byt = new ByteFieldValue((byte)123);
+            Object o = struct.setFieldValue("byt", byt);
+            assertNull(o);
+            assertEquals(byt, struct.getFieldValue("byt"));
+            o = struct.setFieldValue("byt", (byte) 100);
+            assertEquals(byt, o);
+            assertFalse(byt.equals(struct.getFieldValue("byt")));
+        }
+    }
+
+    @Test
+    public void testSetGetAggregateTypes() throws Exception {
+        StructDataType type = new StructDataType("teststr");
+        type.addField(new Field("intarray", DataType.getArray(DataType.INT)));
+        type.addField(new Field("strws", DataType.getWeightedSet(DataType.STRING)));
+
+        Struct struct = new Struct(type);
+
+        {
+            //TEST USING OUR IMPLEMENTATION OF LIST
+            Array integerArray = new Array(type.getField("intarray").getDataType());
+            integerArray.add(new IntegerFieldValue(5));
+            integerArray.add(new IntegerFieldValue(10));
+            assertEquals(2, integerArray.size());
+            struct.setFieldValue("intarray", integerArray);
+            assertEquals(2, integerArray.size());
+            List outList = (List) struct.getFieldValue("intarray");
+            integerArray.add(new IntegerFieldValue(322));
+            integerArray.add(new IntegerFieldValue(453));
+            assertEquals(integerArray, outList);
+            assertSame(integerArray, outList);
+            assertEquals(4, integerArray.size());
+            Array anotherArray = new Array(type.getField("intarray").getDataType());
+            anotherArray.add(new IntegerFieldValue(5324));
+            Object o = struct.setFieldValue("intarray", anotherArray);
+            assertEquals(integerArray, o);
+            assertSame(integerArray, o);
+            outList = (List) struct.getFieldValue("intarray");
+            assertFalse(integerArray.equals(outList));
+            assertEquals(anotherArray, outList);
+            assertSame(anotherArray, outList);
+        }
+        {
+            WeightedSet<StringFieldValue> strWs = new WeightedSet<>(type.getField("strws").getDataType());
+            strWs.put(new StringFieldValue("banana"), 10);
+            strWs.add(new StringFieldValue("apple"));
+            assertEquals(2, strWs.size());
+            Object o = struct.setFieldValue("strws", strWs);
+            assertNull(o);
+            assertEquals(2, strWs.size());
+            WeightedSet<StringFieldValue> outWs = (WeightedSet<StringFieldValue>) struct.getFieldValue("strws");
+            strWs.add(new StringFieldValue("poison"));
+            strWs.put(new StringFieldValue("pie"), 599);
+            assertEquals(strWs, outWs);
+            assertSame(strWs, outWs);
+            assertEquals(4, strWs.size());
+            WeightedSet anotherWs = new WeightedSet(type.getField("strws").getDataType());
+            anotherWs.add(new StringFieldValue("be bop"));
+            o = struct.setFieldValue("strws", anotherWs);
+            assertEquals(strWs, o);
+            assertSame(strWs, o);
+            outWs = (WeightedSet<StringFieldValue>) struct.getFieldValue("strws");
+
+            System.out.println("OutWS " + outWs);
+            System.out.println("StrWS " + strWs);
+
+            assertFalse(strWs.equals(outWs));
+            assertEquals(anotherWs, outWs);
+            assertSame(anotherWs, outWs);
+        }
+    }
+
+    @Test
+    public void testSetUnknownType() {
+        StructDataType type = new StructDataType("teststr");
+        type.addField(new Field("int", 0, DataType.INT, true));
+
+        Struct struct = new Struct(type);
+        try {
+            struct.setFieldValue(new Field("alien", DataType.STRING), new StringFieldValue("foo"));
+            fail("Alien type worked");
+        } catch (IllegalArgumentException expected) {
+            assertTrue(expected.getMessage().matches(".*No such field.*"));
+        }
+    }
+
+    @Test
+    public void testCompareToDoesNotMutateStateBug6394548() {
+        StructDataType type = new StructDataType("test");
+        // NOTE: non-increasing ID order!
+        type.addField(new Field("int", 2, DataType.INT, true));
+        type.addField(new Field("flt", 1, DataType.FLOAT, true));
+        type.addField(new Field("str", 0, DataType.STRING, true));
+
+        Struct a = new Struct(type);
+        a.setFieldValue("int", new IntegerFieldValue(123));
+        a.setFieldValue("flt", new DoubleFieldValue(45.6));
+        a.setFieldValue("str", new StringFieldValue("hello world"));
+        Struct b = new Struct(type);
+        b.setFieldValue("int", new IntegerFieldValue(100));
+        b.setFieldValue("flt", new DoubleFieldValue(45.6));
+        b.setFieldValue("str", new StringFieldValue("hello world"));
+
+        String xmlBefore = a.toXml();
+        int hashBefore = a.hashCode();
+
+        assertEquals(1, a.compareTo(b));
+
+        assertEquals(xmlBefore, a.toXml());
+        assertEquals(hashBefore, a.hashCode());
+    }
+
+    @Test
+    public void sortingFirstOrderedByNumberOfSetFields() {
+        StructDataType type = new StructDataType("test");
+        type.addField(new Field("int", DataType.INT));
+        type.addField(new Field("flt", DataType.FLOAT));
+        type.addField(new Field("str", DataType.STRING));
+
+        Struct a = new Struct(type);
+        a.setFieldValue("int", new IntegerFieldValue(123));
+        Struct b = new Struct(type);
+
+        assertTrue(b.compareTo(a) < 0);
+        assertTrue(a.compareTo(b) > 0);
+        assertEquals(0, b.compareTo(b));
+        assertFalse(a.equals(b));
+        assertFalse(b.equals(a));
+
+        b.setFieldValue("int", new IntegerFieldValue(123));
+
+        assertEquals(0, a.compareTo(b));
+        assertEquals(0, b.compareTo(a));
+        assertTrue(a.equals(b));
+        assertTrue(b.equals(a));
+
+        b.setFieldValue("str", new StringFieldValue("hello world"));
+        assertTrue(b.compareTo(a) > 0);
+        assertTrue(a.compareTo(b) < 0);
+        assertFalse(a.equals(b));
+        assertFalse(b.equals(a));
+    }
+
+    @Test
+    public void sortingOrderIndependentOfValueInsertionOrder() {
+        StructDataType type = new StructDataType("test");
+        type.addField(new Field("int", DataType.INT));
+        type.addField(new Field("flt", DataType.FLOAT));
+        type.addField(new Field("str", DataType.STRING));
+
+        Struct a = new Struct(type);
+        a.setFieldValue("int", new IntegerFieldValue(123));
+        a.setFieldValue("flt", new DoubleFieldValue(45.6));
+        a.setFieldValue("str", new StringFieldValue("hello world"));
+        Struct b = new Struct(type);
+        b.setFieldValue("str", new StringFieldValue("hello world"));
+        b.setFieldValue("flt", new DoubleFieldValue(45.6));
+        b.setFieldValue("int", new IntegerFieldValue(123));
+
+        assertEquals(0, a.compareTo(b));
+        assertEquals(0, b.compareTo(a));
+        assertTrue(a.equals(b));
+        assertTrue(b.equals(a));
+
+        b.setFieldValue("int", new IntegerFieldValue(122));
+        assertTrue(a.compareTo(b) > 0);
+        assertTrue(b.compareTo(a) < 0);
+        assertFalse(a.equals(b));
+        assertFalse(b.equals(a));
+    }
+
+    @Test
+    public void sortingOrderDependsOnTypeFieldOrderWhenNonEqual() {
+        StructDataType type = new StructDataType("test");
+        type.addField(new Field("int", DataType.INT));
+        type.addField(new Field("intnotset", DataType.INT));
+        type.addField(new Field("flt", DataType.FLOAT));
+        type.addField(new Field("str", DataType.STRING));
+
+        Struct a = new Struct(type);
+        a.setFieldValue("int", new IntegerFieldValue(123));
+        a.setFieldValue("flt", new DoubleFieldValue(45.6));
+        Struct b = new Struct(type);
+        b.setFieldValue("int", new IntegerFieldValue(123));
+        b.setFieldValue("str", new StringFieldValue("hello world"));
+
+        // a sorts before b as it has flt set which occurs before str in the type
+        assertTrue(a.compareTo(b) < 0);
+        assertTrue(b.compareTo(a) > 0);
+        assertFalse(a.equals(b));
+        assertFalse(b.equals(a));
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/datatypes/TensorFieldValueTestCase.java b/document/src/test/java/com/yahoo/document/datatypes/TensorFieldValueTestCase.java
new file mode 100644
index 00000000000..7c01f8ab300
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/datatypes/TensorFieldValueTestCase.java
@@ -0,0 +1,43 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.datatypes;
+
+import com.yahoo.tensor.MapTensor;
+import com.yahoo.tensor.Tensor;
+import org.junit.Test;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+
+/**
+ * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a>
+ */
+public class TensorFieldValueTestCase {
+
+    private static TensorFieldValue createFieldValue(String tensor) {
+        return new TensorFieldValue(MapTensor.from(tensor));
+    }
+
+    @Test
+    public void requireThatDifferentFieldValueTypesAreNotEqual() {
+        assertFalse(createFieldValue("{{x:0}:2.0}").equals(new IntegerFieldValue(5)));
+    }
+
+    @Test
+    public void requireThatDifferentTensorValuesAreNotEqual() {
+        TensorFieldValue lhs = createFieldValue("{{x:0}:2.0}");
+        TensorFieldValue rhs = createFieldValue("{{x:0}:3.0}");
+        assertFalse(lhs.equals(rhs));
+        assertFalse(lhs.equals(new TensorFieldValue()));
+    }
+
+    @Test
+    public void requireThatSameTensorValueIsEqual() {
+        Tensor tensor = MapTensor.from("{{x:0}:2.0}");
+        TensorFieldValue lhs = new TensorFieldValue(tensor);
+        TensorFieldValue rhs = new TensorFieldValue(tensor);
+        assertTrue(lhs.equals(lhs));
+        assertTrue(lhs.equals(rhs));
+        assertTrue(lhs.equals(createFieldValue("{{x:0}:2.0}")));
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/datatypes/UriFieldValueTest.java b/document/src/test/java/com/yahoo/document/datatypes/UriFieldValueTest.java
new file mode 100644
index 00000000000..8fae87e5a29
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/datatypes/UriFieldValueTest.java
@@ -0,0 +1,22 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.datatypes;
+
+import org.junit.Test;
+
+import java.net.URI;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a>
+ */
+public class UriFieldValueTest {
+
+    @Test
+    public void requireThatURICanBeAssigned() {
+        UriFieldValue value = new UriFieldValue();
+        String uri = "http://user:pass@localhost:69/path#fragment?query";
+        value.assign(URI.create(uri));
+        assertEquals(uri, value.getWrappedValue());
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/datatypes/WeightedSetTestCase.java b/document/src/test/java/com/yahoo/document/datatypes/WeightedSetTestCase.java
new file mode 100644
index 00000000000..fb2122d9828
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/datatypes/WeightedSetTestCase.java
@@ -0,0 +1,160 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.datatypes;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.MapDataType;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class WeightedSetTestCase extends junit.framework.TestCase {
+    public void testSet() {
+        WeightedSet<StringFieldValue> wset = new WeightedSet<>(DataType.TAG);
+
+        //ADD:
+
+        Object ok;
+        ok = wset.put(new StringFieldValue("this is a test"), 5);
+        assertNull(ok);
+        ok = wset.put(new StringFieldValue("this is a test"), 10);
+        assertEquals(5, ok);
+
+        assertEquals(1, wset.size());
+        assertEquals(wset.get(new StringFieldValue("this is a test")), new Integer(10));
+
+        //REMOVE:
+
+        ok = wset.put(new StringFieldValue("another test"), 7);
+        assertNull(ok);
+
+        assertEquals(2, wset.size());
+
+        ok = wset.remove(new StringFieldValue("this is a test"));
+        assertNotNull(ok);
+        assertEquals(1, wset.size());
+
+        ok = wset.remove(new StringFieldValue("another test"));
+        assertNotNull(ok);
+        assertEquals(0, wset.size());
+
+        //CONTAINS:
+
+        wset.put(new StringFieldValue("ballooo"), 50);
+        wset.put(new StringFieldValue("bananaa"), 51);
+
+        ok = wset.containsKey(new StringFieldValue("bananaa"));
+        assertEquals(true, ok);
+        ok = wset.containsKey(new StringFieldValue("ballooo"));
+        assertEquals(true, ok);
+
+        //EQUALS // Make sure order of input doesn't affect equals
+        WeightedSet<StringFieldValue> wset2 = new WeightedSet<>(DataType.TAG);
+        wset2.put(new StringFieldValue("bananaa"), 51);
+        wset2.put(new StringFieldValue("ballooo"), 50);
+        assertEquals(wset, wset2);
+
+    }
+
+    public void testAssignDoesNotIgnoreSpecialProperties() {
+        DataType type = DataType.getWeightedSet(DataType.STRING);
+        WeightedSet<StringFieldValue> set = new WeightedSet<>(type);
+        set.put(new StringFieldValue("hello"), 5);
+        set.put(new StringFieldValue("aba"), 10);
+        assertEquals(2, set.size());
+        assertEquals(new Integer(5), set.get(new StringFieldValue("hello")));
+        assertEquals(new Integer(10), set.get(new StringFieldValue("aba")));
+
+        DataType type2 = DataType.getWeightedSet(DataType.STRING, true, true);
+        WeightedSet<StringFieldValue> set2 = new WeightedSet<>(type2);
+        set2.put(new StringFieldValue("hi"), 6);
+        set2.put(new StringFieldValue("bye"), 13);
+        set2.put(new StringFieldValue("see you"), 15);
+        assertEquals(3, set2.size());
+        assertEquals(new Integer(6), set2.get(new StringFieldValue("hi")));
+        assertEquals(new Integer(13), set2.get(new StringFieldValue("bye")));
+        assertEquals(new Integer(15), set2.get(new StringFieldValue("see you")));
+
+        try {
+            set.assign(set2);
+            fail("it shouldn't be possible to assign a weighted set to another when types differ");
+        } catch (IllegalArgumentException iae) {
+            //success
+        }
+
+        assertEquals(2, set.size());
+        assertEquals(new Integer(5), set.get(new StringFieldValue("hello")));
+        assertEquals(new Integer(10), set.get(new StringFieldValue("aba")));
+    }
+
+    public void testWrappedMap() {
+        WeightedSet<StringFieldValue> ws = new WeightedSet<>(DataType.getWeightedSet(DataType.STRING));
+        Map<String, Integer> map = new LinkedHashMap<>();
+        map.put("foo", 1);
+        map.put("bar", 2);
+
+        ws.assign(map);
+
+        assertEquals(2, ws.size());
+        assertEquals(2, map.size());
+
+        assertTrue(ws.containsKey(new StringFieldValue("foo")));
+        assertTrue(ws.containsKey(new StringFieldValue("bar")));
+        assertFalse(ws.containsKey(new StringFieldValue("babar")));
+
+        ws.put(new StringFieldValue("banana"), 55);
+
+        assertEquals(3, ws.size());
+        assertEquals(3, map.size());
+
+        assertTrue(ws.containsValue(55));
+        assertFalse(ws.isEmpty());
+        ws.clear();
+        assertEquals(0, ws.size());
+        assertEquals(0, map.size());
+        assertTrue(map.isEmpty());
+        assertTrue(ws.isEmpty());
+
+        Map<StringFieldValue, Integer> tmp = new LinkedHashMap<>();
+        tmp.put(new StringFieldValue("cocacola"), 999);
+        tmp.put(new StringFieldValue("pepsicola"), 99999);
+        ws.putAll(tmp);
+
+        assertEquals(2, ws.size());
+        assertEquals(2, map.size());
+
+        ws.remove(new StringFieldValue("cocacola"));
+
+        assertEquals(1, ws.size());
+        assertEquals(1, map.size());
+
+        assertTrue(ws.contains(new StringFieldValue("pepsicola")));
+
+        ws.put(new StringFieldValue("solo"), 4);
+
+        assertEquals(2, ws.size());
+        assertEquals(2, map.size());
+
+        ws.add(new StringFieldValue("sitronbrus"));
+
+        assertEquals(3, ws.size());
+        assertEquals(3, map.size());
+
+        assertEquals(new Integer(1), ws.get(new StringFieldValue("sitronbrus")));
+    }
+
+    public void testAssigningWrappedSetToMapFieldValue() {
+        WeightedSet<StringFieldValue> weightedSet = new WeightedSet<>(DataType.getWeightedSet(DataType.STRING));
+        WeightedSet<StringFieldValue> assignmentTarget = new WeightedSet<>(DataType.getWeightedSet(DataType.STRING));
+        Map<String, Integer> rawMap = new LinkedHashMap<>();
+        rawMap.put("foo", 1);
+        rawMap.put("bar", 2);
+        weightedSet.assign(rawMap);
+        assignmentTarget.assign(weightedSet);
+        assertEquals(2, assignmentTarget.size());
+        assertEquals(Integer.valueOf(1), assignmentTarget.get(new StringFieldValue("foo")));
+        assertEquals(Integer.valueOf(2), assignmentTarget.get(new StringFieldValue("bar")));
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/datatypes/blog.sd b/document/src/test/java/com/yahoo/document/datatypes/blog.sd
new file mode 100644
index 00000000000..1263c09ff95
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/datatypes/blog.sd
@@ -0,0 +1,49 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+search blog {
+
+  document blog {
+
+    field title type string {
+	    header
+        indexing: summary | index
+	    # index-to: default
+    }
+
+    field author type string {
+	    header
+	    indexing: summary | index
+	    # index-to: default
+    }
+
+    field body type string {
+        header
+	    indexing: summary | index
+    }
+
+    field url type uri {
+        header
+	    indexing: index | summary
+	    # index-to: default
+    }
+  }
+
+  annotation industry {
+    field vertical type string {}
+  }
+
+  annotation company inherits industry {
+    field name type string {}
+	field ceo type string {}	
+	field lat type long {}
+	field lon type long {}
+  }
+
+  annotation person {
+	field name type string {}
+  }
+
+  annotation location {
+  	field name type string {}
+  }
+}
+
diff --git a/document/src/test/java/com/yahoo/document/datatypes/documentmanager.blog.sd b/document/src/test/java/com/yahoo/document/datatypes/documentmanager.blog.sd
new file mode 100644
index 00000000000..313e8a88a3b
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/datatypes/documentmanager.blog.sd
@@ -0,0 +1,127 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+enablecompression false
+datatype[7]
+datatype[0].id -945638949
+datatype[0].arraytype[0]
+datatype[0].weightedsettype[0]
+datatype[0].structtype[1]
+datatype[0].structtype[0].name "blog.header"
+datatype[0].structtype[0].version 0
+datatype[0].structtype[0].field[4]
+datatype[0].structtype[0].field[0].name "title"
+datatype[0].structtype[0].field[0].datatype 2
+datatype[0].structtype[0].field[0].id[0]
+datatype[0].structtype[0].field[1].name "author"
+datatype[0].structtype[0].field[1].datatype 2
+datatype[0].structtype[0].field[1].id[0]
+datatype[0].structtype[0].field[2].name "body"
+datatype[0].structtype[0].field[2].datatype 2
+datatype[0].structtype[0].field[2].id[0]
+datatype[0].structtype[0].field[3].name "url"
+datatype[0].structtype[0].field[3].datatype 10
+datatype[0].structtype[0].field[3].id[0]
+datatype[0].structtype[0].inherits[0]
+datatype[0].documenttype[0]
+datatype[0].annotationreftype[0]
+datatype[1].id 1387420336
+datatype[1].arraytype[0]
+datatype[1].weightedsettype[0]
+datatype[1].structtype[1]
+datatype[1].structtype[0].name "blog.body"
+datatype[1].structtype[0].version 0
+datatype[1].structtype[0].field[0]
+datatype[1].structtype[0].inherits[0]
+datatype[1].documenttype[0]
+datatype[1].annotationreftype[0]
+datatype[2].id -1386162972
+datatype[2].arraytype[0]
+datatype[2].weightedsettype[0]
+datatype[2].structtype[0]
+datatype[2].documenttype[1]
+datatype[2].documenttype[0].name "blog"
+datatype[2].documenttype[0].version 0
+datatype[2].documenttype[0].headerstruct -945638949
+datatype[2].documenttype[0].bodystruct 1387420336
+datatype[2].documenttype[0].inherits[0]
+datatype[2].annotationreftype[0]
+datatype[3].id 912259135
+datatype[3].arraytype[0]
+datatype[3].weightedsettype[0]
+datatype[3].structtype[1]
+datatype[3].structtype[0].name "annotation.industry"
+datatype[3].structtype[0].version 0
+datatype[3].structtype[0].field[1]
+datatype[3].structtype[0].field[0].name "vertical"
+datatype[3].structtype[0].field[0].datatype 2
+datatype[3].structtype[0].field[0].id[0]
+datatype[3].structtype[0].inherits[0]
+datatype[3].documenttype[0]
+datatype[3].annotationreftype[0]
+datatype[4].id -476092672
+datatype[4].arraytype[0]
+datatype[4].weightedsettype[0]
+datatype[4].structtype[1]
+datatype[4].structtype[0].name "annotation.company"
+datatype[4].structtype[0].version 0
+datatype[4].structtype[0].field[4]
+datatype[4].structtype[0].field[0].name "name"
+datatype[4].structtype[0].field[0].datatype 2
+datatype[4].structtype[0].field[0].id[0]
+datatype[4].structtype[0].field[1].name "ceo"
+datatype[4].structtype[0].field[1].datatype 2
+datatype[4].structtype[0].field[1].id[0]
+datatype[4].structtype[0].field[2].name "lat"
+datatype[4].structtype[0].field[2].datatype 4
+datatype[4].structtype[0].field[2].id[0]
+datatype[4].structtype[0].field[3].name "lon"
+datatype[4].structtype[0].field[3].datatype 4
+datatype[4].structtype[0].field[3].id[0]
+datatype[4].structtype[0].inherits[1]
+datatype[4].structtype[0].inherits[0].name "annotation.industry"
+datatype[4].structtype[0].inherits[0].version 0
+datatype[4].documenttype[0]
+datatype[4].annotationreftype[0]
+datatype[5].id -1466283082
+datatype[5].arraytype[0]
+datatype[5].weightedsettype[0]
+datatype[5].structtype[1]
+datatype[5].structtype[0].name "annotation.person"
+datatype[5].structtype[0].version 0
+datatype[5].structtype[0].field[1]
+datatype[5].structtype[0].field[0].name "name"
+datatype[5].structtype[0].field[0].datatype 2
+datatype[5].structtype[0].field[0].id[0]
+datatype[5].structtype[0].inherits[0]
+datatype[5].documenttype[0]
+datatype[5].annotationreftype[0]
+datatype[6].id 515587158
+datatype[6].arraytype[0]
+datatype[6].weightedsettype[0]
+datatype[6].structtype[1]
+datatype[6].structtype[0].name "annotation.location"
+datatype[6].structtype[0].version 0
+datatype[6].structtype[0].field[1]
+datatype[6].structtype[0].field[0].name "name"
+datatype[6].structtype[0].field[0].datatype 2
+datatype[6].structtype[0].field[0].id[0]
+datatype[6].structtype[0].inherits[0]
+datatype[6].documenttype[0]
+datatype[6].annotationreftype[0]
+annotationtype[4]
+annotationtype[0].name "person"
+annotationtype[0].id 609952424
+annotationtype[0].datatype -1466283082
+annotationtype[0].inherits[0]
+annotationtype[1].name "location"
+annotationtype[1].id -270471211
+annotationtype[1].datatype 515587158
+annotationtype[1].inherits[0]
+annotationtype[2].name "company"
+annotationtype[2].id 1278713514
+annotationtype[2].datatype -476092672
+annotationtype[2].inherits[1]
+annotationtype[2].inherits[0].id 9765800
+annotationtype[3].name "industry"
+annotationtype[3].id 9765800
+annotationtype[3].datatype 912259135
+annotationtype[3].inherits[0]
diff --git a/document/src/test/java/com/yahoo/document/declaration/.gitignore b/document/src/test/java/com/yahoo/document/declaration/.gitignore
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/declaration/.gitignore
diff --git a/document/src/test/java/com/yahoo/document/docindoc.sd b/document/src/test/java/com/yahoo/document/docindoc.sd
new file mode 100644
index 00000000000..83bc4254fb2
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/docindoc.sd
@@ -0,0 +1,7 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+search docindoc {
+  document docindoc {
+    field name type string { header }
+    field content type string { body }
+  }
+}
diff --git a/document/src/test/java/com/yahoo/document/documentmanager.docindoc.cfg b/document/src/test/java/com/yahoo/document/documentmanager.docindoc.cfg
new file mode 100644
index 00000000000..3347c3127b5
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/documentmanager.docindoc.cfg
@@ -0,0 +1,42 @@
+enablecompression false
+datatype[7]
+datatype[0].id -1407012075
+datatype[0].structtype[1]
+datatype[0].structtype[0].name "outerdoc.body"
+datatype[0].structtype[0].version 0
+datatype[0].structtype[0].field[1]
+datatype[0].structtype[0].field[0].datatype -2035324352
+datatype[0].structtype[0].field[0].name "innerdocuments"
+datatype[1].id -1686125086
+datatype[1].structtype[1]
+datatype[1].structtype[0].name "docindoc.header"
+datatype[1].structtype[0].version 0
+datatype[1].structtype[0].field[1]
+datatype[1].structtype[0].field[0].datatype 2
+datatype[1].structtype[0].field[0].name "name"
+datatype[2].id -2035324352
+datatype[2].arraytype[1]
+datatype[2].arraytype[0].datatype 1447635645
+datatype[3].id -2040625920
+datatype[3].structtype[1]
+datatype[3].structtype[0].name "outerdoc.header"
+datatype[3].structtype[0].version 0
+datatype[4].id 1447635645
+datatype[4].documenttype[1]
+datatype[4].documenttype[0].bodystruct 2030224503
+datatype[4].documenttype[0].headerstruct -1686125086
+datatype[4].documenttype[0].name "docindoc"
+datatype[4].documenttype[0].version 0
+datatype[5].id 1748635999
+datatype[5].documenttype[1]
+datatype[5].documenttype[0].bodystruct -1407012075
+datatype[5].documenttype[0].headerstruct -2040625920
+datatype[5].documenttype[0].name "outerdoc"
+datatype[5].documenttype[0].version 0
+datatype[6].id 2030224503
+datatype[6].structtype[1]
+datatype[6].structtype[0].name "docindoc.body"
+datatype[6].structtype[0].version 0
+datatype[6].structtype[0].field[1]
+datatype[6].structtype[0].field[0].datatype 2
+datatype[6].structtype[0].field[0].name "content"
diff --git a/document/src/test/java/com/yahoo/document/fieldset/FieldSetTestCase.java b/document/src/test/java/com/yahoo/document/fieldset/FieldSetTestCase.java
new file mode 100644
index 00000000000..ef8346768ba
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/fieldset/FieldSetTestCase.java
@@ -0,0 +1,183 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.fieldset;
+
+import com.yahoo.document.Document;
+import com.yahoo.document.DocumentTestCaseBase;
+import com.yahoo.document.Field;
+import com.yahoo.document.datatypes.FieldValue;
+import org.junit.Test;
+
+import java.util.Iterator;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Test for field sets
+ */
+public class FieldSetTestCase extends DocumentTestCaseBase {
+
+    @Test
+    public void testClone() throws Exception {
+        assertTrue(new AllFields().clone() instanceof AllFields);
+        assertTrue(new NoFields().clone() instanceof NoFields);
+        assertTrue(new HeaderFields().clone() instanceof HeaderFields);
+        assertTrue(new BodyFields().clone() instanceof BodyFields);
+        assertTrue(new DocIdOnly().clone() instanceof DocIdOnly);
+    }
+
+    @Test
+    public void testParsing() {
+        FieldSetRepo repo = new FieldSetRepo();
+
+        assertTrue(repo.parse(docMan, "[all]") instanceof AllFields);
+        assertTrue(repo.parse(docMan, "[none]") instanceof NoFields);
+        assertTrue(repo.parse(docMan, "[id]") instanceof DocIdOnly);
+        assertTrue(repo.parse(docMan, "[header]") instanceof HeaderFields);
+        assertTrue(repo.parse(docMan, "[body]") instanceof BodyFields);
+
+        FieldCollection collection = (FieldCollection)repo.parse(docMan, "testdoc:stringattr,intattr");
+        assertEquals(2, collection.size());
+    }
+
+    void assertContains(String str1, String str2) throws Exception {
+        FieldSetRepo repo = new FieldSetRepo();
+        FieldSet set1 = repo.parse(docMan, str1);
+        FieldSet set2 = repo.parse(docMan, str2);
+        assertTrue(set1.clone().contains(set2.clone()));
+    }
+
+    void assertNotContains(String str1, String str2) throws Exception {
+        FieldSetRepo repo = new FieldSetRepo();
+        FieldSet set1 = repo.parse(docMan, str1);
+        FieldSet set2 = repo.parse(docMan, str2);
+        assertFalse(set1.clone().contains(set2.clone()));
+    }
+
+    void assertError(String str) {
+        try {
+            new FieldSetRepo().parse(docMan, str);
+            assertTrue(false);
+        } catch (Exception e) {
+        }
+    }
+
+    @Test
+    public void testContains() throws Exception {
+        Field headerField = testDocType.getField("intattr");
+        Field bodyField = testDocType.getField("rawattr");
+
+        assertFalse(headerField.contains(testDocType.getField("byteattr")));
+        assertTrue(headerField.contains(testDocType.getField("intattr")));
+        assertFalse(headerField.contains(bodyField));
+        assertTrue(headerField.contains(new DocIdOnly()));
+        assertTrue(headerField.contains(new NoFields()));
+        assertFalse(headerField.contains(new AllFields()));
+        assertFalse(headerField.contains(new HeaderFields()));
+        assertFalse(headerField.contains(new BodyFields()));
+
+        assertFalse(new NoFields().contains(headerField));
+        assertFalse(new NoFields().contains(new AllFields()));
+        assertFalse(new NoFields().contains(new DocIdOnly()));
+
+        assertTrue(new AllFields().contains(new HeaderFields()));
+        assertTrue(new AllFields().contains(headerField));
+        assertTrue(new AllFields().contains(bodyField));
+        assertTrue(new AllFields().contains(new BodyFields()));
+        assertTrue(new AllFields().contains(new DocIdOnly()));
+        assertTrue(new AllFields().contains(new NoFields()));
+        assertTrue(new AllFields().contains(new AllFields()));
+
+        assertTrue(new DocIdOnly().contains(new NoFields()));
+        assertTrue(new DocIdOnly().contains(new DocIdOnly()));
+        assertFalse(new DocIdOnly().contains(headerField));
+
+        assertTrue(new HeaderFields().contains(headerField));
+        assertFalse(new HeaderFields().contains(bodyField));
+        assertTrue(new HeaderFields().contains(new DocIdOnly()));
+        assertTrue(new HeaderFields().contains(new NoFields()));
+
+        assertContains("[body]", "testdoc:rawattr");
+        assertContains("[header]", "testdoc:intattr");
+        assertNotContains("[header]", "testdoc:rawattr");
+        assertContains("testdoc:rawattr,intattr", "testdoc:intattr");
+        assertNotContains("testdoc:intattr", "testdoc:rawattr,intattr");
+        assertContains("testdoc:intattr,rawattr", "testdoc:rawattr,intattr");
+
+        assertError("nodoctype");
+        assertError("unknowndoctype:foo");
+        assertError("testdoc:unknownfield");
+        assertError("[badid]");
+    }
+
+    String stringifyFields(Document doc) {
+        String retVal = "";
+        for (Iterator<Map.Entry<Field, FieldValue>> i = doc.iterator(); i.hasNext(); ) {
+            Map.Entry<Field, FieldValue> v = i.next();
+
+            if (retVal.length() > 0) {
+                retVal += ",";
+            }
+            retVal += v.getKey().getName() + ":" + v.getValue().toString();
+        }
+
+        return retVal;
+    }
+
+    String doCopyFields(Document source, String fieldSet) {
+        FieldSetRepo repo = new FieldSetRepo();
+        Document target = new Document(source.getDataType(), source.getId());
+        repo.copyFields(source, target, repo.parse(docMan, fieldSet));
+        return stringifyFields(target);
+    }
+
+    @Test
+    public void testCopyDocumentFields() {
+        Document doc = getTestDocument();
+        doc.removeFieldValue("rawattr");
+
+        assertEquals("floatattr:3.56", doCopyFields(doc, "[body]"));
+        assertEquals("stringattr:tjohei,intattr:50,byteattr:30,floatattr:3.56", doCopyFields(doc, "[all]"));
+        assertEquals("stringattr:tjohei,intattr:50,byteattr:30", doCopyFields(doc, "[header]"));
+        assertEquals("byteattr:30,floatattr:3.56", doCopyFields(doc, "testdoc:floatattr,byteattr"));
+    }
+
+    String doStripFields(Document source, String fieldSet) {
+        FieldSetRepo repo = new FieldSetRepo();
+        Document target = source.clone();
+        repo.stripFields(target, repo.parse(docMan, fieldSet));
+        return stringifyFields(target);
+    }
+
+    @Test
+    public void testStripFields() {
+        Document doc = getTestDocument();
+        doc.removeFieldValue("rawattr");
+
+        assertEquals("floatattr:3.56", doStripFields(doc, "[body]"));
+        assertEquals("stringattr:tjohei,intattr:50,byteattr:30,floatattr:3.56", doStripFields(doc, "[all]"));
+        assertEquals("stringattr:tjohei,intattr:50,byteattr:30", doStripFields(doc, "[header]"));
+        assertEquals("byteattr:30,floatattr:3.56", doStripFields(doc, "testdoc:floatattr,byteattr"));
+    }
+
+    @Test
+    public void testSerialize() {
+        String fieldSets[] =
+                {
+                        "[all]",
+                        "[none]",
+                        "[header]",
+                        "[docid]",
+                        "[body]",
+                        "testdoc:rawattr",
+                        "testdoc:rawattr,intattr"
+                };
+
+        FieldSetRepo repo = new FieldSetRepo();
+        for (String fieldSet : fieldSets) {
+            assertEquals(fieldSet, repo.serialize(repo.parse(docMan, fieldSet)));
+        }
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java
new file mode 100644
index 00000000000..860b2c9fba8
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java
@@ -0,0 +1,1252 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.json;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.io.JsonStringEncoder;
+import com.google.common.base.Joiner;
+import com.yahoo.collections.Tuple2;
+import com.yahoo.document.ArrayDataType;
+import com.yahoo.document.DataType;
+import com.yahoo.document.Document;
+import com.yahoo.document.DocumentId;
+import com.yahoo.document.DocumentOperation;
+import com.yahoo.document.DocumentPut;
+import com.yahoo.document.DocumentRemove;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.DocumentTypeManager;
+import com.yahoo.document.DocumentUpdate;
+import com.yahoo.document.Field;
+import com.yahoo.document.MapDataType;
+import com.yahoo.document.PositionDataType;
+import com.yahoo.document.StructDataType;
+import com.yahoo.document.WeightedSetDataType;
+import com.yahoo.document.datatypes.Array;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.IntegerFieldValue;
+import com.yahoo.document.datatypes.MapFieldValue;
+import com.yahoo.document.datatypes.Raw;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.document.datatypes.Struct;
+import com.yahoo.document.datatypes.TensorFieldValue;
+import com.yahoo.document.datatypes.WeightedSet;
+import com.yahoo.document.update.*;
+import com.yahoo.document.update.ArithmeticValueUpdate.Operator;
+import com.yahoo.tensor.MapTensor;
+import com.yahoo.text.Utf8;
+import org.apache.commons.codec.binary.Base64;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.mockito.internal.matchers.Contains;
+
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
+
+import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.*;
+
+/**
+ * Basic test of JSON streams to Vespa document instances.
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ * @author vegard
+ */
+public class JsonReaderTestCase {
+    DocumentTypeManager types;
+    JsonFactory parserFactory;
+
+    @Rule
+    public ExpectedException exception = ExpectedException.none();
+
+    @Before
+    public void setUp() throws Exception {
+        parserFactory = new JsonFactory();
+        types = new DocumentTypeManager();
+        {
+            DocumentType x = new DocumentType("smoke");
+            x.addField(new Field("something", DataType.STRING));
+            x.addField(new Field("nalle", DataType.STRING));
+            x.addField(new Field("int1", DataType.INT));
+            types.registerDocumentType(x);
+        }
+        {
+            DocumentType x = new DocumentType("mirrors");
+            StructDataType woo = new StructDataType("woo");
+            woo.addField(new Field("sandra", DataType.STRING));
+            woo.addField(new Field("cloud", DataType.STRING));
+            x.addField(new Field("skuggsjaa", woo));
+            types.registerDocumentType(x);
+        }
+        {
+            DocumentType x = new DocumentType("testarray");
+            DataType d = new ArrayDataType(DataType.STRING);
+            x.addField(new Field("actualarray", d));
+            types.registerDocumentType(x);
+        }
+        {
+            DocumentType x = new DocumentType("testset");
+            DataType d = new WeightedSetDataType(DataType.STRING, true, true);
+            x.addField(new Field("actualset", d));
+            types.registerDocumentType(x);
+        }
+        {
+            DocumentType x = new DocumentType("testmap");
+            DataType d = new MapDataType(DataType.STRING, DataType.STRING);
+            x.addField(new Field("actualmap", d));
+            types.registerDocumentType(x);
+        }
+        {
+            DocumentType x = new DocumentType("testraw");
+            DataType d = DataType.RAW;
+            x.addField(new Field("actualraw", d));
+            types.registerDocumentType(x);
+        }
+        {
+            DocumentType x = new DocumentType("testMapStringToArrayOfInt");
+            DataType value = new ArrayDataType(DataType.INT);
+            DataType d = new MapDataType(DataType.STRING, value);
+            x.addField(new Field("actualMapStringToArrayOfInt", d));
+            types.registerDocumentType(x);
+        }
+        {
+            DocumentType x = new DocumentType("testsinglepos");
+            DataType d = PositionDataType.INSTANCE;
+            x.addField(new Field("singlepos", d));
+            types.registerDocumentType(x);
+        }
+        {
+            DocumentType x = new DocumentType("testtensor");
+            x.addField(new Field("tensorfield", DataType.TENSOR));
+            types.registerDocumentType(x);
+        }
+        {
+            DocumentType x = new DocumentType("testpredicate");
+            x.addField(new Field("boolean", DataType.PREDICATE));
+            types.registerDocumentType(x);
+        }
+        {
+            DocumentType x = new DocumentType("testint");
+            x.addField(new Field("integerfield", DataType.INT));
+            types.registerDocumentType(x);
+        }
+    }
+
+    @After
+    public void tearDown() throws Exception {
+        types = null;
+        parserFactory = null;
+        exception = ExpectedException.none();
+    }
+
+    @Test
+    public final void readSingleDocumentPut() {
+        InputStream rawDoc = new ByteArrayInputStream(
+                Utf8.toBytes("{\"put\": \"id:unittest:smoke::whee\","
+                        + " \"fields\": { \"something\": \"smoketest\","
+                        + " \"nalle\": \"bamse\"}}"));
+        JsonReader r = new JsonReader(types, rawDoc, parserFactory);
+        DocumentPut put = (DocumentPut) r.readSingleDocument(JsonReader.SupportedOperation.PUT, "id:unittest:smoke::whee");
+        smokeTestDoc(put.getDocument());
+    }
+
+    @Test
+    public final void readSingleDocumentUpdate() {
+        InputStream rawDoc = new ByteArrayInputStream(
+                Utf8.toBytes("{\"update\": \"id:unittest:smoke::whee\","
+                        + " \"fields\": { \"something\": {"
+                        + " \"assign\": \"orOther\" }}" + " }"));
+        JsonReader r = new JsonReader(types, rawDoc, parserFactory);
+        DocumentUpdate doc = (DocumentUpdate) r.readSingleDocument(JsonReader.SupportedOperation.UPDATE, "id:unittest:smoke::whee");
+        FieldUpdate f = doc.getFieldUpdate("something");
+        assertEquals(1, f.size());
+        assertTrue(f.getValueUpdate(0) instanceof AssignValueUpdate);
+    }
+
+    @Test
+    public final void readClearField() {
+        InputStream rawDoc = new ByteArrayInputStream(
+                Utf8.toBytes("{\"update\": \"id:unittest:smoke::whee\","
+                        + " \"fields\": { \"int1\": {"
+                        + " \"assign\": null }}" + " }"));
+        JsonReader r = new JsonReader(types, rawDoc, parserFactory);
+        DocumentUpdate doc = (DocumentUpdate) r.readSingleDocument(JsonReader.SupportedOperation.UPDATE, "id:unittest:smoke::whee");
+        FieldUpdate f = doc.getFieldUpdate("int1");
+        assertEquals(1, f.size());
+        assertTrue(f.getValueUpdate(0) instanceof ClearValueUpdate);
+        assertNull(f.getValueUpdate(0).getValue());
+    }
+
+
+    @Test
+    public final void smokeTest() {
+        InputStream rawDoc = new ByteArrayInputStream(
+                Utf8.toBytes("{\"put\": \"id:unittest:smoke::whee\","
+                        + " \"fields\": { \"something\": \"smoketest\","
+                        + " \"nalle\": \"bamse\"}}"));
+        JsonReader r = new JsonReader(types, rawDoc, parserFactory);
+        JsonReader.DocumentParseInfo parseInfo = r.parseDocument().get();
+        DocumentType docType = r.readDocumentType(parseInfo.documentId);
+        DocumentPut put = new DocumentPut(new Document(docType, parseInfo.documentId));
+        r.readPut(put);
+        smokeTestDoc(put.getDocument());
+    }
+
+    @Test
+    public final void docIdLookaheadTest() {
+        InputStream rawDoc = new ByteArrayInputStream(
+                Utf8.toBytes("{"
+                        + " \"fields\": { \"something\": \"smoketest\","
+                        + " \"nalle\": \"bamse\"},"
+                        + "\"put\": \"id:unittest:smoke::whee\""
+                        + "}"));
+        JsonReader r = new JsonReader(types, rawDoc, parserFactory);
+        JsonReader.DocumentParseInfo parseInfo = r.parseDocument().get();
+        DocumentType docType = r.readDocumentType(parseInfo.documentId);
+        DocumentPut put = new DocumentPut(new Document(docType, parseInfo.documentId));
+        r.readPut(put);
+        smokeTestDoc(put.getDocument());
+    }
+
+
+    @Test
+    public final void emptyDocTest() {
+        InputStream rawDoc = new ByteArrayInputStream(
+                Utf8.toBytes("{\"put\": \"id:unittest:smoke::whee\","
+                        + " \"fields\": {}}"));
+        JsonReader r = new JsonReader(types, rawDoc, parserFactory);
+        JsonReader.DocumentParseInfo parseInfo = r.parseDocument().get();
+        DocumentType docType = r.readDocumentType(parseInfo.documentId);
+        DocumentPut put = new DocumentPut(new Document(docType, parseInfo.documentId));
+        r.readPut(put);
+        assertEquals("id:unittest:smoke::whee", parseInfo.documentId.toString());
+    }
+
+    @Test
+    public final void testStruct() {
+        InputStream rawDoc = new ByteArrayInputStream(
+                Utf8.toBytes("{\"put\": \"id:unittest:mirrors::whee\","
+                        + " \"fields\": { "
+                        + "\"skuggsjaa\": {"
+                        + "\"sandra\": \"person\","
+                        + " \"cloud\": \"another person\"}}}"));
+        JsonReader r = new JsonReader(types, rawDoc, parserFactory);
+        JsonReader.DocumentParseInfo parseInfo = r.parseDocument().get();
+        DocumentType docType = r.readDocumentType(parseInfo.documentId);
+        DocumentPut put = new DocumentPut(new Document(docType, parseInfo.documentId));
+        r.readPut(put);
+        Document doc = put.getDocument();
+        FieldValue f = doc.getFieldValue(doc.getField("skuggsjaa"));
+        assertSame(Struct.class, f.getClass());
+        Struct s = (Struct) f;
+        assertEquals("person", ((StringFieldValue) s.getFieldValue("sandra")).getString());
+    }
+
+    @Test
+    public final void testUpdateArray() {
+        InputStream rawDoc = new ByteArrayInputStream(
+                Utf8.toBytes("{\"update\": \"id:unittest:testarray::whee\","
+                        + " \"fields\": { " + "\"actualarray\": {"
+                        + " \"add\": ["
+                        + " \"person\","
+                        + " \"another person\"]}}}"));
+        JsonReader r = new JsonReader(types, rawDoc, parserFactory);
+        JsonReader.DocumentParseInfo parseInfo = r.parseDocument().get();
+        DocumentType docType = r.readDocumentType(parseInfo.documentId);
+        DocumentUpdate doc = new DocumentUpdate(docType, parseInfo.documentId);
+        r.readUpdate(doc);
+        checkSimpleArrayAdd(doc);
+    }
+
+    @Test
+    public final void testUpdateWeighted() {
+        InputStream rawDoc = new ByteArrayInputStream(
+                Utf8.toBytes("{\"update\": \"id:unittest:testset::whee\","
+                        + " \"fields\": { " + "\"actualset\": {"
+                        + " \"add\": {"
+                        + " \"person\": 37,"
+                        + " \"another person\": 41}}}}"));
+        JsonReader r = new JsonReader(types, rawDoc, parserFactory);
+        JsonReader.DocumentParseInfo parseInfo = r.parseDocument().get();
+        DocumentType docType = r.readDocumentType(parseInfo.documentId);
+        DocumentUpdate doc = new DocumentUpdate(docType, parseInfo.documentId);
+        r.readUpdate(doc);
+        Map<String, Integer> weights = new HashMap<>();
+        FieldUpdate x = doc.getFieldUpdate("actualset");
+        for (ValueUpdate<?> v : x.getValueUpdates()) {
+            AddValueUpdate adder = (AddValueUpdate) v;
+            final String s = ((StringFieldValue) adder.getValue()).getString();
+            weights.put(s, adder.getWeight());
+        }
+        assertEquals(2, weights.size());
+        final String o = "person";
+        final String o2 = "another person";
+        assertTrue(weights.containsKey(o));
+        assertTrue(weights.containsKey(o2));
+        assertEquals(Integer.valueOf(37), weights.get(o));
+        assertEquals(Integer.valueOf(41), weights.get(o2));
+    }
+
+    @Test
+    public final void testUpdateMatch() {
+        InputStream rawDoc = new ByteArrayInputStream(
+                Utf8.toBytes("{\"update\": \"id:unittest:testset::whee\","
+                        + " \"fields\": { " + "\"actualset\": {"
+                        + " \"match\": {"
+                        + " \"element\": \"person\","
+                        + " \"increment\": 13}}}}"));
+        JsonReader r = new JsonReader(types, rawDoc, parserFactory);
+        JsonReader.DocumentParseInfo parseInfo = r.parseDocument().get();
+        DocumentType docType = r.readDocumentType(parseInfo.documentId);
+        DocumentUpdate doc = new DocumentUpdate(docType, parseInfo.documentId);
+
+        r.readUpdate(doc);
+        Map<String, Tuple2<Number, String>> matches = new HashMap<>();
+        FieldUpdate x = doc.getFieldUpdate("actualset");
+        for (ValueUpdate<?> v : x.getValueUpdates()) {
+            MapValueUpdate adder = (MapValueUpdate) v;
+            final String key = ((StringFieldValue) adder.getValue())
+                    .getString();
+            String op = ((ArithmeticValueUpdate) adder.getUpdate())
+                    .getOperator().toString();
+            Number n = ((ArithmeticValueUpdate) adder.getUpdate()).getOperand();
+            matches.put(key, new Tuple2<>(n, op));
+        }
+        assertEquals(1, matches.size());
+        final String o = "person";
+        assertEquals("ADD", matches.get(o).second);
+        assertEquals(Double.valueOf(13), matches.get(o).first);
+    }
+
+    @SuppressWarnings({ "cast", "unchecked", "rawtypes" })
+    @Test
+    public final void testArithmeticOperators() {
+        Tuple2[] operations = new Tuple2[] {
+                new Tuple2<String, Operator>(JsonReader.UPDATE_DECREMENT,
+                        ArithmeticValueUpdate.Operator.SUB),
+                new Tuple2<String, Operator>(JsonReader.UPDATE_DIVIDE,
+                        ArithmeticValueUpdate.Operator.DIV),
+                new Tuple2<String, Operator>(JsonReader.UPDATE_INCREMENT,
+                        ArithmeticValueUpdate.Operator.ADD),
+                new Tuple2<String, Operator>(JsonReader.UPDATE_MULTIPLY,
+                        ArithmeticValueUpdate.Operator.MUL) };
+        for (Tuple2<String, Operator> operator : operations) {
+            InputStream rawDoc = new ByteArrayInputStream(
+                    Utf8.toBytes("{\"update\": \"id:unittest:testset::whee\","
+                            + " \"fields\": { " + "\"actualset\": {"
+                            + " \"match\": {" + " \"element\": \"person\","
+                            + " \"" + (String) operator.first + "\": 13}}}}"));
+
+            JsonReader r = new JsonReader(types, rawDoc, parserFactory);
+            JsonReader.DocumentParseInfo parseInfo = r.parseDocument().get();
+            DocumentType docType = r.readDocumentType(parseInfo.documentId);
+            DocumentUpdate doc = new DocumentUpdate(docType, parseInfo.documentId);
+
+            r.readUpdate(doc);
+            Map<String, Tuple2<Number, Operator>> matches = new HashMap<>();
+            FieldUpdate x = doc.getFieldUpdate("actualset");
+            for (ValueUpdate v : x.getValueUpdates()) {
+                MapValueUpdate adder = (MapValueUpdate) v;
+                final String key = ((StringFieldValue) adder.getValue())
+                        .getString();
+                Operator op = ((ArithmeticValueUpdate) adder
+                        .getUpdate()).getOperator();
+                Number n = ((ArithmeticValueUpdate) adder.getUpdate())
+                        .getOperand();
+                matches.put(key, new Tuple2<>(n, op));
+            }
+            assertEquals(1, matches.size());
+            final String o = "person";
+            assertSame(operator.second, matches.get(o).second);
+            assertEquals(Double.valueOf(13), matches.get(o).first);
+        }
+    }
+
+    @SuppressWarnings("rawtypes")
+    @Test
+    public final void testArrayIndexing() {
+        InputStream rawDoc = new ByteArrayInputStream(
+                Utf8.toBytes("{\"update\": \"id:unittest:testarray::whee\","
+                        + " \"fields\": { " + "\"actualarray\": {"
+                        + " \"match\": {"
+                        + " \"element\": 3,"
+                        + " \"assign\": \"nalle\"}}}}"));
+        JsonReader r = new JsonReader(types, rawDoc, parserFactory);
+        JsonReader.DocumentParseInfo parseInfo = r.parseDocument().get();
+        DocumentType docType = r.readDocumentType(parseInfo.documentId);
+        DocumentUpdate doc = new DocumentUpdate(docType, parseInfo.documentId);
+
+        r.readUpdate(doc);
+        Map<Number, String> matches = new HashMap<>();
+        FieldUpdate x = doc.getFieldUpdate("actualarray");
+        for (ValueUpdate v : x.getValueUpdates()) {
+            MapValueUpdate adder = (MapValueUpdate) v;
+            final Number key = ((IntegerFieldValue) adder.getValue())
+                    .getNumber();
+            String op = ((StringFieldValue) ((AssignValueUpdate) adder.getUpdate())
+                    .getValue()).getString();
+            matches.put(key, op);
+        }
+        assertEquals(1, matches.size());
+        Number n = Integer.valueOf(3);
+        assertEquals("nalle", matches.get(n));
+    }
+
+    @Test
+    public final void testDocumentRemove() {
+        InputStream rawDoc = new ByteArrayInputStream(
+                Utf8.toBytes("{\"remove\": \"id:unittest:smoke::whee\""
+                        + " }}"));
+        JsonReader r = new JsonReader(types, rawDoc, parserFactory);
+        DocumentType docType = r.readDocumentType(new DocumentId("id:unittest:smoke::whee"));
+        assertEquals("smoke", docType.getName());
+    }
+
+    @Test
+    public final void testWeightedSet() {
+        InputStream rawDoc = new ByteArrayInputStream(
+                Utf8.toBytes("{\"put\": \"id:unittest:testset::whee\","
+                        + " \"fields\": { \"actualset\": {"
+                        + " \"nalle\": 2,"
+                        + " \"tralle\": 7 }}}"));
+        JsonReader r = new JsonReader(types, rawDoc, parserFactory);
+        JsonReader.DocumentParseInfo parseInfo = r.parseDocument().get();
+        DocumentType docType = r.readDocumentType(parseInfo.documentId);
+        DocumentPut put = new DocumentPut(new Document(docType, parseInfo.documentId));
+        r.readPut(put);
+        Document doc = put.getDocument();
+        FieldValue f = doc.getFieldValue(doc.getField("actualset"));
+        assertSame(WeightedSet.class, f.getClass());
+        WeightedSet<?> w = (WeightedSet<?>) f;
+        assertEquals(2, w.size());
+        assertEquals(new Integer(2), w.get(new StringFieldValue("nalle")));
+        assertEquals(new Integer(7), w.get(new StringFieldValue("tralle")));
+    }
+
+    @Test
+    public final void testArray() {
+        InputStream rawDoc = new ByteArrayInputStream(
+                Utf8.toBytes("{\"put\": \"id:unittest:testarray::whee\","
+                        + " \"fields\": { \"actualarray\": ["
+                        + " \"nalle\","
+                        + " \"tralle\"]}}"));
+        JsonReader r = new JsonReader(types, rawDoc, parserFactory);
+        JsonReader.DocumentParseInfo parseInfo = r.parseDocument().get();
+        DocumentType docType = r.readDocumentType(parseInfo.documentId);
+        DocumentPut put = new DocumentPut(new Document(docType, parseInfo.documentId));
+        r.readPut(put);
+        Document doc = put.getDocument();
+        FieldValue f = doc.getFieldValue(doc.getField("actualarray"));
+        assertSame(Array.class, f.getClass());
+        Array<?> a = (Array<?>) f;
+        assertEquals(2, a.size());
+        assertEquals(new StringFieldValue("nalle"), a.get(0));
+        assertEquals(new StringFieldValue("tralle"), a.get(1));
+    }
+
+    @Test
+    public final void testMap() {
+        InputStream rawDoc = new ByteArrayInputStream(
+                Utf8.toBytes("{\"put\": \"id:unittest:testmap::whee\","
+                        + " \"fields\": { \"actualmap\": ["
+                        + " { \"key\": \"nalle\", \"value\": \"kalle\"},"
+                        + " { \"key\": \"tralle\", \"value\": \"skalle\"} ]}}"));
+        JsonReader r = new JsonReader(types, rawDoc, parserFactory);
+        JsonReader.DocumentParseInfo parseInfo = r.parseDocument().get();
+        DocumentType docType = r.readDocumentType(parseInfo.documentId);
+        DocumentPut put = new DocumentPut(new Document(docType, parseInfo.documentId));
+        r.readPut(put);
+        Document doc = put.getDocument();
+        FieldValue f = doc.getFieldValue(doc.getField("actualmap"));
+        assertSame(MapFieldValue.class, f.getClass());
+        MapFieldValue<?, ?> m = (MapFieldValue<?, ?>) f;
+        assertEquals(2, m.size());
+        assertEquals(new StringFieldValue("kalle"), m.get(new StringFieldValue("nalle")));
+        assertEquals(new StringFieldValue("skalle"), m.get(new StringFieldValue("tralle")));
+    }
+
+    @Test
+    public final void testPositionPositive() {
+        InputStream rawDoc = new ByteArrayInputStream(
+                Utf8.toBytes("{\"put\": \"id:unittest:testsinglepos::bamf\","
+                        + " \"fields\": { \"singlepos\": \"N63.429722;E10.393333\" }}"));
+        JsonReader r = new JsonReader(types, rawDoc, parserFactory);
+        JsonReader.DocumentParseInfo parseInfo = r.parseDocument().get();
+        DocumentType docType = r.readDocumentType(parseInfo.documentId);
+        DocumentPut put = new DocumentPut(new Document(docType, parseInfo.documentId));
+        r.readPut(put);
+        Document doc = put.getDocument();
+        FieldValue f = doc.getFieldValue(doc.getField("singlepos"));
+        assertSame(Struct.class, f.getClass());
+        assertEquals(10393333, PositionDataType.getXValue(f).getInteger());
+        assertEquals(63429722, PositionDataType.getYValue(f).getInteger());
+    }
+
+    @Test
+    public final void testPositionNegative() {
+        InputStream rawDoc = new ByteArrayInputStream(
+                Utf8.toBytes("{\"put\": \"id:unittest:testsinglepos::bamf\","
+                        + " \"fields\": { \"singlepos\": \"W46.63;S23.55\" }}"));
+        JsonReader r = new JsonReader(types, rawDoc, parserFactory);
+        JsonReader.DocumentParseInfo parseInfo = r.parseDocument().get();
+        DocumentType docType = r.readDocumentType(parseInfo.documentId);
+        DocumentPut put = new DocumentPut(new Document(docType, parseInfo.documentId));
+        r.readPut(put);
+        Document doc = put.getDocument();
+        FieldValue f = doc.getFieldValue(doc.getField("singlepos"));
+        assertSame(Struct.class, f.getClass());
+        assertEquals(-46630000, PositionDataType.getXValue(f).getInteger());
+        assertEquals(-23550000, PositionDataType.getYValue(f).getInteger());
+    }
+
+    @Test
+    public final void testRaw() {
+        String stuff = new String(new JsonStringEncoder().quoteAsString(new Base64().encodeToString(Utf8.toBytes("smoketest"))));
+        InputStream rawDoc = new ByteArrayInputStream(
+                Utf8.toBytes("{\"put\": \"id:unittest:testraw::whee\","
+                        + " \"fields\": { \"actualraw\": \""
+                        + stuff
+                        + "\""
+                        + " }}"));
+        JsonReader r = new JsonReader(types, rawDoc, parserFactory);
+        JsonReader.DocumentParseInfo parseInfo = r.parseDocument().get();
+        DocumentType docType = r.readDocumentType(parseInfo.documentId);
+        DocumentPut put = new DocumentPut(new Document(docType, parseInfo.documentId));
+        r.readPut(put);
+        Document doc = put.getDocument();
+        FieldValue f = doc.getFieldValue(doc.getField("actualraw"));
+        assertSame(Raw.class, f.getClass());
+        Raw s = (Raw) f;
+        ByteBuffer b = s.getByteBuffer();
+        assertEquals("smoketest", Utf8.toString(b));
+    }
+
+    @Test
+    public final void testMapStringToArrayOfInt() {
+        InputStream rawDoc = new ByteArrayInputStream(
+                Utf8.toBytes("{\"put\": \"id:unittest:testMapStringToArrayOfInt::whee\","
+                        + " \"fields\": { \"actualMapStringToArrayOfInt\": ["
+                        + "{ \"key\": \"bamse\", \"value\": [1, 2, 3] }"
+                        + "]}}"));
+        JsonReader r = new JsonReader(types, rawDoc, parserFactory);
+        JsonReader.DocumentParseInfo parseInfo = r.parseDocument().get();
+        DocumentType docType = r.readDocumentType(parseInfo.documentId);
+        DocumentPut put = new DocumentPut(new Document(docType, parseInfo.documentId));
+        r.readPut(put);
+        Document doc = put.getDocument();
+        FieldValue f = doc.getFieldValue("actualMapStringToArrayOfInt");
+        assertSame(MapFieldValue.class, f.getClass());
+        MapFieldValue<?, ?> m = (MapFieldValue<?, ?>) f;
+        Array<?> a = (Array<?>) m.get(new StringFieldValue("bamse"));
+        assertEquals(3, a.size());
+        assertEquals(new IntegerFieldValue(1), a.get(0));
+        assertEquals(new IntegerFieldValue(2), a.get(1));
+        assertEquals(new IntegerFieldValue(3), a.get(2));
+    }
+
+    @Test
+    public final void testAssignToString() {
+        InputStream rawDoc = new ByteArrayInputStream(
+                Utf8.toBytes("{\"update\": \"id:unittest:smoke::whee\","
+                        + " \"fields\": { \"something\": {"
+                        + " \"assign\": \"orOther\" }}" + " }"));
+        JsonReader r = new JsonReader(types, rawDoc, parserFactory);
+        JsonReader.DocumentParseInfo parseInfo = r.parseDocument().get();
+        DocumentType docType = r.readDocumentType(parseInfo.documentId);
+        DocumentUpdate doc = new DocumentUpdate(docType, parseInfo.documentId);
+        r.readUpdate(doc);
+        FieldUpdate f = doc.getFieldUpdate("something");
+        assertEquals(1, f.size());
+        AssignValueUpdate a = (AssignValueUpdate) f.getValueUpdate(0);
+        assertEquals(new StringFieldValue("orOther"), a.getValue());
+    }
+
+    @Test
+    public final void testAssignToArray() {
+        InputStream rawDoc = new ByteArrayInputStream(
+                Utf8.toBytes("{\"update\": \"id:unittest:testMapStringToArrayOfInt::whee\","
+                        + " \"fields\": { \"actualMapStringToArrayOfInt\": {"
+                        + " \"assign\": ["
+                        + "{ \"key\": \"bamse\", \"value\": [1, 2, 3] }"
+                        + "]}}}"));
+        JsonReader r = new JsonReader(types, rawDoc, parserFactory);
+        JsonReader.DocumentParseInfo parseInfo = r.parseDocument().get();
+        DocumentType docType = r.readDocumentType(parseInfo.documentId);
+        DocumentUpdate doc = new DocumentUpdate(docType, parseInfo.documentId);
+        r.readUpdate(doc);
+        FieldUpdate f = doc.getFieldUpdate("actualMapStringToArrayOfInt");
+        assertEquals(1, f.size());
+        AssignValueUpdate assign = (AssignValueUpdate) f.getValueUpdate(0);
+        MapFieldValue<?, ?> m = (MapFieldValue<?, ?>) assign.getValue();
+        Array<?> a = (Array<?>) m.get(new StringFieldValue("bamse"));
+        assertEquals(3, a.size());
+        assertEquals(new IntegerFieldValue(1), a.get(0));
+        assertEquals(new IntegerFieldValue(2), a.get(1));
+        assertEquals(new IntegerFieldValue(3), a.get(2));
+    }
+
+    @Test
+    public final void testAssignToWeightedSet() {
+        InputStream rawDoc = new ByteArrayInputStream(
+                Utf8.toBytes("{\"update\": \"id:unittest:testset::whee\","
+                        + " \"fields\": { " + "\"actualset\": {"
+                        + " \"assign\": {"
+                        + " \"person\": 37,"
+                        + " \"another person\": 41}}}}"));
+        JsonReader r = new JsonReader(types, rawDoc, parserFactory);
+        JsonReader.DocumentParseInfo parseInfo = r.parseDocument().get();
+        DocumentType docType = r.readDocumentType(parseInfo.documentId);
+        DocumentUpdate doc = new DocumentUpdate(docType, parseInfo.documentId);
+        r.readUpdate(doc);
+        FieldUpdate x = doc.getFieldUpdate("actualset");
+        assertEquals(1, x.size());
+        AssignValueUpdate assign = (AssignValueUpdate) x.getValueUpdate(0);
+        WeightedSet<?> w = (WeightedSet<?>) assign.getValue();
+        assertEquals(2, w.size());
+        assertEquals(new Integer(37), w.get(new StringFieldValue("person")));
+        assertEquals(new Integer(41), w.get(new StringFieldValue("another person")));
+    }
+
+
+    @Test
+    public final void testCompleteFeed() {
+        InputStream rawDoc = new ByteArrayInputStream(
+                Utf8.toBytes("[{\"put\": \"id:unittest:smoke::whee\","
+                        + " \"fields\": { \"something\": \"smoketest\","
+                        + " \"nalle\": \"bamse\"}}" + ", "
+                        + "{\"update\": \"id:unittest:testarray::whee\","
+                        + " \"fields\": { " + "\"actualarray\": {"
+                        + " \"add\": [" + " \"person\","
+                        + " \"another person\"]}}}" + ", "
+                        + "{\"remove\": \"id:unittest:smoke::whee\"}]"));
+        JsonReader r = new JsonReader(types, rawDoc, parserFactory);
+
+        controlBasicFeed(r);
+    }
+
+    @Test
+    public final void testCompleteFeedWithCreateAndCondition() {
+        InputStream rawDoc = new ByteArrayInputStream(
+                Utf8.toBytes("[{\"put\": \"id:unittest:smoke::whee\","
+                        + " \"fields\": { \"something\": \"smoketest\","
+                        + " \"nalle\": \"bamse\"}}" + ", "
+                        + "{"
+                        +  "\"condition\":\"bla\","
+                        +  "\"update\": \"id:unittest:testarray::whee\","
+                        + " \"create\":true,"
+                        + " \"fields\": { " + "\"actualarray\": {"
+                        + " \"add\": [" + " \"person\","
+                        + " \"another person\"]}}}" + ", "
+                        + "{\"remove\": \"id:unittest:smoke::whee\"}]"));
+        JsonReader r = new JsonReader(types, rawDoc, parserFactory);
+
+        DocumentOperation d = r.next();
+        Document doc = ((DocumentPut) d).getDocument();
+        smokeTestDoc(doc);
+
+        d = r.next();
+        DocumentUpdate update = (DocumentUpdate) d;
+        checkSimpleArrayAdd(update);
+        assertThat(update.getCreateIfNonExistent(), is(true));
+        assertThat(update.getCondition().getSelection(), is("bla"));
+
+        d = r.next();
+        DocumentRemove remove = (DocumentRemove) d;
+        assertEquals("smoke", remove.getId().getDocType());
+
+        assertNull(r.next());
+    }
+
+    @Test
+    public final void testUpdateWithConditionAndCreateInDifferentOrdering() {
+        final int  documentsCreated = 106;
+        List<String> parts = Arrays.asList(
+                "\"condition\":\"bla\"",
+                "\"update\": \"id:unittest:testarray::whee\"",
+                " \"fields\": { " + "\"actualarray\": { \"add\": [" + " \"person\",\"another person\"]}}",
+                " \"create\":true");
+        final Random random = new Random(42);
+        StringBuilder documents = new StringBuilder("[");
+        for (int x = 0; x < documentsCreated; x++) {
+            Collections.shuffle(parts, random);
+            documents.append("{").append(Joiner.on(",").join(parts)).append("}");
+            if (x < documentsCreated -1) {
+                documents.append(",");
+            }
+        }
+        documents.append("]");
+        InputStream rawDoc = new ByteArrayInputStream(
+                Utf8.toBytes(documents.toString()));
+
+        JsonReader r = new JsonReader(types, rawDoc, parserFactory);
+
+        for (int x = 0; x < documentsCreated; x++) {
+            DocumentUpdate update = (DocumentUpdate) r.next();
+            checkSimpleArrayAdd(update);
+            assertThat(update.getCreateIfNonExistent(), is(true));
+            assertThat(update.getCondition().getSelection(), is("bla"));
+
+        }
+
+        assertNull(r.next());
+    }
+
+
+    @Test(expected=RuntimeException.class)
+    public final void testCreateIfNonExistentInPut() {
+        InputStream rawDoc = new ByteArrayInputStream(
+                Utf8.toBytes("[{"
+                        + " \"create\":true,"
+                        + " \"fields\": { \"something\": \"smoketest\","
+                        + " \"nalle\": \"bamse\"},"
+                        + "\"put\": \"id:unittest:smoke::whee\""
+                        + "}]"));
+        JsonReader r = new JsonReader(types, rawDoc, parserFactory);
+        r.next();
+    }
+
+    @Test
+    public final void testCompleteFeedWithIdAfterFields() {
+        InputStream rawDoc = new ByteArrayInputStream(
+                Utf8.toBytes("[{"
+                        + " \"fields\": { \"something\": \"smoketest\","
+                        + " \"nalle\": \"bamse\"},"
+                        + "\"put\": \"id:unittest:smoke::whee\""
+                        + "}" + ", "
+                        + "{"
+                        + " \"fields\": { " + "\"actualarray\": {"
+                        + " \"add\": [" + " \"person\","
+                        + " \"another person\"]}},"
+                        + "\"update\": \"id:unittest:testarray::whee\""
+                        + "}" + ", "
+                        + "{\"remove\": \"id:unittest:smoke::whee\"}]"));
+        JsonReader r = new JsonReader(types, rawDoc, parserFactory);
+
+        controlBasicFeed(r);
+    }
+
+    protected void controlBasicFeed(JsonReader r) {
+        DocumentOperation d = r.next();
+        Document doc = ((DocumentPut) d).getDocument();
+        smokeTestDoc(doc);
+
+        d = r.next();
+        DocumentUpdate update = (DocumentUpdate) d;
+        checkSimpleArrayAdd(update);
+
+        d = r.next();
+        DocumentRemove remove = (DocumentRemove) d;
+        assertEquals("smoke", remove.getId().getDocType());
+
+        assertNull(r.next());
+    }
+
+
+    @Test
+    public final void testCompleteFeedWithEmptyDoc() {
+        InputStream rawDoc = new ByteArrayInputStream(
+                Utf8.toBytes("[{\"put\": \"id:unittest:smoke::whee\","
+                        + " \"fields\": {}}" + ", "
+                        + "{\"update\": \"id:unittest:testarray::whee\","
+                        + " \"fields\": {}}" + ", "
+                        + "{\"remove\": \"id:unittest:smoke::whee\"}]"));
+        JsonReader r = new JsonReader(types, rawDoc, parserFactory);
+
+
+        DocumentOperation d = r.next();
+        Document doc = ((DocumentPut) d).getDocument();
+        assertEquals("smoke", doc.getId().getDocType());
+
+        d = r.next();
+        DocumentUpdate update = (DocumentUpdate) d;
+        assertEquals("testarray", update.getId().getDocType());
+
+        d = r.next();
+        DocumentRemove remove = (DocumentRemove) d;
+        assertEquals("smoke", remove.getId().getDocType());
+
+        assertNull(r.next());
+
+    }
+
+    private void checkSimpleArrayAdd(DocumentUpdate update) {
+        Set<String> toAdd = new HashSet<>();
+        FieldUpdate x = update.getFieldUpdate("actualarray");
+        for (ValueUpdate<?> v : x.getValueUpdates()) {
+            AddValueUpdate adder = (AddValueUpdate) v;
+            toAdd.add(((StringFieldValue) adder.getValue()).getString());
+        }
+        assertEquals(2, toAdd.size());
+        assertTrue(toAdd.contains("person"));
+        assertTrue(toAdd.contains("another person"));
+    }
+
+    private void smokeTestDoc(Document doc) {
+        FieldValue f = doc.getFieldValue(doc.getField("nalle"));
+        assertSame(StringFieldValue.class, f.getClass());
+        StringFieldValue s = (StringFieldValue) f;
+        assertEquals("bamse", s.getString());
+    }
+
+    @Test
+    public final void misspelledFieldTest() {
+        InputStream rawDoc = new ByteArrayInputStream(
+                Utf8.toBytes("{\"put\": \"id:unittest:smoke::whee\","
+                        + " \"fields\": { \"smething\": \"smoketest\","
+                        + " \"nalle\": \"bamse\"}}"));
+        JsonReader r = new JsonReader(types, rawDoc, parserFactory);
+        JsonReader.DocumentParseInfo parseInfo = r.parseDocument().get();
+        DocumentType docType = r.readDocumentType(parseInfo.documentId);
+        DocumentPut put = new DocumentPut(new Document(docType, parseInfo.documentId));
+        exception.expect(NullPointerException.class);
+        exception.expectMessage("Could not get field \"smething\" in the structure of type \"smoke\".");
+        r.readPut(put);
+    }
+
+    @Test
+    public final void feedWithBasicErrorTest() {
+        InputStream rawDoc = new ByteArrayInputStream(
+                Utf8.toBytes("["
+                        + "  { \"put\": \"id:test:smoke::0\", \"fields\": { \"something\": \"foo\" } },"
+                        + "  { \"put\": \"id:test:smoke::1\", \"fields\": { \"something\": \"foo\" } },"
+                        + "  { \"put\": \"id:test:smoke::2\", \"fields\": { \"something\": \"foo\" } },"
+                        + "]"));
+        JsonReader r = new JsonReader(types, rawDoc, parserFactory);
+        exception.expect(RuntimeException.class);
+        exception.expectMessage("JsonParseException");
+        while (r.next() != null);
+    }
+
+    @Test
+    public final void idAsAliasForPutTest() {
+        InputStream rawDoc = new ByteArrayInputStream(
+                Utf8.toBytes("{\"id\": \"id:unittest:smoke::whee\","
+                        + " \"fields\": { \"something\": \"smoketest\","
+                        + " \"nalle\": \"bamse\"}}"));
+        JsonReader r = new JsonReader(types, rawDoc, parserFactory);
+        JsonReader.DocumentParseInfo parseInfo = r.parseDocument().get();
+        DocumentType docType = r.readDocumentType(parseInfo.documentId);
+        DocumentPut put = new DocumentPut(new Document(docType, parseInfo.documentId));
+        r.readPut(put);
+        smokeTestDoc(put.getDocument());
+    }
+
+    private void testFeedWithTestAndSetCondition(String jsonDoc) {
+        final ByteArrayInputStream parseInfoDoc = new ByteArrayInputStream(Utf8.toBytes(jsonDoc));
+        final JsonReader reader = new JsonReader(types, parseInfoDoc, parserFactory);
+        final int NUM_OPERATIONS_IN_FEED = 3;
+
+        for (int i = 0; i < NUM_OPERATIONS_IN_FEED; i++) {
+            DocumentOperation operation = reader.next();
+
+            assertTrue("A test and set condition should be present",
+                    operation.getCondition().isPresent());
+
+            assertEquals("DocumentOperation's test and set condition should be equal to the one in the JSON feed",
+                    "smoke.something == \"smoketest\"",
+                    operation.getCondition().getSelection());
+        }
+
+        assertNull(reader.next());
+    }
+
+    @Test
+    public final void testFeedWithTestAndSetConditionOrderingOne() {
+        testFeedWithTestAndSetCondition(
+                inputJson("[",
+                        "      {",
+                        "          'put': 'id:unittest:smoke::whee',",
+                        "          'condition': 'smoke.something == \\'smoketest\\'',",
+                        "          'fields': {",
+                        "              'something': 'smoketest',",
+                        "              'nalle': 'bamse'",
+                        "          }",
+                        "      },",
+                        "      {",
+                        "          'update': 'id:unittest:testarray::whee',",
+                        "          'condition': 'smoke.something == \\'smoketest\\'',",
+                        "          'fields': {",
+                        "              'actualarray': {",
+                        "                  'add': [",
+                        "                      'person',",
+                        "                      'another person'",
+                        "                   ]",
+                        "              }",
+                        "          }",
+                        "      },",
+                        "      {",
+                        "          'remove': 'id:unittest:smoke::whee',",
+                        "          'condition': 'smoke.something == \\'smoketest\\''",
+                        "      }",
+                        "]"
+                ));
+    }
+
+    @Test
+    public final void testFeedWithTestAndSetConditionOrderingTwo() {
+        testFeedWithTestAndSetCondition(
+                inputJson("[",
+                        "      {",
+                        "          'condition': 'smoke.something == \\'smoketest\\'',",
+                        "          'put': 'id:unittest:smoke::whee',",
+                        "          'fields': {",
+                        "              'something': 'smoketest',",
+                        "              'nalle': 'bamse'",
+                        "          }",
+                        "      },",
+                        "      {",
+                        "          'condition': 'smoke.something == \\'smoketest\\'',",
+                        "          'update': 'id:unittest:testarray::whee',",
+                        "          'fields': {",
+                        "              'actualarray': {",
+                        "                  'add': [",
+                        "                      'person',",
+                        "                      'another person'",
+                        "                   ]",
+                        "              }",
+                        "          }",
+                        "      },",
+                        "      {",
+                        "          'condition': 'smoke.something == \\'smoketest\\'',",
+                        "          'remove': 'id:unittest:smoke::whee'",
+                        "      }",
+                        "]"
+                ));
+    }
+
+    @Test
+    public final void testFeedWithTestAndSetConditionOrderingThree() {
+        testFeedWithTestAndSetCondition(
+                inputJson("[",
+                        "      {",
+                        "          'put': 'id:unittest:smoke::whee',",
+                        "          'fields': {",
+                        "              'something': 'smoketest',",
+                        "              'nalle': 'bamse'",
+                        "          },",
+                        "          'condition': 'smoke.something == \\'smoketest\\''",
+                        "      },",
+                        "      {",
+                        "          'update': 'id:unittest:testarray::whee',",
+                        "          'fields': {",
+                        "              'actualarray': {",
+                        "                  'add': [",
+                        "                      'person',",
+                        "                      'another person'",
+                        "                   ]",
+                        "              }",
+                        "          },",
+                        "          'condition': 'smoke.something == \\'smoketest\\''",
+                        "      },",
+                        "      {",
+                        "          'remove': 'id:unittest:smoke::whee',",
+                        "          'condition': 'smoke.something == \\'smoketest\\''",
+                        "      }",
+                        "]"
+                ));
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testInvalidFieldAfterFieldsFieldShouldFailParse() {
+        final String jsonData = inputJson(
+                "[",
+                "      {",
+                "          'put': 'id:unittest:smoke::whee',",
+                "          'fields': {",
+                "              'something': 'smoketest',",
+                "              'nalle': 'bamse'",
+                "          },",
+                "          'bjarne': 'stroustrup'",
+                "      }",
+                "]");
+
+        new JsonReader(types, jsonToInputStream(jsonData), parserFactory).next();
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testInvalidFieldBeforeFieldsFieldShouldFailParse() {
+        final String jsonData = inputJson(
+                "[",
+                "      {",
+                "          'update': 'id:unittest:testarray::whee',",
+                "          'what is this': 'nothing to see here',",
+                "          'fields': {",
+                "              'actualarray': {",
+                "                  'add': [",
+                "                      'person',",
+                "                      'another person'",
+                "                   ]",
+                "              }",
+                "          }",
+                "      }",
+                "]");
+
+        new JsonReader(types, jsonToInputStream(jsonData), parserFactory).next();
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testInvalidFieldWithoutFieldsFieldShouldFailParse() {
+        final String jsonData = inputJson(
+                "[",
+                "      {",
+                "          'remove': 'id:unittest:smoke::whee',",
+                "          'what is love': 'baby, do not hurt me... much'",
+                "      }",
+                "]");
+
+        new JsonReader(types, jsonToInputStream(jsonData), parserFactory).next();
+    }
+
+    static ByteArrayInputStream jsonToInputStream(String json) {
+        return new ByteArrayInputStream(Utf8.toBytes(json));
+    }
+
+    /**
+     * Convenience method to input JSON without escaping double quotes and newlines
+     * Each parameter represents a line of JSON encoded data
+     * The lines are joined with newline and single quotes are replaced with double quotes
+     */
+    static String inputJson(String... lines) {
+        return Joiner.on("\n").join(lines).replaceAll("'", "\"");
+    }
+
+    @Test
+    public void testParsingWithoutTensorField() {
+        Document doc = createPutWithoutTensor().getDocument();
+        assertEquals("testtensor", doc.getId().getDocType());
+        assertEquals("id:unittest:testtensor::0", doc.getId().toString());
+        TensorFieldValue fieldValue = (TensorFieldValue)doc.getFieldValue(doc.getField("tensorfield"));
+        assertNull(fieldValue);
+    }
+
+    @Test
+    public void testParsingOfEmptyTensor() {
+        assertTensorField("{}", createPutWithTensor("{}"));
+    }
+
+    @Test
+    public void testParsingOfTensorWithEmptyDimensions() {
+        assertTensorField("{}",
+                createPutWithTensor("{ "
+                        + "  \"dimensions\": [] "
+                        + "}"));
+    }
+
+    @Test
+    public void testParsingOfTensorWithEmptyCells() {
+        assertTensorField("{}",
+                createPutWithTensor("{ "
+                        + "  \"cells\": [] "
+                        + "}"));
+    }
+
+    @Test
+    public void testParsingOfTensorWithDimensions() {
+        assertTensorField("( {{x:-,y:-}:1.0} * {} )",
+                createPutWithTensor("{ "
+                        + "  \"dimensions\": [\"x\",\"y\"] "
+                        + "}"));
+    }
+
+    @Test
+    public void testParsingOfTensorWithCells() {
+        assertTensorField("{{x:a,y:b}:2.0,{x:c}:3.0}}",
+                createPutWithTensor("{ "
+                        + "  \"cells\": [ "
+                        + "    { \"address\": { \"x\": \"a\", \"y\": \"b\" }, "
+                        + "      \"value\": 2.0 }, "
+                        + "    { \"address\": { \"x\": \"c\" }, "
+                        + "      \"value\": 3.0 } "
+                        + "  ]"
+                        + "}"));
+    }
+
+    @Test
+    public void testParsingOfTensorWithSingleCellInDifferentJsonOrder() {
+        assertTensorField("{{x:a,y:b}:2.0}",
+                createPutWithTensor("{ "
+                        + "  \"cells\": [ "
+                        + "    { \"value\": 2.0, "
+                        + "      \"address\": { \"x\": \"a\", \"y\": \"b\" } } "
+                        + "  ]"
+                        + "}"));
+    }
+
+    @Test
+    public void testParsingOfTensorWithSingleCellWithoutAddress() {
+        assertTensorField("{{}:2.0}",
+                createPutWithTensor("{ "
+                        + "  \"cells\": [ "
+                        + "    { \"value\": 2.0 } "
+                        + "  ]"
+                        + "}"));
+    }
+
+    @Test
+    public void testParsingOfTensorWithSingleCellWithoutValue() {
+        assertTensorField("{{x:a}:0.0}",
+                createPutWithTensor("{ "
+                        + "  \"cells\": [ "
+                        + "    { \"address\": { \"x\": \"a\" } } "
+                        + "  ]"
+                        + "}"));
+    }
+
+    @Test
+    public void testParsingOfTensorWithDimensionsAndCells() {
+        assertTensorField("( {{z:-}:1.0} * {{x:a,y:b}:2.0,{x:c}:3.0} )",
+                createPutWithTensor("{ "
+                        + "  \"dimensions\": [\"x\",\"y\",\"z\"], "
+                        + "  \"cells\": [ "
+                        + "    { \"address\": { \"x\": \"a\", \"y\": \"b\" }, "
+                        + "      \"value\": 2.0 }, "
+                        + "    { \"address\": { \"x\": \"c\" }, "
+                        + "      \"value\": 3.0 } "
+                        + "  ]"
+                        + "}"));
+    }
+
+    @Test
+    public void testParsingOfTensorWithDimensionsAndCellsInDifferentJsonOrder() {
+        assertTensorField("( {{z:-}:1.0} * {{x:a,y:b}:2.0,{x:c}:3.0} )",
+                createPutWithTensor("{ "
+                        + "  \"cells\": [ "
+                        + "    { \"address\": { \"x\": \"a\", \"y\": \"b\" }, "
+                        + "      \"value\": 2.0 }, "
+                        + "    { \"address\": { \"x\": \"c\" }, "
+                        + "      \"value\": 3.0 } "
+                        + "  ],"
+                        + "  \"dimensions\": [\"x\",\"y\",\"z\"] "
+                        + "}"));
+    }
+
+    @Test
+    public void require_that_parser_propagates_datatype_parser_errors_predicate() {
+        assertParserErrorMatches(
+                "Error in document 'id:unittest:testpredicate::0' - could not parse field 'boolean' of type 'predicate': " +
+                        "line 1:10 no viable alternative at character '>'",
+
+                "[",
+                "      {",
+                "          'fields': {",
+                "              'boolean': 'timestamp > 9000'",
+                "          },",
+                "          'put': 'id:unittest:testpredicate::0'",
+                "      }",
+                "]"
+        );
+    }
+
+    @Test
+    public void require_that_parser_propagates_datatype_parser_errors_string_as_int() {
+        assertParserErrorMatches(
+                "Error in document 'id:unittest:testint::0' - could not parse field 'integerfield' of type 'int': " +
+                        "For input string: \" 1\"",
+
+                "[",
+                "      {",
+                "          'fields': {",
+                "              'integerfield': ' 1'",
+                "          },",
+                "          'put': 'id:unittest:testint::0'",
+                "      }",
+                "]"
+        );
+    }
+
+    @Test
+    public void require_that_parser_propagates_datatype_parser_errors_overflowing_int() {
+        assertParserErrorMatches(
+                "Error in document 'id:unittest:testint::0' - could not parse field 'integerfield' of type 'int': " +
+                        "For input string: \"281474976710656\"",
+
+                "[",
+                "      {",
+                "          'fields': {",
+                "              'integerfield': 281474976710656",
+                "          },",
+                "          'put': 'id:unittest:testint::0'",
+                "      }",
+                "]"
+        );
+    }
+
+    @Test
+    public void requireThatUpdatesForTensorFieldsAreNotSupported() {
+        try {
+            InputStream rawDoc = new ByteArrayInputStream(
+                    Utf8.toBytes("[ { \"update\": \"" + TENSOR_DOC_ID + "\", \"fields\": { \"tensorfield\": {"
+                            + "\"assign\": {} } } } ]"));
+            new JsonReader(types, rawDoc, parserFactory).next();
+            assertTrue("Exception not thrown", false);
+        } catch (IllegalArgumentException e) {
+            assertEquals("Updates to fields of type TENSOR is not yet supported (id='id:unittest:testtensor::0', field='tensorfield')",
+                    e.getMessage());
+        }
+    }
+
+    @Test
+    public void requireThatUnknownDocTypeThrowsIllegalArgumentException() {
+        exception.expect(IllegalArgumentException.class);
+        exception.expectMessage(new Contains("Document type walrus does not exist"));
+
+        final String jsonData = inputJson(
+                "[",
+                "      {",
+                "          'put': 'id:ns:walrus::walrus1',",
+                "          'fields': {",
+                "              'aField': 42",
+                "          }",
+                "      }",
+                "]");
+
+        new JsonReader(types, jsonToInputStream(jsonData), parserFactory).next();
+    }
+
+    private static final String TENSOR_DOC_ID = "id:unittest:testtensor::0";
+
+    private DocumentPut createPutWithoutTensor() {
+        InputStream rawDoc = new ByteArrayInputStream(
+                Utf8.toBytes("[ { \"put\": \"" + TENSOR_DOC_ID + "\", \"fields\": { } } ]"));
+        JsonReader reader = new JsonReader(types, rawDoc, parserFactory);
+        return (DocumentPut) reader.next();
+    }
+
+    private DocumentPut createPutWithTensor(String inputTensor) {
+        InputStream rawDoc = new ByteArrayInputStream(
+                Utf8.toBytes("["
+                        + "  { \"put\": \"" + TENSOR_DOC_ID + "\", \"fields\": { \"tensorfield\": "
+                        + inputTensor
+                        + "  }}"
+                        + "]"));
+        JsonReader reader = new JsonReader(types, rawDoc, parserFactory);
+        return (DocumentPut) reader.next();
+    }
+
+    private static void assertTensorField(String expectedTensor, DocumentPut put) {
+        final Document doc = put.getDocument();
+        assertEquals("testtensor", doc.getId().getDocType());
+        assertEquals(TENSOR_DOC_ID, doc.getId().toString());
+        TensorFieldValue fieldValue = (TensorFieldValue)doc.getFieldValue(doc.getField("tensorfield"));
+        assertEquals(MapTensor.from(expectedTensor), fieldValue.getTensor().get());
+    }
+
+    // NOTE: Do not call this method multiple times from a test method as it's using the ExpectedException rule
+    private void assertParserErrorMatches(String expectedError, String... json) {
+        exception.expect(JsonReaderException.class);
+        exception.expectMessage(new Contains(expectedError));
+        String jsonData = inputJson(json);
+        new JsonReader(types, jsonToInputStream(jsonData), parserFactory).next();
+    }
+
+}
diff --git a/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java b/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java
new file mode 100644
index 00000000000..2e8354e3c6a
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java
@@ -0,0 +1,384 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.json;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertSame;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import com.yahoo.document.ArrayDataType;
+import com.yahoo.document.DataType;
+import com.yahoo.document.Document;
+import com.yahoo.document.DocumentId;
+import com.yahoo.document.DocumentOperation;
+import com.yahoo.document.DocumentPut;
+import com.yahoo.document.DocumentRemove;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.DocumentTypeManager;
+import com.yahoo.document.Field;
+import com.yahoo.document.MapDataType;
+import com.yahoo.document.PositionDataType;
+import com.yahoo.document.StructDataType;
+import com.yahoo.document.WeightedSetDataType;
+import com.yahoo.document.datatypes.TensorFieldValue;
+import org.apache.commons.codec.binary.Base64;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonParseException;
+import com.fasterxml.jackson.core.io.JsonStringEncoder;
+import com.fasterxml.jackson.databind.JsonMappingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.yahoo.text.Utf8;
+
+/**
+ * Functional tests for com.yahoo.document.json.JsonWriter.
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+public class JsonWriterTestCase {
+
+    private static final JsonFactory parserFactory = new JsonFactory();
+    private DocumentTypeManager types;
+
+    @Before
+    public void setUp() throws Exception {
+        types = new DocumentTypeManager();
+        {
+            DocumentType x = new DocumentType("smoke");
+            x.addField(new Field("something", DataType.STRING));
+            x.addField(new Field("nalle", DataType.STRING));
+            types.registerDocumentType(x);
+        }
+        {
+            DocumentType x = new DocumentType("mirrors");
+            StructDataType woo = new StructDataType("woo");
+            woo.addField(new Field("sandra", DataType.STRING));
+            woo.addField(new Field("cloud", DataType.STRING));
+            x.addField(new Field("skuggsjaa", woo));
+            types.registerDocumentType(x);
+        }
+        {
+            DocumentType x = new DocumentType("testarray");
+            DataType d = new ArrayDataType(DataType.STRING);
+            x.addField(new Field("actualarray", d));
+            types.registerDocumentType(x);
+        }
+        {
+            DocumentType x = new DocumentType("testset");
+            DataType d = new WeightedSetDataType(DataType.STRING, true, true);
+            x.addField(new Field("actualset", d));
+            types.registerDocumentType(x);
+        }
+        {
+            DocumentType x = new DocumentType("testmap");
+            DataType d = new MapDataType(DataType.STRING, DataType.STRING);
+            x.addField(new Field("actualmap", d));
+            types.registerDocumentType(x);
+        }
+        {
+            DocumentType x = new DocumentType("testraw");
+            DataType d = DataType.RAW;
+            x.addField(new Field("actualraw", d));
+            types.registerDocumentType(x);
+        }
+        {
+            DocumentType x = new DocumentType("testpredicate");
+            DataType d = DataType.PREDICATE;
+            x.addField(new Field("actualpredicate", d));
+            types.registerDocumentType(x);
+        }
+        {
+            DocumentType x = new DocumentType("testMapStringToArrayOfInt");
+            DataType value = new ArrayDataType(DataType.INT);
+            DataType d = new MapDataType(DataType.STRING, value);
+            x.addField(new Field("actualMapStringToArrayOfInt", d));
+            types.registerDocumentType(x);
+        }
+        {
+            DocumentType x = new DocumentType("testsinglepos");
+            DataType d = PositionDataType.INSTANCE;
+            x.addField(new Field("singlepos", d));
+            types.registerDocumentType(x);
+        }
+        {
+            DocumentType x = new DocumentType("testmultipos");
+            DataType d = new ArrayDataType(PositionDataType.INSTANCE);
+            x.addField(new Field("multipos", d));
+            types.registerDocumentType(x);
+        }
+        {
+            DocumentType x = new DocumentType("testtensor");
+            x.addField(new Field("tensorfield", DataType.TENSOR));
+            types.registerDocumentType(x);
+        }
+    }
+
+    @After
+    public void tearDown() throws Exception {
+        types = null;
+    }
+
+    @Test
+    public final void smokeTest() throws JsonParseException,
+            JsonMappingException, IOException {
+        roundTripEquality("id:unittest:smoke::whee", "{"
+                + " \"something\": \"smoketest\"," + " \"nalle\": \"bamse\""
+                + "}");
+    }
+
+    @Test
+    public final void hideEmptyStringsTest() throws JsonParseException,
+            JsonMappingException, IOException {
+        final String fields = "{"
+                        + " \"something\": \"\"," + " \"nalle\": \"bamse\""
+                        + "}";
+        final String filteredFields = "{"
+                + " \"nalle\": \"bamse\""
+                + "}";
+
+        Document doc = readDocumentFromJson("id:unittest:smoke::whee", fields);
+        assertEqualJson(asDocument("id:unittest:smoke::whee", filteredFields), JsonWriter.toByteArray(doc));
+    }
+
+    private void roundTripEquality(final String docId, final String fields)
+            throws JsonParseException, JsonMappingException, IOException {
+        Document doc = readDocumentFromJson(docId, fields);
+        assertEqualJson(asDocument(docId, fields), JsonWriter.toByteArray(doc));
+    }
+
+    @Test
+    public final void structTest() throws JsonParseException,
+            JsonMappingException, IOException {
+        roundTripEquality("id:unittest:mirrors::whee", "{ "
+                + "\"skuggsjaa\": {" + "\"sandra\": \"person\","
+                + " \"cloud\": \"another person\"}}");
+    }
+
+    @Test
+    public final void singlePosTest() throws JsonParseException,
+            JsonMappingException, IOException {
+        roundTripEquality("id:unittest:testsinglepos::bamf", "{ \"singlepos\": \"N60.222333;E10.12\" }");
+    }
+
+    @Test
+    public final void multiPosTest() throws JsonParseException,
+            JsonMappingException, IOException {
+        roundTripEquality("id:unittest:testmultipos::bamf", "{ \"multipos\": [ \"N0.0;E0.0\", \"S1.1;W1.1\", \"N10.2;W122.2\" ] }");
+    }
+
+    @Test
+    public final void arrayTest() throws JsonParseException,
+            JsonMappingException, IOException {
+        roundTripEquality("id:unittest:testarray::whee", "{ \"actualarray\": ["
+                + " \"nalle\"," + " \"tralle\"]}");
+    }
+
+    @Test
+    public final void weightedSetTest() throws JsonParseException,
+            JsonMappingException, IOException {
+        roundTripEquality("id:unittest:testset::whee", "{ \"actualset\": {"
+                + " \"nalle\": 2," + " \"tralle\": 7 }}");
+    }
+
+    @Test
+    public final void mapTest() throws JsonParseException,
+            JsonMappingException, IOException {
+        final String fields = "{ \"actualmap\": ["
+                        + " { \"key\": \"nalle\", \"value\": \"kalle\"},"
+                        + " { \"key\": \"tralle\", \"value\": \"skalle\"} ]}";
+        final String docId = "id:unittest:testmap::whee";
+        Document doc = readDocumentFromJson(docId, fields);
+        // we have to do everything by hand to check, as maps are unordered, but
+        // are serialized as an ordered structure
+
+        ObjectMapper m = new ObjectMapper();
+        Map<?, ?> generated = m.readValue(JsonWriter.toByteArray(doc), Map.class);
+        assertEquals(docId, generated.get("id"));
+        // and from here on down there will be lots of unchecked casting and
+        // other fun. This is OK here, because if the casts fail, the should and
+        // will fail anyway
+        List<?> inputMap = (List<?>) m.readValue(Utf8.toBytes(fields), Map.class).get("actualmap");
+        List<?> generatedMap = (List<?>) ((Map<?, ?>) generated.get("fields")).get("actualmap");
+        assertEquals(populateMap(inputMap), populateMap(generatedMap));
+    }
+
+    // should very much blow up if the assumptions are incorrect
+    @SuppressWarnings("rawtypes")
+    private Map<Object, Object> populateMap(List<?> actualMap) {
+        Map<Object, Object> m = new HashMap<>();
+        for (Object o : actualMap) {
+            Object key = ((Map) o).get(JsonReader.MAP_KEY);
+            Object value = ((Map) o).get(JsonReader.MAP_VALUE);
+            m.put(key, value);
+        }
+        return m;
+    }
+
+    @Test
+    public final void rawTest() throws JsonParseException, JsonMappingException, IOException {
+        String payload = new String(
+                new JsonStringEncoder().quoteAsString(new Base64()
+                        .encodeToString(Utf8.toBytes("smoketest"))));
+        String docId = "id:unittest:testraw::whee";
+
+        String fields = "{ \"actualraw\": \"" + payload + "\"" + " }";
+        roundTripEquality(docId, fields);
+    }
+
+    @Test
+    public final void predicateTest() throws JsonParseException,
+            JsonMappingException, IOException {
+        roundTripEquality("id:unittest:testpredicate::whee", "{ "
+                + "\"actualpredicate\": \"foo in [bar]\" }");
+    }
+
+    @Test
+    public final void stringToArrayOfIntMapTest() throws JsonParseException,
+            JsonMappingException, IOException {
+        String docId = "id:unittest:testMapStringToArrayOfInt::whee";
+        String fields = "{ \"actualMapStringToArrayOfInt\": ["
+                + "{ \"key\": \"bamse\", \"value\": [1, 2, 3] }" + "]}";
+        Document doc = readDocumentFromJson(docId, fields);
+        // we have to do everything by hand to check, as maps are unordered, but
+        // are serialized as an ordered structure
+
+        ObjectMapper m = new ObjectMapper();
+        Map<?, ?> generated = m.readValue(JsonWriter.toByteArray(doc), Map.class);
+        assertEquals(docId, generated.get("id"));
+        // and from here on down there will be lots of unchecked casting and
+        // other fun. This is OK here, because if the casts fail, the should and
+        // will fail anyway
+        List<?> inputMap = (List<?>) m.readValue(Utf8.toBytes(fields), Map.class).get("actualMapStringToArrayOfInt");
+        List<?> generatedMap = (List<?>) ((Map<?, ?>) generated.get("fields")).get("actualMapStringToArrayOfInt");
+        assertEquals(populateMap(inputMap), populateMap(generatedMap));
+    }
+
+    private Document readDocumentFromJson(final String docId,
+            final String fields) {
+        InputStream rawDoc = new ByteArrayInputStream(asFeed(
+                docId, fields));
+        JsonReader r = new JsonReader(types, rawDoc, parserFactory);
+        JsonReader.DocumentParseInfo raw = r.parseDocument().get();
+        DocumentType docType = r.readDocumentType(raw.documentId);
+        DocumentPut put = new DocumentPut(new Document(docType, raw.documentId));
+        r.readPut(put);
+        return put.getDocument();
+    }
+
+    private void assertEqualJson(byte[] expected, byte[] generated)
+            throws JsonParseException, JsonMappingException, IOException {
+        ObjectMapper m = new ObjectMapper();
+        Map<?, ?> exp = m.readValue(expected, Map.class);
+        Map<?, ?> gen = m.readValue(generated, Map.class);
+        assertEquals(exp, gen);
+    }
+
+    private byte[] asFeed(String docId, String fields) {
+        return completeDocString("put", docId, fields);
+    }
+
+    private byte[] asDocument(String docId, String fields) {
+        return completeDocString("id", docId, fields);
+    }
+
+    private byte[] completeDocString(String operation, String docId, String fields) {
+        return Utf8.toBytes("{\"" + operation + "\": \"" + docId + "\", \"fields\": " + fields + "}");
+    }
+
+    @Test
+    public void removeTest() {
+        final DocumentId documentId = new DocumentId("id:unittest:smoke::whee");
+        InputStream rawDoc = new ByteArrayInputStream(
+                Utf8.toBytes("["
+                + Utf8.toString(JsonWriter.documentRemove(documentId))
+                + "]"));
+        JsonReader r = new JsonReader(types, rawDoc, parserFactory);
+        DocumentOperation actualRemoveAsBaseType = r.next();
+        assertSame(DocumentRemove.class, actualRemoveAsBaseType.getClass());
+        assertEquals(actualRemoveAsBaseType.getId(), documentId);
+    }
+
+    @Test
+    public void testWritingWithoutTensorFieldValue() throws IOException {
+        roundTripEquality("id:unittest:testtensor::0", "{}");
+    }
+
+    @Test
+    public void testWritingOfEmptyTensor() throws IOException {
+        assertTensorRoundTripEquality("{}",
+                "{ \"dimensions\": [], \"cells\": [] }");
+    }
+
+    @Test
+    public void testWritingOfTensorWithCellsOnly() throws IOException {
+        assertTensorRoundTripEquality("{ "
+                + "  \"cells\": [ "
+                + "    { \"address\": { \"x\": \"a\", \"y\": \"b\" }, "
+                + "      \"value\": 2.0 }, "
+                + "    { \"address\": { \"x\": \"c\" }, "
+                + "      \"value\": 3.0 } "
+                + "  ]"
+                + "}", "{ "
+                + "  \"dimensions\": [\"x\", \"y\"], "
+                + "  \"cells\": [ "
+                + "    { \"address\": { \"x\": \"a\", \"y\": \"b\" }, "
+                + "      \"value\": 2.0 }, "
+                + "    { \"address\": { \"x\": \"c\" }, "
+                + "      \"value\": 3.0 } "
+                + "  ]"
+                + "}");
+    }
+
+    @Test
+    public void testWritingOfTensorWithSingleCellWithEmptyAddress() throws IOException {
+        assertTensorRoundTripEquality("{ "
+                + "  \"dimensions\": [], "
+                + "  \"cells\": [ "
+                + "    { \"address\": {}, \"value\": 2.0 } "
+                + "  ]"
+                + "}");
+    }
+
+    @Test
+    public void testWritingOfTensorWithDimensionsAndCells() throws IOException {
+        assertTensorRoundTripEquality("{ "
+                + "  \"dimensions\": [\"x\",\"y\",\"z\"], "
+                + "  \"cells\": [ "
+                + "    { \"address\": { \"x\": \"a\", \"y\": \"b\" }, "
+                + "      \"value\": 2.0 }, "
+                + "    { \"address\": { \"x\": \"c\" }, "
+                + "      \"value\": 3.0 } "
+                + "  ]"
+                + "}");
+    }
+
+    @Test
+    public void testWritingOfTensorFieldValueWithoutTensor() throws IOException {
+        DocumentType tensorType = types.getDocumentType("testtensor");
+        String docId = "id:unittest:testtensor::0";
+        Document doc = new Document(tensorType, docId);
+        doc.setFieldValue(tensorType.getField("tensorfield"), new TensorFieldValue());
+        assertEqualJson(asDocument(docId, "{ \"tensorfield\": {} }"), JsonWriter.toByteArray(doc));
+    }
+
+    private void assertTensorRoundTripEquality(String tensorField) throws IOException {
+        assertTensorRoundTripEquality(tensorField, tensorField);
+    }
+
+    private void assertTensorRoundTripEquality(String inputTensorField, String outputTensorField) throws IOException {
+        String inputFields = "{ \"tensorfield\": " + inputTensorField + " }";
+        String outputFields = "{ \"tensorfield\": " + outputTensorField + " }";
+        String docId = "id:unittest:testtensor::0";
+        Document doc = readDocumentFromJson(docId, inputFields);
+        assertEqualJson(asDocument(docId, outputFields), JsonWriter.toByteArray(doc));
+    }
+
+}
diff --git a/document/src/test/java/com/yahoo/document/outerdoc.sd b/document/src/test/java/com/yahoo/document/outerdoc.sd
new file mode 100644
index 00000000000..dbdf3c32cd5
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/outerdoc.sd
@@ -0,0 +1,6 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+search outerdoc {
+  document outerdoc {
+    field innerdocuments type array<docindoc> { body }
+  }
+}
diff --git a/document/src/test/java/com/yahoo/document/select/BucketSelectorTestCase.java b/document/src/test/java/com/yahoo/document/select/BucketSelectorTestCase.java
new file mode 100644
index 00000000000..80282255dc5
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/select/BucketSelectorTestCase.java
@@ -0,0 +1,79 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.select;
+
+import com.yahoo.document.BucketId;
+import com.yahoo.document.BucketIdFactory;
+import com.yahoo.document.select.BucketSelector;
+
+import java.util.Set;
+import java.util.TreeSet;
+
+/**
+ * Date: Sep 6, 2007
+ *
+ * @author <a href="mailto:humbe@yahoo-inc.com">H&aring;kon Humberset</a>
+ */
+public class BucketSelectorTestCase extends junit.framework.TestCase {
+
+    public BucketSelectorTestCase(String name) {
+        super(name);
+    }
+
+    public void testExpressions() throws Exception {
+        assertBucketCount("id = \"userdoc:ns:123:foobar\"", 1);
+        assertBucketCount("id = \"userdoc:ns:123:foo*\"", 0);
+        assertBucketCount("id == \"userdoc:ns:123:f?oo*\"", 1);
+        assertBucketCount("id =~ \"userdoc:ns:123:foo*\"", 0);
+        assertBucketCount("id =~ \"userdoc:ns:123:foo?\"", 0);
+        assertBucketCount("id.user = 123", 1);
+        assertBucketCount("id.user == 123", 1);
+        assertBucketCount("id.group = \"yahoo.com\"", 1);
+        assertBucketCount("id.group = \"yahoo.com\" or id.user=123", 2);
+        assertBucketCount("id.group = \"yahoo.com\" and id.user=123", 0);
+        assertBucketCount("id.group = \"yahoo.com\" and testdoctype1.hstringval=\"Doe\"", 1);
+        assertBucketCount("not id.group = \"yahoo.com\"", 0);
+        assertBucketCount("id.group != \"yahoo.com\"", 0);
+        assertBucketCount("id.group <= \"yahoo.com\"", 0);
+
+        assertBucketCount("id.bucket = 0x4000000000003018", 1); // Bucket 16:12312
+        assertBucketCount("id.bucket == 0x4000000000000258", 1); // Bucket 16:600
+
+        assertBucketCount("searchcolumn.3 == 1", 21845);
+
+        // Check that the correct buckets is found
+        assertBucket("id.bucket = 0x4000000000003018", new BucketId(16, 12312));
+        assertBucket("id.bucket == 0x4000000000000258", new BucketId(16, 600));
+
+        assertBucket("id = \"userdoc:ns:123:foobar\"", new BucketId(0xeafff5320000007bl));
+        assertBucket("id.user = 123", new BucketId(32, 123));
+        assertBucket("id.group = \"yahoo.com\"", new BucketId(32, 0x035837189a1acd50l));
+
+        // Check that overlapping works
+        Set<BucketId> expected = new TreeSet<BucketId>();
+        expected.add(new BucketId(32, 123));
+        expected.add(new BucketId(16, 123));
+        assertBuckets("id.user == 123 or id.bucket == 0x400000000000007b", expected);
+    }
+
+    public void assertBucketCount(String expr, int count) throws Exception {
+        BucketIdFactory factory = new BucketIdFactory();
+        BucketSelector selector = new BucketSelector(factory);
+        Set<BucketId> buckets = selector.getBucketList(expr);
+        assertEquals(count, buckets == null ? 0 : buckets.size());
+    }
+
+    public void assertBucket(String expr, BucketId bucket) throws Exception {
+        BucketIdFactory factory = new BucketIdFactory();
+        BucketSelector selector = new BucketSelector(factory);
+        Set<BucketId> buckets = selector.getBucketList(expr);
+        assertEquals(1, buckets == null ? 0 : buckets.size());
+        assertEquals(bucket, buckets.toArray()[0]);
+    }
+
+    public void assertBuckets(String expr, Set<BucketId> expected) throws Exception {
+        BucketIdFactory factory = new BucketIdFactory();
+        BucketSelector selector = new BucketSelector(factory);
+        Set<BucketId> actual = new TreeSet<BucketId>(selector.getBucketList(expr));
+        assertEquals(expected, actual);
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java b/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java
new file mode 100644
index 00000000000..1677592eecf
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java
@@ -0,0 +1,782 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.select;
+
+import com.yahoo.document.*;
+import com.yahoo.document.datatypes.*;
+import com.yahoo.document.select.convert.SelectionExpressionConverter;
+import com.yahoo.document.select.parser.ParseException;
+import com.yahoo.document.select.parser.TokenMgrError;
+import com.yahoo.yolean.Exceptions;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ * @author bratseth
+ */
+public class DocumentSelectorTestCase extends junit.framework.TestCase {
+
+    private static DocumentTypeManager manager = new DocumentTypeManager();
+
+    public DocumentSelectorTestCase(String name) {
+        super(name);
+    }
+
+    public void setUp() {
+        DocumentType type = new DocumentType("test");
+        type.addHeaderField("hint", DataType.INT);
+        type.addHeaderField("hfloat", DataType.FLOAT);
+        type.addHeaderField("hstring", DataType.STRING);
+        type.addField("content", DataType.STRING);
+
+        StructDataType mystruct = new StructDataType("mystruct");
+        mystruct.addField(new Field("key", DataType.INT, false));
+        mystruct.addField(new Field("value", DataType.STRING, false));
+        type.addHeaderField("mystruct", mystruct);
+
+        ArrayDataType structarray = new ArrayDataType(mystruct);
+        type.addField("structarray", structarray);
+
+        type.addField("stringweightedset", new WeightedSetDataType(DataType.STRING, false, false));
+        type.addField("mymap", new MapDataType(DataType.INT, DataType.STRING));
+        type.addField("structarrmap", new MapDataType(DataType.STRING, structarray));
+
+        ArrayDataType intarray = new ArrayDataType(DataType.INT);
+        type.addField("intarray", intarray);
+
+        manager.registerDocumentType(type);
+
+        // Create strange doctypes using identifiers within them, which we
+        // can use to verify they are still parsed as doctypes.
+        manager.registerDocumentType(new DocumentType("notandor"));
+        manager.registerDocumentType(new DocumentType("ornotand"));
+        manager.registerDocumentType(new DocumentType("andornot"));
+        manager.registerDocumentType(new DocumentType("idid"));
+        manager.registerDocumentType(new DocumentType("usergroup"));
+    }
+
+    public void testParsing() throws ParseException {
+        assertParse("3.14 > 0");
+        assertParse("-999 > 0");
+        assertParse("150000.0 > 0", "15e4 > 0");
+        assertParse("3.4E-4 > 0", "3.4e-4 > 0");
+        assertParse("\" Test \" = \"*\"");
+        assertParse("id = \"*\"", "id = '*'");
+        assertParse("id.group == 3");
+        assertParse("id.namespace = \"*\"");
+        assertParse("id.hash() > 0");
+        assertParse("id.namespace.hash() > 0");
+        assertParse("id.order(5,2) > 100");
+        assertParse("searchcolumn.10 = 6");
+        assertParse("music.artist = \"*\"");
+        assertParse("music.artist.lowercase() = \"*\"");
+        assertParse("music_.artist = \"*\"");
+        assertParse("music_foo.artist = \"*\"");
+        assertParse("music_foo_.artist = \"*\"");
+        assertParse("(4 + 3) > 0", "(4+3) > 0");
+        assertParse("1 + 1 > 0", "1 +1 > 0");
+        assertParse("1 + -1 > 0", "1 + -1 > 0");
+        assertParse("1 + 1.0 > 0", "1 + +1.0 > 0");
+        assertParse("1 - 1 > 0", "1 -1 > 0");
+        assertParse("1 - -1 > 0", "1 - -1 > 0");
+        assertParse("1 - 1.0 > 0", "1 - +1.0 > 0");
+        assertParse("1 + 2 * 3 - 10 % 2 / 3 > 0", "1   +2 *  3- 10%2   /3  > 0");
+        assertParse("((43 + 14) / 34) > 0");
+        assertParse("(34 * ((3 - 1) % 4)) > 0");
+        assertParse("true");
+        assertParse("false");
+        assertParse("music");
+        assertParse("(music or book)");
+        assertParse("music or book", "music or book");
+        assertParse("(music or (book and video))");
+        assertParse("music or (book and video)", "music or (book and video)");
+        assertParse("((music or book) and video)");
+        assertParse("(music or book) and video", "(music or book) and video");
+        assertParse("music.test > 0");
+        assertParse("music.artist = \"*john*\"");
+        assertParse("music.length >= 180");
+        assertParse("true or not false and true", "true oR nOt false And true");
+        assertParse("(true or false) and true", "(true oR false) aNd true");
+        assertParse("music.expire > now()");
+        assertParse("music.expire > now() - 300");
+        assertParse("now or now_search");
+        assertParse("(music.expire / 1000) > (now() - 300)");
+    }
+
+    public void testReservedWords() throws ParseException {
+        assertParse(null, "id == 'id' or id_t or idtype"); // ignore canonical form
+        assertParse(null, "searchcolumn == 1 or searchcolumn_t or searchcolumntype");
+        assertParse(null, "id.scheme == 'scheme' or scheme_t or schemetype");
+        assertParse(null, "id.namespace == 'namespace' or namespace_t or namespacetype");
+        assertParse(null, "id.specific == 'specific' or specific_t or specifictype");
+        assertParse(null, "id.user == 'user' or user_t or usertype");
+        assertParse(null, "id.group == 'group' or group_t or grouptype");
+        assertParse(null, "id.bucket == 'bucket' or bucket_t or buckettype");
+        assertParse(null, "null == 'null' or null_t or nulltype");
+        assertParse(null, "true or true_t or truetype");
+        assertParse(null, "false or false_t or falsetype");
+        assertParse(null, "true or and_t or andtype");
+        assertParse(null, "true or or_t or ortype");
+    }
+
+    public void testCjkParsing() throws ParseException {
+        assertParse("music.artist = \"\\u4f73\\u80fd\\u7d22\\u5c3c\\u60e0\\u666e\"",
+                    "music.artist = \"\u4f73\u80fd\u7d22\u5c3c\u60e0\u666e\"");
+        assertParse("music.artist = \"\\u4f73\\u80fd\\u7d22\\u5c3c\\u60e0\\u666e\"",
+                    "music.artist = \"\\u4f73\\u80fd\\u7d22\\u5c3c\\u60e0\\u666e\"");
+    }
+
+    public void testParseTerminals() throws ParseException {
+        // Test number values.
+        assertParse("true");
+        assertParse("music.hmm == 123");
+        assertParse("music.hmm == 123.53", "music.hmm == +123.53");
+        assertParse("music.hmm == -123.5");
+        assertParse("music.hmm == 2.3412352E8", "music.hmm == 234123.52e3");
+        assertParse("music.hmm == -234.12352", "music.hmm == -234123.52E-3");
+        assertParse("music.hmm < aaa");
+
+        // Test string values.
+        assertParse("music.hmm == \"test\"");
+
+        // Test map and struct stuff.
+        assertParse("music.hmm{test} == \"test\"");
+        assertParse("music.hmm{test}.foo[3].key == \"test\"");
+
+        // Test whitespaces.
+        assertParse("music.hmm == \"te st \"");
+        assertParse("music.hmm == \"test\"", " \t music.hmm\t==  \t \"test\"\t");
+
+        // Test escaping.
+        assertParse("music.hmm == \"tab\\ttest\"");
+        assertParse("music.hmm == \"tab\\u0666test\"", "music.hmm == \"tab\\u0666test\"");
+        assertParse("music.hmm == \"tabcomplete\"", "music.hmm == \"tabcomplete\"");
+        assertParse("music.hmm == \"tabysf\"", "music.hmm == \"tab\\ysf\""); // excessive escapes are removed
+        assertParse("music.h == \"\\ttx48 \\n\"", "music.h == \"\\tt\\x48 \\n\"");
+
+        // Test illegal operator.
+        assertParseError("music.hmm <> 12", "Exception parsing document selector 'music.hmm <> 12': Encountered \" \">\" \"> \"\" at line 1, column 12.");
+
+        // Test comparison operators.
+        assertParse("music.hmm >= 123");
+        assertParse("music.hmm > 123");
+        assertParse("music.hmm <= 123");
+        assertParse("music.hmm < 123");
+        assertParse("music.hmm != 123");
+
+        // Test defined.
+        assertParse("music.hmm");
+
+        // Test boolean expressions.
+        assertParse("true", "TRUE");
+        assertParse("false", "FALSE");
+        assertParse("true", "true");
+        assertParse("false", "false");
+        assertParse("false", "faLSe");
+
+        // Test document types.
+        assertParse("mytype");
+
+        // Test document id.
+        assertParse("id == \"userdoc:ns:mytest\"");
+        assertParse("id.namespace == \"myspace\"");
+        assertParse("id.scheme == \"userdoc\"");
+        assertParse("id.type == \"mytype\"");
+        assertParse("id.user == 1234");
+        assertParse("id.bucket == 8388608", "id.bucket == 0x800000");
+        assertParse("id.bucket == 8429568", "id.bucket == 0x80a000");
+        assertParse("id.bucket == -9223372036854775566",
+                    "id.bucket == 0x80000000000000f2");
+        assertParse("id.group == \"yahoo.com\"");
+        assertParse("id.specific == \"mypart\"");
+
+        // Test search column stuff.
+        assertParse("searchcolumn.10 = 6");
+
+        // Test other operators.
+        assertParse("id.scheme = \"*doc\"");
+        assertParse("music.artist =~ \"(john|barry|shrek)\"");
+
+        // Verify functions.
+        assertParse("id.hash() == 124");
+        assertParse("id.specific.hash() == 124");
+        assertParse("music.artist.lowercase() == \"chang\"");
+        assertParse("music.artist.lowercase().hash() == 124");
+        assertParse("music.version() == 8");
+        assertParse("music == 8"); // will evaluate to false
+
+        // Value grouping.
+        assertParse("(123) < (200)", "(123) < (200)");
+        assertParse("(\"hmm\") < (id.scheme)", "(\"hmm\") < (id.scheme)");
+
+        // Arithmetics.
+        assertParse("(1 + 2) > 1");
+        assertParse("1 + 2 > 1", "1 + 2 > 1");
+        assertParse("(1 - 2) > 1");
+        assertParse("(1 * 2) > 1");
+        assertParse("(1 / 2) > 1");
+        assertParse("(1 % 2) > 1");
+        assertParse("((1 + 2) * (4 - 2)) == 1");
+        assertParse("(1 + 2) * (4 - 2) == 1", "(1 + 2) * (4 - 2) == 1");
+        assertParse("((23 + 643) / (34 % 10)) > 34");
+        assertParse("23 + 643 / 34 % 10 > 34", "23 + 643 / 34 % 10 > 34");
+    }
+
+    public void testParseReservedTokens() throws ParseException {
+        assertParse("user.fieldName == \"fieldValue\""); // reserved word as document type name
+        assertParse("documentName.user == \"fieldValue\""); // reserved word as field type name
+        assertParse("group.fieldName == \"fieldValue\""); // reserved word as document type name
+        assertParse("documentName.group == \"fieldValue\""); // reserved word as field type name
+    }
+
+    public void testParseBranches() throws ParseException {
+        assertParse("((true or false) and (false or true))");
+        assertParse("(true or (not false and not true))");
+        assertParse("((243) < 300 and (\"FOO\").lowercase() == (\"foo\"))");
+    }
+
+    public void testDocumentUpdate() throws ParseException {
+        DocumentUpdate upd = new DocumentUpdate(manager.getDocumentType("test"), new DocumentId("doc:myspace:anything"));
+        assertEquals(Result.TRUE, evaluate("test", upd));
+        assertEquals(Result.FALSE, evaluate("music", upd));
+        assertEquals(Result.TRUE, evaluate("test or music", upd));
+        assertEquals(Result.FALSE, evaluate("test and music", upd));
+        assertEquals(Result.INVALID, evaluate("test.hint", upd));
+        assertEquals(Result.INVALID, evaluate("test.anything", upd));
+        assertEquals(Result.INVALID, evaluate("test.hint < 24", upd));
+        // TODO Fails: assertEquals(Result.TRUE, evaluate("test.hint + 1 > 13", upd));
+    }
+
+    public void testInvalidLogic() throws ParseException {
+        DocumentPut put = new DocumentPut(manager.getDocumentType("test"), new DocumentId("doc:scheme:"));
+        DocumentUpdate upd = new DocumentUpdate(manager.getDocumentType("test"), new DocumentId("doc:scheme:"));
+
+        assertEquals(Result.FALSE, evaluate("test.content", put)); // BROKEN
+        assertEquals(Result.INVALID, evaluate("test.content", upd));
+
+        assertEquals(Result.FALSE, evaluate("test.content = 1", put)); // BROKEN
+        assertEquals(Result.INVALID, evaluate("test.content = 1", upd));
+
+        assertEquals(Result.FALSE, evaluate("test.content = 1 and true", put)); // BROKEN
+        assertEquals(Result.INVALID, evaluate("test.content = 1 and true", upd));
+
+        assertEquals(Result.FALSE, evaluate("test.content = 1 or true", put)); // BROKEN
+        assertEquals(Result.TRUE, evaluate("test.content = 1 or true", upd));
+
+        assertEquals(Result.FALSE, evaluate("test.content = 1 and false",  put));
+        assertEquals(Result.FALSE, evaluate("test.content = 1 and false",  upd));
+
+        assertEquals(Result.FALSE, evaluate("test.content = 1 or false", put)); // BROKEN
+        assertEquals(Result.INVALID, evaluate("test.content = 1 or false", upd));
+
+        assertEquals(Result.FALSE, evaluate("true and test.content = 1",  put)); // BROKEN
+        assertEquals(Result.INVALID, evaluate("true and test.content = 1",  upd));
+
+        assertEquals(Result.FALSE, evaluate("true or test.content = 1", put)); // BROKEN
+        assertEquals(Result.TRUE, evaluate("true or test.content = 1", upd));
+
+        assertEquals(Result.FALSE, evaluate("false and test.content = 1",  put));
+        assertEquals(Result.FALSE, evaluate("false and test.content = 1",  upd));
+
+        assertEquals(Result.FALSE, evaluate("false or test.content = 1",  put)); // BROKEN
+        assertEquals(Result.INVALID, evaluate("false or test.content = 1",  upd));
+    }
+
+    List<DocumentPut> createDocs() {
+        List<DocumentPut> documents = new ArrayList<>();
+        documents.add(createDocument("doc:myspace:anything", 24, 2.0f, "foo", "bar"));
+        documents.add(createDocument("doc:anotherspace:foo", 13, 4.1f, "bar", "foo"));
+        documents.add(createDocument("userdoc:myspace:1234:mail1", 15, 1.0f, "some", "some"));
+        documents.add(createDocument("userdoc:myspace:5678:bar", 14, 2.4f, "Yet", "More"));
+        documents.add(createDocument("orderdoc(31,19):ns2:1234:5678:foo", 14, 2.4f, "Yet", "More"));
+        documents.add(createDocument("id:myspace:test:n=2345:mail2", 15, 1.0f, "bar", "baz"));
+        documents.add(createDocument("id:myspace:test:g=mygroup:qux", 15, 1.0f, "quux", "corge"));
+        documents.add(createDocument("doc:myspace:missingint", null, 2.0f, null, "bar"));
+
+        // Add some array/struct info to doc 1
+        Struct sval = new Struct(documents.get(1).getDocument().getField("mystruct").getDataType());
+        sval.setFieldValue("key", new IntegerFieldValue(14));
+        sval.setFieldValue("value", new StringFieldValue("structval"));
+        documents.get(1).getDocument().setFieldValue("mystruct", sval);
+        Array<Struct> aval = new Array<>(documents.get(1).getDocument().getField("structarray").getDataType());
+        {
+            Struct sval1 = new Struct(aval.getDataType().getNestedType());
+            sval1.setFieldValue("key", new IntegerFieldValue(15));
+            sval1.setFieldValue("value", new StringFieldValue("structval1"));
+            Struct sval2 = new Struct(aval.getDataType().getNestedType());
+            sval2.setFieldValue("key", new IntegerFieldValue(16));
+            sval2.setFieldValue("value", new StringFieldValue("structval2"));
+            aval.add(sval1);
+            aval.add(sval2);
+        }
+        documents.get(1).getDocument().setFieldValue("structarray", aval);
+
+        MapFieldValue<IntegerFieldValue, StringFieldValue> mval =
+                new MapFieldValue<>((MapDataType)documents.get(1).getDocument().getField("mymap")
+                                                                                             .getDataType());
+        mval.put(new IntegerFieldValue(3), new StringFieldValue("a"));
+        mval.put(new IntegerFieldValue(5), new StringFieldValue("b"));
+        mval.put(new IntegerFieldValue(7), new StringFieldValue("c"));
+        documents.get(1).getDocument().setFieldValue("mymap", mval);
+
+        MapFieldValue<StringFieldValue, Array> amval =
+                new MapFieldValue<>((MapDataType)documents.get(1).getDocument().getField("structarrmap")
+                                                                                 .getDataType());
+        amval.put(new StringFieldValue("foo"), aval);
+
+        Array<Struct> abval = new Array<>(documents.get(1).getDocument().getField("structarray").getDataType());
+        {
+            Struct sval1 = new Struct(aval.getDataType().getNestedType());
+            sval1.setFieldValue("key", new IntegerFieldValue(17));
+            sval1.setFieldValue("value", new StringFieldValue("structval3"));
+            Struct sval2 = new Struct(aval.getDataType().getNestedType());
+            sval2.setFieldValue("key", new IntegerFieldValue(18));
+            sval2.setFieldValue("value", new StringFieldValue("structval4"));
+            abval.add(sval1);
+            abval.add(sval2);
+        }
+
+        amval.put(new StringFieldValue("bar"), abval);
+        documents.get(1).getDocument().setFieldValue("structarrmap", amval);
+
+        WeightedSet<StringFieldValue> wsval = new WeightedSet<>(documents.get(1).getDocument().getField("stringweightedset")
+                                                                     .getDataType());
+        wsval.add(new StringFieldValue("foo"));
+        wsval.add(new StringFieldValue("val1"));
+        wsval.add(new StringFieldValue("val2"));
+        wsval.add(new StringFieldValue("val3"));
+        wsval.add(new StringFieldValue("val4"));
+        documents.get(1).getDocument().setFieldValue("stringweightedset", wsval);
+
+        // Add empty array/struct to doc 2
+        Struct sval3 = new Struct(documents.get(2).getDocument().getField("mystruct").getDataType());
+        documents.get(2).getDocument().setFieldValue("mystruct", sval3);
+        Array aval2 = new Array(documents.get(2).getDocument().getField("structarray").getDataType());
+        documents.get(2).getDocument().setFieldValue("structarray", aval2);
+
+        Array<IntegerFieldValue> intvals1 = new Array<>(documents.get(0).getDocument().getField("intarray")
+                                                                                  .getDataType());
+        intvals1.add(new IntegerFieldValue(12));
+        intvals1.add(new IntegerFieldValue(40));
+        intvals1.add(new IntegerFieldValue(60));
+        intvals1.add(new IntegerFieldValue(84));
+        documents.get(0).getDocument().setFieldValue("intarray", intvals1);
+
+        Array<IntegerFieldValue> intvals2 = new Array<>(documents.get(1).getDocument().getField("intarray")
+                                                                                  .getDataType());
+        intvals2.add(new IntegerFieldValue(3));
+        intvals2.add(new IntegerFieldValue(56));
+        intvals2.add(new IntegerFieldValue(23));
+        intvals2.add(new IntegerFieldValue(9));
+        documents.get(1).getDocument().setFieldValue("intarray", intvals2);
+
+        return documents;
+    }
+
+    public void testOperators() throws ParseException {
+        List<DocumentPut> documents = createDocs();
+
+        // Check that comparison operators work.
+        assertEquals(Result.TRUE, evaluate("", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("30 < 10", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("10 < 30", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("30 < 10", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("10 < 30", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("30 <= 10", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("10 <= 30", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("30 <= 30", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("10 >= 30", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("30 >= 10", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("30 >= 30", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("10 > 30", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("30 > 10", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("30 == 10", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("30 == 30", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("30 != 10", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("30 != 30", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("\"foo\" != \"bar\"", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("\"foo\" != \"foo\"", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("'foo' == \"bar\"", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("'foo' == \"foo\"", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("\"bar\" = \"a\"", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("\"bar\" = \"*a*\"", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("\"bar\" = \"\"", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("\"\" = \"\"", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("\"bar\" =~ \"^a$\"", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("\"bar\" =~ \"a\"", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("\"bar\" =~ \"\"", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("\"\" =~ \"\"", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("30 = 10", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("30 = 30", documents.get(0)));
+
+        // Mix of types should within numbers, but otherwise not match
+        assertEquals(Result.FALSE, evaluate("30 < 10.2", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("10.2 < 30", documents.get(0)));
+        assertEquals(Result.INVALID, evaluate("30 < \"foo\"", documents.get(0)));
+        assertEquals(Result.INVALID, evaluate("30 > \"foo\"", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("30 != \"foo\"", documents.get(0)));
+        assertEquals(Result.INVALID, evaluate("14.2 <= \"foo\"", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("null == null", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("null = null", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("\"bar\" == null", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("14.3 == null", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("null = 0", documents.get(0)));
+
+        // Field values
+        assertEquals(Result.TRUE, evaluate("test.hint = 24", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("test.hint = 24", documents.get(1)));
+        assertEquals(Result.FALSE, evaluate("test.hint = 13", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("test.hint = 13", documents.get(1)));
+        assertEquals(Result.TRUE, evaluate("test.hfloat = 2.0", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("test.hfloat = 1.0", documents.get(1)));
+        assertEquals(Result.FALSE, evaluate("test.hfloat = 4.1", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("test.hfloat > 4.09 and test.hfloat < 4.11", documents.get(1)));
+        assertEquals(Result.TRUE, evaluate("test.content = \"bar\"", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("test.content = \"bar\"", documents.get(1)));
+        assertEquals(Result.FALSE, evaluate("test.content = \"foo\"", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("test.content = \"foo\"", documents.get(1)));
+        assertEquals(Result.FALSE, evaluate("test.hstring == test.content", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("test.hstring == test.content", documents.get(2)));
+        assertEquals(Result.TRUE, evaluate("test.hint + 1 > 13", documents.get(1)));
+
+        // Document types.
+        assertEquals(Result.TRUE, evaluate("test", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("nonexisting", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("nonexisting.reallynonexisting", documents.get(0)));
+        assertEquals(Result.INVALID, evaluate("nonexisting.reallynonexisting > 13", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("true.foo", documents.get(0)));
+
+        // Field existence
+        assertEquals(Result.TRUE, evaluate("test.hint", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("test.hstring", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("test.hint", documents.get(7)));
+        assertEquals(Result.FALSE, evaluate("test.hstring", documents.get(7)));
+        assertEquals(Result.TRUE, evaluate("not test.hint", documents.get(7)));
+        assertEquals(Result.TRUE, evaluate("not test.hstring", documents.get(7)));
+
+        // Id values.
+        assertEquals(Result.TRUE, evaluate("id == \"doc:myspace:anything\"", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate(" iD==  \"doc:myspace:anything\"  ", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("id == \"doc:myspa:nything\"", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("Id.scHeme == \"doc\"", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("id.scheme == \"userdoc\"", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("id.type == \"test\"", documents.get(5)));
+        assertEquals(Result.FALSE, evaluate("id.type == \"wrong\"", documents.get(5)));
+        assertEquals(Result.TRUE, evaluate("Id.namespaCe == \"myspace\"", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("id.NaMespace == \"pace\"", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("id.specific == \"anything\"", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("id.user=1234", documents.get(2)));
+        assertEquals(Result.TRUE, evaluate("id.user=1234", documents.get(4)));
+        assertEquals(Result.TRUE, evaluate("id.group=\"1234\"", documents.get(4)));
+        assertEquals(Result.TRUE, evaluate("id.user=2345", documents.get(5)));
+        assertEquals(Result.TRUE, evaluate("id.group=\"mygroup\"", documents.get(6)));
+
+        assertEquals(Result.TRUE, evaluate("id.order(31,19)=5678", documents.get(4)));
+        assertEquals(Result.TRUE, evaluate("id.order(31,19)<=5678", documents.get(4)));
+        assertEquals(Result.FALSE, evaluate("id.order(31,19)>5678", documents.get(4)));
+        assertEquals(Result.FALSE, evaluate("id.order(25,23)==5678", documents.get(4)));
+        assertEquals(Result.FALSE, evaluate("id.order(31,19)=5678", documents.get(3)));
+
+        assertError("id.user == 1234", documents.get(0), "User identifier is null.");
+        assertError("id.group == 1234", documents.get(3), "Group identifier is null.");
+        assertError("id.group == \"yahoo\"", documents.get(3), "Group identifier is null.");
+        assertError("id.type == \"unknown\"", documents.get(0), "Document id doesn't have doc type.");
+
+        assertEquals(Result.TRUE, evaluate("searchcolumn.10 == 6", documents.get(3)));
+
+        // Branch operators.
+        assertEquals(Result.FALSE, evaluate("true and false", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("true and true", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("true or false", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("false or false", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("false and true or true and true", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("false or true and true or false", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("not false", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("not true", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("true and not false or false", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("((243 < 300) and (\"FOO\".lowercase() == \"foo\"))", documents.get(0)));
+
+        // Invalid branching. test.content = 1 is invalid.
+        assertEquals(Result.FALSE, evaluate("test.content = 1 and true", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("test.content = 1 or true", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("test.content = 1 and false", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("test.content = 1 or false", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("true and test.content = 1", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("true or test.content = 1", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("false and test.content = 1", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("false or test.content = 1", documents.get(0)));
+
+        // Functions.
+        assertEquals(Result.FALSE, evaluate("test.hstring.lowercase() == \"Yet\"", documents.get(3)));
+        assertEquals(Result.TRUE, evaluate("test.hstring.lowercase() == \"yet\"", documents.get(3)));
+        assertEquals(Result.FALSE, evaluate("test.hfloat.lowercase() == \"yet\"", documents.get(3)));
+        assertEquals(Result.TRUE, evaluate("\"bar\".hash() == -270124981", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("\"bar\".hash().abs() == 270124981", documents.get(0)));
+        assertError("null.hash() == 22460089", documents.get(0), "Can not invoke 'hash()' on 'null' because that term evaluated to null");
+        assertEquals(Result.FALSE, evaluate("(0.234).hash() == 123", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("(0.234).lowercase() == 123", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("\"foo\".hash() == 123", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("(234).hash() == 123", documents.get(0)));
+        // Arithmetics
+        assertEquals(Result.TRUE, evaluate("id.specific.hash() = 596580044", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("id.specific.hash() % 10 = 4", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("id.specific.hash() % 10 = 2", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("\"foo\" + \"bar\" = \"foobar\"", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("\"foo\" + 4 = 25", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("34.0 % 4 = 4", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("-6 % 10 = -6", documents.get(0)));
+
+        // Test now(). Assumes that now() is never 0
+        assertEquals(Result.FALSE, evaluate("0 > now()", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("0 < now()", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("0 < now() - 10", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("now() - 20 < now() - 10", documents.get(0)));
+        long secondsNow = System.currentTimeMillis() / 1000;
+        assertEquals(Result.TRUE, evaluate("now() - " + secondsNow + " < 2", documents.get(0)));
+
+        // Structs and arrays
+        // Commented out tests work in C++.. Don't want to start altering
+        // java code before talking to Simon.. Leaving it as is, as the
+        // needed functionality of testing for existance already works.
+        assertEquals(Result.FALSE, evaluate("test.mystruct", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("test.mystruct", documents.get(1)));
+        assertEquals(Result.FALSE, evaluate("test.mystruct", documents.get(2)));
+        assertEquals(Result.TRUE, evaluate("test.mystruct == test.mystruct", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("test.mystruct == test.mystruct", documents.get(1)));
+        assertEquals(Result.FALSE, evaluate("test.mystruct != test.mystruct", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("test.mystruct != test.mystruct", documents.get(1)));
+        //assertEquals(Result.INVALID, evaluate("test.mystruct < test.mystruct", documents.get(0)));
+        //assertEquals(Result.FALSE, evaluate("test.mystruct < test.mystruct", documents.get(1)));
+        //assertEquals(Result.INVALID, evaluate("test.mystruct < 5", documents.get(1)));
+        //assertEquals(Result.INVALID, evaluate("test.mystruct == \"foo\"", documents.get(1)));
+        assertEquals(Result.FALSE, evaluate("test.structarray", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("test.structarray", documents.get(1)));
+        assertEquals(Result.FALSE, evaluate("test.structarray", documents.get(2)));
+        assertEquals(Result.TRUE, evaluate("test.structarray == test.structarray", documents.get(0)));
+        //assertEquals(Result.INVALID, evaluate("test.structarray < test.structarray", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("test.structarray == test.structarray", documents.get(1)));
+        //assertEquals(Result.FALSE, evaluate("test.structarray < test.structarray", documents.get(1)));
+
+        assertEquals(Result.FALSE, evaluate("test.structarray.key == 15", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("test.structarray[4].key == 15", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("test.structarray", documents.get(1)));
+        assertEquals(Result.TRUE, evaluate("test.structarray.key == 15", documents.get(1)));
+        assertEquals(Result.TRUE, evaluate("test.structarray[1].key == 16", documents.get(1)));
+        assertEquals(Result.TRUE, evaluate("test.structarray[1].key = 16", documents.get(1)));
+        assertEquals(Result.FALSE, evaluate("test.structarray.value == \"structval1\"", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("test.structarray[4].value == \"structval1\"", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("test.structarray.value == \"structval1\"", documents.get(1)));
+        assertEquals(Result.TRUE, evaluate("test.structarray[0].value == \"structval1\"", documents.get(1)));
+        // Globbing of array-of-struct fields
+        assertEquals(Result.FALSE, evaluate("test.structarray.key = 15", documents.get(0))); // Fallback
+        assertEquals(Result.FALSE, evaluate("test.structarray.key = 15", documents.get(2))); // Fallback
+        assertEquals(Result.TRUE, evaluate("test.structarray.key = 15", documents.get(1))); // Fallback
+        assertEquals(Result.FALSE, evaluate("test.structarray.value = \"structval2\"", documents.get(2)));
+        assertEquals(Result.TRUE, evaluate("test.structarray.value = \"*ctval*\"", documents.get(1)));
+        assertEquals(Result.TRUE, evaluate("test.structarray[1].value = \"structval2\"", documents.get(1)));
+        assertEquals(Result.FALSE, evaluate("test.structarray[1].value = \"batman\"", documents.get(1)));
+        // Regexp of array-of-struct fields
+        assertEquals(Result.TRUE, evaluate("test.structarray.value =~ \"structval[1-9]\"", documents.get(1)));
+        assertEquals(Result.FALSE, evaluate("test.structarray.value =~ \"structval[a-z]\"", documents.get(1)));
+        // Globbing/regexp of struct fields
+        assertEquals(Result.FALSE, evaluate("test.mystruct.value = \"struc?val\"", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("test.mystruct.value = \"struc?val\"", documents.get(1)));
+        assertEquals(Result.FALSE, evaluate("test.mystruct.value =~ \"struct.*\"", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("test.mystruct.value =~ \"struct.*\"", documents.get(1)));
+
+        assertEquals(Result.FALSE, evaluate("test.intarray < 5", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("test.intarray < 5", documents.get(1)));
+        assertEquals(Result.TRUE, evaluate("test.intarray > 80", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("test.intarray > 80", documents.get(1)));
+        assertEquals(Result.TRUE, evaluate("test.intarray >= 84", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("test.intarray <= 3", documents.get(1)));
+
+        // Interesting property ...
+        assertEquals(Result.TRUE, evaluate("test.intarray == 84", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("test.intarray != 84", documents.get(0)));
+
+        assertEquals(Result.TRUE, evaluate("test.structarray[$x].key == 15 AND test.structarray[$x].value == \"structval1\"", documents.get(1)));
+        assertEquals(Result.FALSE, evaluate("test.structarray[$x].key == 15 AND test.structarray[$x].value == \"structval2\"", documents.get(1)));
+        assertEquals(Result.TRUE, evaluate("test.structarray[$x].key == 15 AND test.structarray[$y].value == \"structval2\"", documents.get(1)));
+
+        assertEquals(Result.FALSE, evaluate("test.mymap", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("test.mymap", documents.get(1)));
+        assertEquals(Result.TRUE, evaluate("test.mymap{3}", documents.get(1)));
+        assertEquals(Result.FALSE, evaluate("test.mymap{9}", documents.get(1)));
+        assertEquals(Result.TRUE, evaluate("test.mymap{3} == \"a\"", documents.get(1)));
+        assertEquals(Result.FALSE, evaluate("test.mymap{3} == \"b\"", documents.get(1)));
+        assertEquals(Result.FALSE, evaluate("test.mymap{9} == \"b\"", documents.get(1)));
+        assertEquals(Result.FALSE, evaluate("test.mymap == \"a\"", documents.get(1))); // Keys only
+        assertEquals(Result.TRUE, evaluate("test.mymap{3} = \"a\"", documents.get(1)));
+        assertEquals(Result.FALSE, evaluate("test.mymap{3} = \"b\"", documents.get(1)));
+        assertEquals(Result.TRUE, evaluate("test.mymap{3} =~ \"a\"", documents.get(1)));
+        assertEquals(Result.FALSE, evaluate("test.mymap{3} =~ \"b\"", documents.get(1)));
+        assertEquals(Result.TRUE, evaluate("test.mymap.value = \"a\"", documents.get(1)));
+        assertEquals(Result.FALSE, evaluate("test.mymap.value = \"d\"", documents.get(1)));
+        assertEquals(Result.TRUE, evaluate("test.mymap.value =~ \"a\"", documents.get(1)));
+        assertEquals(Result.FALSE, evaluate("test.mymap.value =~ \"d\"", documents.get(1)));
+        assertEquals(Result.TRUE, evaluate("test.mymap == 3", documents.get(1)));
+        assertEquals(Result.FALSE, evaluate("test.mymap == 4", documents.get(1)));
+        assertEquals(Result.TRUE, evaluate("test.mymap = 3", documents.get(1))); // Fallback to ==
+        assertEquals(Result.FALSE, evaluate("test.mymap = 4", documents.get(1))); // Fallback to ==
+
+        assertEquals(Result.TRUE, evaluate("test.structarrmap{$x}[$y].key == 15 AND test.structarrmap{$x}[$y].value == \"structval1\"", documents.get(1)));
+        assertEquals(Result.TRUE, evaluate("test.structarrmap.value[$y].key == 15 AND test.structarrmap.value[$y].value == \"structval1\"", documents.get(1)));
+        assertEquals(Result.FALSE, evaluate("test.structarrmap{$x}[$y].key == 15 AND test.structarrmap{$x}[$y].value == \"structval2\"", documents.get(1)));
+        assertEquals(Result.FALSE, evaluate("test.structarrmap.value[$y].key == 15 AND test.structarrmap.value[$y].value == \"structval2\"", documents.get(1)));
+        assertEquals(Result.FALSE, evaluate("test.structarrmap{$x}[$y].key == 15 AND test.structarrmap{$y}[$x].value == \"structval2\"", documents.get(1)));
+
+        assertEquals(Result.TRUE, evaluate("test.stringweightedset", documents.get(1)));
+        assertEquals(Result.TRUE, evaluate("test.stringweightedset{val1}", documents.get(1)));
+        assertEquals(Result.TRUE, evaluate("test.stringweightedset{val1} == 1", documents.get(1)));
+        assertEquals(Result.FALSE, evaluate("test.stringweightedset{val1} == 2", documents.get(1)));
+        assertEquals(Result.TRUE, evaluate("test.stringweightedset == \"val1\"", documents.get(1)));
+        assertEquals(Result.TRUE, evaluate("test.stringweightedset = \"val*\"", documents.get(1)));
+        assertEquals(Result.TRUE, evaluate("test.stringweightedset =~ \"val[0-9]\"", documents.get(1)));
+        assertEquals(Result.FALSE, evaluate("test.stringweightedset == \"val5\"", documents.get(1)));
+        assertEquals(Result.FALSE, evaluate("test.stringweightedset = \"val5\"", documents.get(1)));
+        assertEquals(Result.FALSE, evaluate("test.stringweightedset =~ \"val5\"", documents.get(1)));
+
+        assertEquals(Result.TRUE, evaluate("test.structarrmap{$x}.key == 15 AND test.stringweightedset{$x}", documents.get(1)));
+        assertEquals(Result.FALSE, evaluate("test.structarrmap{$x}.key == 17 AND test.stringweightedset{$x}", documents.get(1)));
+    }
+
+    public void testTicket1769674() throws ParseException {
+        assertParseError("music.uri=\"junk",
+                         "Lexical error at line -1, column 17.  Encountered: <EOF> after : \"\\\"junk\"");
+    }
+
+    public void testThatVisitingReportsCorrectResult() throws ParseException {
+        assertVisitWithValidNowWorks("music.expire > now()");
+        assertVisitWithValidNowWorks("music.expire > now() and video.expire > now()");
+        assertVisitWithValidNowWorks("music.expire > now() or video");
+        assertVisitWithValidNowWorks("music.expire > now() or video.date < 300");
+        assertVisitWithValidNowWorks("video.date < 300 or music.expire > now()");
+        assertVisitWithValidNowWorks("video.date < 300 and music.expire > now()");
+        assertVisitWithValidNowWorks("music.insertdate > now() - 300 and video.expire > now() - 3600");
+
+        assertVisitWithoutNowWorks("test.structarrmap{$x}[$y].key == 15 AND test.structarrmap{$x}[$y].value == \"structval1\"");
+        assertVisitWithoutNowWorks("music.artist.lowercase() == \"chang\"");
+
+        assertVisitWithInvalidNowFails("music.expire > now() + 300", "Arithmetic operator '+' is not supported");
+        assertVisitWithInvalidNowFails("music.expire < now()", "Comparison operator '<' is not supported");
+        assertVisitWithInvalidNowFails("music.expire >= now()", "Comparison operator '>=' is not supported");
+        assertVisitWithInvalidNowFails("now() > now()", "Left hand side of comparison must be a document field");
+        assertVisitWithInvalidNowFails("music.name.hash() > now()", "Only attribute items are supported");
+    }
+
+    public void testThatSelectionIsConvertedToQueries() throws ParseException {
+        assertThatQueriesAreCreated("music.expire > now()", Arrays.asList("music"), Arrays.asList("expire:>now(0)"));
+        assertThatQueriesAreCreated("music.expire > now() - 300", Arrays.asList("music"), Arrays.asList("expire:>now(300)"));
+        assertThatQueriesAreCreated("music.expire > now() - 300 and video.expire > now() - 3600", Arrays.asList("music", "video"), Arrays.asList("expire:>now(300)", "expire:>now(3600)"));
+        assertThatQueriesAreCreated("music.expire > now() - 300 or video", Arrays.asList("music"), Arrays.asList("expire:>now(300)"));
+        assertVisitWithInvalidNowFails("music.field1 > now() - 300 and music.field2 > now() - 300", "Specifying multiple document types is not allowed");
+        assertVisitWithInvalidNowFails("music.field1 > now() - 300 and video.field1 > now() and music.field2 > now() - 300", "Specifying multiple document types is not allowed");
+        assertVisitWithInvalidNowFails("now() > music.field", "Left hand side of comparison must be a document field");
+    }
+
+    public void assertThatQueriesAreCreated(String selection, List<String> expectedDoctypes, List<String> expectedQueries) throws ParseException {
+        DocumentSelector selector = new DocumentSelector(selection);
+        NowCheckVisitor visitor = new NowCheckVisitor();
+        selector.visit(visitor);
+        assertTrue(visitor.requiresConversion());
+        SelectionExpressionConverter converter = new SelectionExpressionConverter();
+        selector.visit(converter);
+        Map<String, String> queryMap = converter.getQueryMap();
+        assertEquals(expectedQueries.size(), queryMap.size());
+        for (int i = 0; i < expectedQueries.size(); i++) {
+            assertTrue(queryMap.containsKey(expectedDoctypes.get(i)));
+            assertEquals(expectedQueries.get(i), queryMap.get(expectedDoctypes.get(i)));
+        }
+    }
+
+    public void assertVisitWithoutNowWorks(String expression) throws ParseException {
+        DocumentSelector selector = new DocumentSelector(expression);
+        NowCheckVisitor visitor = new NowCheckVisitor();
+        selector.visit(visitor);
+        assertFalse(visitor.requiresConversion());
+    }
+
+    public void assertVisitWithValidNowWorks(String expression) throws ParseException {
+        DocumentSelector selector = new DocumentSelector(expression);
+        NowCheckVisitor visitor = new NowCheckVisitor();
+        selector.visit(visitor);
+        assertTrue(visitor.requiresConversion());
+        SelectionExpressionConverter converter = new SelectionExpressionConverter();
+        try {
+            selector.visit(converter);
+        } catch (Exception e) {
+            assertFalse("Converter throws exception : " + e.getMessage(), true);
+        }
+    }
+
+    public void assertVisitWithInvalidNowFails(String expression, String expectedError) throws ParseException {
+        DocumentSelector selector = new DocumentSelector(expression);
+        NowCheckVisitor visitor = new NowCheckVisitor();
+        selector.visit(visitor);
+        assertTrue(visitor.requiresConversion());
+        SelectionExpressionConverter converter = new SelectionExpressionConverter();
+        try {
+            selector.visit(converter);
+            assertFalse("Should not be able to convert " + expression + " query", true);
+        } catch (Exception e) {
+            assertEquals(expectedError, e.getMessage());
+        }
+    }
+
+    private static DocumentPut createDocument(String id, Integer hInt, float hFloat, String hString, String content) {
+        Document doc = new Document(manager.getDocumentType("test"), new DocumentId(id));
+        if (hInt != null)
+            doc.setFieldValue("hint", new IntegerFieldValue(hInt));
+        doc.setFieldValue("hfloat", new FloatFieldValue(hFloat));
+        if (hString != null)
+            doc.setFieldValue("hstring", new StringFieldValue(hString));
+        doc.setFieldValue("content", new StringFieldValue(content));
+        return new DocumentPut(doc);
+    }
+
+    private static void assertParse(String expression) throws ParseException {
+        assertParse(expression, expression);
+    }
+
+    private static void assertParse(String expectedString, String expressionString) throws ParseException {
+        DocumentSelector selector = new DocumentSelector(expressionString);
+        if (expectedString != null) {
+            assertEquals(expectedString, selector.toString());
+        }
+    }
+
+    private static void assertParseError(String expressionString, String expectedError) {
+        try {
+            new DocumentSelector(expressionString);
+            fail("The expression '" + expressionString + "' should throw an exception.");
+        }
+        catch (ParseException e) {
+            Throwable t = e;
+            if (t.getCause() instanceof TokenMgrError) {
+                t = t.getCause();
+            }
+            assertEquals(expectedError, Exceptions.toMessageString(t).substring(0, expectedError.length()));
+        }
+    }
+
+    private static Result evaluate(String expressionString, DocumentOperation op) throws ParseException {
+        return new DocumentSelector(expressionString).accepts(op);
+    }
+
+    private static void assertError(String expressionString, DocumentOperation op, String expectedError) {
+        try {
+            evaluate(expressionString, op);
+            fail("The evaluation of '" + expressionString + "' should throw an exception.");
+        } catch (ParseException e) {
+            fail("The expression '" + expressionString + "' should assertEquals ok.");
+        } catch (RuntimeException e) {
+            System.err.println("Error was : " + e);
+            assertTrue(e.getMessage().length() >= expectedError.length());
+            assertEquals(expectedError, e.getMessage().substring(0, expectedError.length()));
+        }
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/select/OrderingSpecificationTestCase.java b/document/src/test/java/com/yahoo/document/select/OrderingSpecificationTestCase.java
new file mode 100644
index 00000000000..e47fe4e993e
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/select/OrderingSpecificationTestCase.java
@@ -0,0 +1,51 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.select;
+
+import com.yahoo.document.select.*;
+
+/**
+ * Date: Sep 6, 2007
+ *
+ * @author <a href="mailto:humbe@yahoo-inc.com">H&aring;kon Humberset</a>
+ */
+public class OrderingSpecificationTestCase extends junit.framework.TestCase {
+
+    public OrderingSpecificationTestCase(String name) {
+        super(name);
+    }
+
+    public void testExpressions() throws Exception {
+        assertSelection("id.order(10,10) < 100", OrderingSpecification.DESCENDING,
+                        new OrderingSpecification(OrderingSpecification.DESCENDING, (long)99, (short)10, (short)10));
+        assertSelection("id.order(10,10) <= 100", OrderingSpecification.DESCENDING,
+                        new OrderingSpecification(OrderingSpecification.DESCENDING, (long)100, (short)10, (short)10));
+        assertSelection("id.order(10,10) > 100", OrderingSpecification.DESCENDING, null);
+        assertSelection("id.order(10,10) > 100", OrderingSpecification.ASCENDING,
+                        new OrderingSpecification(OrderingSpecification.ASCENDING, (long)101, (short)10, (short)10));
+        assertSelection("id.user==1234 AND id.order(10,10) > 100", OrderingSpecification.ASCENDING,
+                        new OrderingSpecification(OrderingSpecification.ASCENDING, (long)101, (short)10, (short)10));
+        assertSelection("id.order(10,10) >= 100", OrderingSpecification.ASCENDING,
+                        new OrderingSpecification(OrderingSpecification.ASCENDING, (long)100, (short)10, (short)10));
+        assertSelection("id.order(10,10) == 100", OrderingSpecification.ASCENDING,
+                        new OrderingSpecification(OrderingSpecification.ASCENDING, (long)100, (short)10, (short)10));
+        assertSelection("id.order(10,10) = 100", OrderingSpecification.DESCENDING,
+                        new OrderingSpecification(OrderingSpecification.DESCENDING, (long)100, (short)10, (short)10));
+        assertSelection("id.order(10,10) > 30 AND id.order(10,10) < 100", OrderingSpecification.ASCENDING,
+                        new OrderingSpecification(OrderingSpecification.ASCENDING, (long)31, (short)10, (short)10));
+        assertSelection("id.order(10,10) > 30 AND id.order(10,10) < 100", OrderingSpecification.DESCENDING,
+                        new OrderingSpecification(OrderingSpecification.DESCENDING, (long)99, (short)10, (short)10));
+        assertSelection("id.order(10,10) > 30 OR id.order(10,10) > 70", OrderingSpecification.ASCENDING,
+                        new OrderingSpecification(OrderingSpecification.ASCENDING, (long)31, (short)10, (short)10));
+        assertSelection("id.order(10,10) < 30 OR id.order(10,10) < 70", OrderingSpecification.DESCENDING,
+                        new OrderingSpecification(OrderingSpecification.DESCENDING, (long)69, (short)10, (short)10));
+    }
+
+    public void assertSelection(String selection, int ordering, OrderingSpecification wanted) throws Exception {
+        DocumentSelector selector = new DocumentSelector(selection);
+        if (wanted != null) {
+            assertEquals(wanted, selector.getOrdering(ordering));
+        } else {
+            assertNull(selector.getOrdering(ordering));
+        }
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/serialization/PredicateFieldValueSerializationTestCase.java b/document/src/test/java/com/yahoo/document/serialization/PredicateFieldValueSerializationTestCase.java
new file mode 100644
index 00000000000..eb4e06dc149
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/serialization/PredicateFieldValueSerializationTestCase.java
@@ -0,0 +1,74 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.serialization;
+
+import com.yahoo.document.DataType;
+
+import com.yahoo.document.Document;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.datatypes.PredicateFieldValue;
+import com.yahoo.document.predicate.BooleanPredicate;
+import com.yahoo.document.predicate.Conjunction;
+import com.yahoo.document.predicate.Disjunction;
+import com.yahoo.document.predicate.FeatureRange;
+import com.yahoo.document.predicate.FeatureSet;
+import com.yahoo.document.predicate.Negation;
+import com.yahoo.document.predicate.Predicate;
+import org.junit.Test;
+import java.io.IOException;
+
+import static com.yahoo.document.serialization.SerializationTestUtils.deserializeDocument;
+import static com.yahoo.document.serialization.SerializationTestUtils.serializeDocument;
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a>
+ */
+public class PredicateFieldValueSerializationTestCase {
+
+    private final static String PREDICATE_FIELD = "my_predicate";
+    private final static String PREDICATE_FILES = "src/test/resources/predicates/";
+    private final static TestDocumentFactory docFactory =
+            new TestDocumentFactory(createDocType(), "id:test:my_type::foo");
+
+    private static DocumentType createDocType() {
+        DocumentType type = new DocumentType("my_type");
+        type.addField(PREDICATE_FIELD, DataType.PREDICATE);
+        return type;
+    }
+
+    @Test
+    public void requireThatPredicateFieldValuesAreDeserialized() {
+        Document prevDocument = docFactory.createDocument();
+        PredicateFieldValue prevPredicate = new PredicateFieldValue(new Conjunction(new FeatureSet("foo", "bar"),
+                                                                                    new FeatureRange("baz", 6L, 9L)));
+        prevDocument.setFieldValue(PREDICATE_FIELD, prevPredicate);
+        byte[] buf = serializeDocument(prevDocument);
+        Document nextDocument = deserializeDocument(buf, docFactory);
+        assertEquals(prevDocument, nextDocument);
+        assertEquals(prevPredicate, nextDocument.getFieldValue(PREDICATE_FIELD));
+    }
+
+    @Test
+    public void requireThatPredicateDeserializationMatchesCpp() throws IOException {
+        assertDeserialize("foo_in_bar_and_baz_in_cox", new Conjunction(new FeatureSet("foo", "bar"),
+                                                                       new FeatureSet("baz", "cox")));
+        assertDeserialize("foo_in_bar_or_baz_in_cox", new Disjunction(new FeatureSet("foo", "bar"),
+                                                                      new FeatureSet("baz", "cox")));
+        assertDeserialize("foo_in_6_9", new FeatureRange("foo", 6L, 9L));
+        assertDeserialize("foo_in_6_x", new FeatureRange("foo", 6L, null));
+        assertDeserialize("foo_in_x_9", new FeatureRange("foo", null, 9L));
+        assertDeserialize("foo_in_x_x", new FeatureRange("foo", null, null));
+        assertDeserialize("foo_in_x", new FeatureSet("foo"));
+        assertDeserialize("foo_in_bar", new FeatureSet("foo", "bar"));
+        assertDeserialize("foo_in_bar_baz", new FeatureSet("foo", "bar", "baz"));
+        assertDeserialize("not_foo_in_bar", new Negation(new FeatureSet("foo", "bar")));
+        assertDeserialize("true", new BooleanPredicate(true));
+        assertDeserialize("false", new BooleanPredicate(false));
+    }
+
+    private static void assertDeserialize(String fileName, Predicate expected) throws IOException {
+        Document document = docFactory.createDocument();
+        document.setFieldValue(PREDICATE_FIELD, new PredicateFieldValue(expected));
+        SerializationTestUtils.assertSerializationMatchesCpp(PREDICATE_FILES, fileName, document, docFactory);
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/serialization/SerializationHelperTestCase.java b/document/src/test/java/com/yahoo/document/serialization/SerializationHelperTestCase.java
new file mode 100644
index 00000000000..fbd3ea3afdc
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/serialization/SerializationHelperTestCase.java
@@ -0,0 +1,63 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.serialization;
+
+import com.yahoo.document.datatypes.Raw;
+import com.yahoo.io.GrowableByteBuffer;
+import com.yahoo.text.Utf8;
+import com.yahoo.text.Utf8Array;
+import com.yahoo.vespa.objects.BufferSerializer;
+
+import java.io.UnsupportedEncodingException;
+import java.nio.ByteBuffer;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class SerializationHelperTestCase extends junit.framework.TestCase {
+    public SerializationHelperTestCase(String name) {
+        super(name);
+    }
+
+    public void testGetNullTerminatedString() throws Exception {
+        //This is a test.0ab
+        byte[] test = {0x54, 0x68, 0x69, 0x73, 0x20, 0x69, 0x73, 0x20, 0x61, 0x20, 0x74, 0x65, 0x73, 0x74, 0x2e, 0x0,
+                0x61, 0x62};
+
+        BufferSerializer data = BufferSerializer.wrap(test);
+
+        assertTrue(data.position() == 0);
+
+        Utf8Array thisIsATest = VespaDocumentDeserializer42.parseNullTerminatedString(data.getBuf().getByteBuffer());
+
+        assertTrue(thisIsATest.equals(new Utf8Array(Utf8.toBytes("This is a test."))));
+        assertTrue(data.position() == 16);
+        assertTrue(test[16] == 0x61); //a
+
+        data.position(0);
+
+        assertTrue(data.position() == 0);
+
+        Utf8Array thisIsATestAgain = VespaDocumentDeserializer42.parseNullTerminatedString(data.getBuf().getByteBuffer(), 15);
+
+        assertTrue(thisIsATestAgain.equals(new Utf8Array(Utf8.toBytes("This is a test."))));
+        assertTrue(data.position() == 16);
+        assertTrue(test[16] == 0x61); //a
+    }
+
+    public void testSerializeRawField() throws UnsupportedEncodingException {
+        GrowableByteBuffer gbuf = new GrowableByteBuffer();
+        ByteBuffer rawValue = ByteBuffer.wrap(Utf8.toBytes("0123456789"));
+        rawValue.position(7);
+        Raw value = new Raw(rawValue);
+        value.serialize(gbuf);
+
+        assertEquals(7, gbuf.position());
+        assertEquals(7, rawValue.position());
+
+        value = new Raw(rawValue);
+        value.serialize(gbuf);
+
+        assertEquals(14, gbuf.position());
+        assertEquals(7, rawValue.position());
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/serialization/SerializationTestUtils.java b/document/src/test/java/com/yahoo/document/serialization/SerializationTestUtils.java
new file mode 100644
index 00000000000..18e517fb74e
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/serialization/SerializationTestUtils.java
@@ -0,0 +1,54 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.serialization;
+
+import com.yahoo.document.Document;
+import com.yahoo.io.GrowableByteBuffer;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Helper class with utils used in serialization and deserialization test cases.
+ *
+ * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a>
+ */
+public class SerializationTestUtils {
+
+    public static byte[] serializeDocument(Document doc) {
+        GrowableByteBuffer out = new GrowableByteBuffer();
+        DocumentSerializerFactory.create42(out).write(doc);
+        out.flip();
+        byte[] buf = new byte[out.remaining()];
+        out.get(buf);
+        return buf;
+    }
+
+    public static Document deserializeDocument(byte[] buf, TestDocumentFactory factory) {
+        Document document = factory.createDocument();
+        DocumentDeserializerFactory.create42(factory.typeManager(), new GrowableByteBuffer(ByteBuffer.wrap(buf))).read(document);
+        return document;
+    }
+
+    public static void assertSerializationMatchesCpp(String binaryFilesFolder, String fileName,
+                                                     Document document, TestDocumentFactory factory) throws IOException {
+        byte[] buf = serializeDocument(document);
+        Files.write(Paths.get(binaryFilesFolder, fileName + "__java"), buf,
+                StandardOpenOption.CREATE, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING);
+
+        assertDeserializeFromFile(Paths.get(binaryFilesFolder, fileName + "__java"), document, factory);
+        assertDeserializeFromFile(Paths.get(binaryFilesFolder, fileName + "__cpp"), document, factory);
+    }
+
+    private static void assertDeserializeFromFile(Path path, Document document, TestDocumentFactory factory) throws IOException {
+        byte[] buf = Files.readAllBytes(path);
+        Document deserializedDocument = deserializeDocument(buf, factory);
+        assertEquals(path.toString(), document, deserializedDocument);
+    }
+
+}
diff --git a/document/src/test/java/com/yahoo/document/serialization/SerializeAnnotationsTestCase.java b/document/src/test/java/com/yahoo/document/serialization/SerializeAnnotationsTestCase.java
new file mode 100644
index 00000000000..d8ed160e1c4
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/serialization/SerializeAnnotationsTestCase.java
@@ -0,0 +1,213 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.serialization;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.DocumentTypeManager;
+import com.yahoo.document.DocumentTypeManagerConfigurer;
+import com.yahoo.document.Field;
+import com.yahoo.document.StructDataType;
+import com.yahoo.document.annotation.*;
+import com.yahoo.document.datatypes.Array;
+import com.yahoo.document.datatypes.DoubleFieldValue;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.document.datatypes.Struct;
+import com.yahoo.io.GrowableByteBuffer;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+import java.nio.channels.ReadableByteChannel;
+import java.util.ArrayList;
+
+/**
+ * Tests that serialization of annotations from Java generates the
+ * output expected by the C++ deserialization system.
+ *
+ * If the format changes and this test starts failing, you should
+ * generate new golden files to check against. This will cause the C++
+ * test to fail, so you will need to update the
+ * AnnotationDeserialization component to handle the format changes.
+ */
+public class SerializeAnnotationsTestCase extends junit.framework.TestCase {
+    private static final String PATH = "src/tests/serialization/";
+    DocumentTypeManager docMan = new DocumentTypeManager();
+
+    @Override
+    public void setUp() {
+        DocumentTypeManagerConfigurer.configure(docMan,
+                                                "file:src/tests/serialization/" +
+                                                "annotation.serialize.test.cfg");
+    }
+
+    public void testSerializeSimpleTree() throws IOException {
+        SpanList root = new SpanList();
+        root.add(new Span(0, 19))
+            .add(new Span(19, 5))
+            .add(new Span(24, 21))
+            .add(new Span(45, 23))
+            .add(new Span(68, 14));
+        SpanTree tree = new SpanTree("html", root);
+        StringFieldValue value = new StringFieldValue("lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj " +
+                                                      "lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj l jlkj lkj lkj " +
+                                                      "lkjoijoij oij oij oij oij oij oijoijoij oij oij oij oij oij " +
+                                                      "oijoijoijoijoijoijoijoijoijoijoijoijoij oij oij oij oij " +
+                                                      "oijaosdifjoai fdoais jdoasi jai os oafoai ai dfojsfoa dfoi dsf" +
+                                                      "aosifjasofija sodfij oasdifj aosdiosifjsi ooai oais osi");
+        value.setSpanTree(tree);
+
+        /*
+        Important note! The iteration order of annotations in SpanTree.iterator() is non-deterministic, meaning
+        that the order which annotations are serialized will differ between test runs. Thus, we cannot assert
+        that a serialized buffer is equal to a buffer written earlier. We can, however, assert that the size stays
+        the same, and the deserialized values from the buffers should be equal.
+         */
+
+        //important! call readFile() before writeFile()!
+        ByteBuffer serializedFromFile = readFile("test_data_serialized_simple");
+        ByteBuffer serialized = writeFile(value, "test_data_serialized_simple");
+        assertEquals(serialized.limit(), serializedFromFile.limit());
+
+        StringFieldValue valueFromFile = new StringFieldValue();
+        DocumentDeserializer deserializer = DocumentDeserializerFactory.create42(docMan, new GrowableByteBuffer(serializedFromFile));
+        deserializer.read(null, valueFromFile);
+        assertEquals(value, valueFromFile);
+    }
+
+    public void testSerializeAdvancedTree() throws IOException {
+        SpanList root = new SpanList();
+        SpanTree tree = new SpanTree("html", root);
+
+        DataType positionType = docMan.getDataType("myposition");
+        StructDataType cityDataType =
+            (StructDataType) docMan.getDataType("annotation.city");
+
+        AnnotationTypeRegistry registry = docMan.getAnnotationTypeRegistry();
+        AnnotationType textType = registry.getType("text");
+        AnnotationType beginTag = registry.getType("begintag");
+        AnnotationType endTag = registry.getType("endtag");
+        AnnotationType bodyType = registry.getType("body");
+        AnnotationType paragraphType = registry.getType("paragraph");
+        AnnotationType cityType = registry.getType("city");
+
+        AnnotationReferenceDataType annRefType =
+            (AnnotationReferenceDataType)
+            docMan.getDataType("annotationreference<text>");
+
+        Struct position = new Struct(positionType);
+        position.setFieldValue("latitude", new DoubleFieldValue(37.774929));
+        position.setFieldValue("longitude", new DoubleFieldValue(-122.419415));
+
+        Annotation sanAnnotation = new Annotation(textType);
+        Annotation franciscoAnnotation = new Annotation(textType);
+
+        Struct positionWithRef = cityDataType.createFieldValue();
+        positionWithRef.setFieldValue("position", position);
+
+        Field referencesField = cityDataType.getField("references");
+        Array<FieldValue> refList =
+            new Array<FieldValue>(referencesField.getDataType());
+        refList.add(new AnnotationReference(annRefType, sanAnnotation));
+        refList.add(new AnnotationReference(annRefType, franciscoAnnotation));
+        positionWithRef.setFieldValue(referencesField, refList);
+
+        Annotation city = new Annotation(cityType, positionWithRef);
+
+        AlternateSpanList paragraph = new AlternateSpanList();
+        paragraph.addChildren(new ArrayList<SpanNode>(), 0);
+        paragraph.setProbability(0, 0.9);
+        paragraph.setProbability(1, 0.1);
+        {
+            Span span1 = new Span(6, 3);
+            Span span2 = new Span(9, 10);
+            Span span3 = new Span(19, 4);
+            Span span4 = new Span(23, 4);
+            paragraph.add(0, span1)
+                .add(0, span2)
+                .add(0, span3)
+                .add(0, span4);
+
+            Span alt_span1 = new Span(6, 13);
+            Span alt_span2 = new Span(19, 8);
+            paragraph.add(1, alt_span1)
+                .add(1, alt_span2);
+
+            tree.annotate(span1, beginTag)
+                .annotate(span2, textType)
+                .annotate(span3, sanAnnotation)
+                .annotate(span4, endTag)
+                .annotate(alt_span1, textType)
+                .annotate(alt_span2, bodyType)
+                .annotate(paragraph, paragraphType);
+        }
+
+        {
+            Span span1 = new Span(0, 6);
+            Span span2 = new Span(27, 9);
+            Span span3 = new Span(36, 8);
+            root.add(span1)
+                .add(paragraph)
+                .add(span2)
+                .add(span3);
+
+            tree.annotate(span1, beginTag)
+                .annotate(span2, franciscoAnnotation)
+                .annotate(span3, endTag)
+                .annotate(root, bodyType)
+                .annotate(city);
+        }
+
+        StringFieldValue value = new StringFieldValue("lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj " +
+                                                      "lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj l jlkj lkj lkj " +
+                                                      "lkjoijoij oij oij oij oij oij oijoijoij oij oij oij oij oij " +
+                                                      "oijoijoijoijoijoijoijoijoijoijoijoijoij oij oij oij oij " +
+                                                      "oijaosdifjoai fdoais jdoasi jai os oafoai ai dfojsfoa dfoi dsf" +
+                                                      "aosifjasofija sodfij oasdifj aosdiosifjsi ooai oais osi");
+        value.setSpanTree(tree);
+
+        //important! call readFile() before writeFile()!
+        ByteBuffer serializedFromFile = readFile("test_data_serialized_advanced");
+        ByteBuffer serialized = writeFile(value, "test_data_serialized_advanced");
+        assertEquals(serialized.limit(), serializedFromFile.limit());
+
+        StringFieldValue valueFromFile = new StringFieldValue();
+        DocumentDeserializer deserializer = DocumentDeserializerFactory.create42(docMan, new GrowableByteBuffer(serializedFromFile));
+        deserializer.read(null, valueFromFile);
+        assertEquals(value, valueFromFile);
+    }
+
+    private static ByteBuffer writeFile(StringFieldValue value, String fileName) throws IOException {
+        fileName = PATH + fileName;
+
+        //serialize our tree to buffer
+        VespaDocumentSerializer42 serializer = new VespaDocumentSerializer42();
+        serializer.write(null, value);
+        ByteBuffer serializedBuf = serializer.getBuf().getByteBuffer();
+        serializedBuf.flip();
+
+        //write our tree to disk
+        File file = new File(fileName);
+        FileChannel wChannel = new FileOutputStream(file, false).getChannel();
+        wChannel.write(serializedBuf);
+        wChannel.close();
+
+        serializedBuf.position(0);
+        return serializedBuf;
+    }
+
+    private static ByteBuffer readFile(String fileName) throws IOException {
+        fileName = PATH + fileName;
+
+        //read tree from disk
+        ReadableByteChannel channel = new FileInputStream(fileName).getChannel();
+        ByteBuffer readBuf = ByteBuffer.allocate(4096);
+        channel.read(readBuf);
+        readBuf.flip();
+        channel.close();
+
+        return readBuf;
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/serialization/TensorFieldValueSerializationTestCase.java b/document/src/test/java/com/yahoo/document/serialization/TensorFieldValueSerializationTestCase.java
new file mode 100644
index 00000000000..62f9ea9f4d3
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/serialization/TensorFieldValueSerializationTestCase.java
@@ -0,0 +1,67 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.serialization;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.Document;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.datatypes.TensorFieldValue;
+import com.yahoo.tensor.MapTensor;
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static com.yahoo.document.serialization.SerializationTestUtils.deserializeDocument;
+import static com.yahoo.document.serialization.SerializationTestUtils.serializeDocument;
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a>
+ */
+public class TensorFieldValueSerializationTestCase {
+
+    private final static String TENSOR_FIELD = "my_tensor";
+    private final static String TENSOR_FILES = "src/test/resources/tensor/";
+    private final static TestDocumentFactory docFactory =
+            new TestDocumentFactory(createDocType(), "id:test:my_type::foo");
+
+    private static DocumentType createDocType() {
+        DocumentType type = new DocumentType("my_type");
+        type.addField(TENSOR_FIELD, DataType.TENSOR);
+        return type;
+    }
+
+    @Test
+    public void requireThatTensorFieldValueIsSerializedAndDeserialized() {
+        assertSerialization(new TensorFieldValue());
+        assertSerialization(createTensor("{}"));
+        assertSerialization(createTensor("{{dimX:a,dimY:bb}:2.0,{dimX:ccc,dimY:dddd}:3.0,{dimX:e}:5.0}"));
+    }
+
+    @Test
+    public void requireThatSerializationMatchesCpp() throws IOException {
+        assertSerializationMatchesCpp("non_existing_tensor", new TensorFieldValue());
+        assertSerializationMatchesCpp("empty_tensor", createTensor("{}"));
+        assertSerializationMatchesCpp("multi_cell_tensor",
+                createTensor("{{dimX:a,dimY:bb}:2.0,{dimX:ccc,dimY:dddd}:3.0,{dimX:e}:5.0}"));
+    }
+
+    private static void assertSerialization(TensorFieldValue tensor) {
+        Document document = docFactory.createDocument();
+        document.setFieldValue(TENSOR_FIELD, tensor);
+        byte[] buf = serializeDocument(document);
+        Document deserializedDocument = deserializeDocument(buf, docFactory);
+        assertEquals(document, deserializedDocument);
+        assertEquals(tensor, deserializedDocument.getFieldValue(TENSOR_FIELD));
+    }
+
+    private static void assertSerializationMatchesCpp(String fileName, TensorFieldValue tensor) throws IOException {
+        Document document = docFactory.createDocument();
+        document.setFieldValue(TENSOR_FIELD, tensor);
+        SerializationTestUtils.assertSerializationMatchesCpp(TENSOR_FILES, fileName, document, docFactory);
+    }
+
+    private static TensorFieldValue createTensor(String tensor) {
+        return new TensorFieldValue(MapTensor.from(tensor));
+    }
+
+}
diff --git a/document/src/test/java/com/yahoo/document/serialization/TestDocumentFactory.java b/document/src/test/java/com/yahoo/document/serialization/TestDocumentFactory.java
new file mode 100644
index 00000000000..d54562862b7
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/serialization/TestDocumentFactory.java
@@ -0,0 +1,37 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.serialization;
+
+import com.yahoo.document.Document;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.DocumentTypeManager;
+
+/**
+ * Helper class for creating a document for a given document type.
+ *
+ * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a>
+ */
+public class TestDocumentFactory {
+
+    private final DocumentTypeManager typeManager = new DocumentTypeManager();
+    private final DocumentType docType;
+    private final String defaultId;
+
+    public TestDocumentFactory(DocumentType docType, String defaultId) {
+        this.docType = docType;
+        this.defaultId = defaultId;
+        typeManager.register(docType);
+    }
+
+    public Document createDocument(String id) {
+        return new Document(docType, id);
+    }
+
+    public Document createDocument() {
+        return createDocument(defaultId);
+    }
+
+    public DocumentTypeManager typeManager() {
+        return typeManager;
+    }
+
+}
diff --git a/document/src/test/java/com/yahoo/document/serialization/VespaDocumentSerializerTestCase.java b/document/src/test/java/com/yahoo/document/serialization/VespaDocumentSerializerTestCase.java
new file mode 100644
index 00000000000..823bb0b196d
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/serialization/VespaDocumentSerializerTestCase.java
@@ -0,0 +1,48 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.serialization;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.Document;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.Field;
+import com.yahoo.document.datatypes.IntegerFieldValue;
+import com.yahoo.document.datatypes.PredicateFieldValue;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.io.GrowableByteBuffer;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class VespaDocumentSerializerTestCase {
+
+    @Test
+    public void requireThatGetSerializedSizeUsesLatestSerializer() {
+        DocumentType docType = new DocumentType("my_type");
+        docType.addField("my_str", DataType.STRING);
+        docType.addField("my_int", DataType.INT);
+        Document doc = new Document(docType, "doc:scheme:");
+        doc.setFieldValue("my_str", new StringFieldValue("foo"));
+        doc.setFieldValue("my_int", new IntegerFieldValue(69));
+
+        GrowableByteBuffer buf = new GrowableByteBuffer();
+        doc.serialize(buf);
+        assertEquals(buf.position(), VespaDocumentSerializer42.getSerializedSize(doc));
+    }
+
+    @Test
+    public void requireThatPredicateFieldValuesAreSerialized() {
+        DocumentType docType = new DocumentType("my_type");
+        Field field = new Field("my_predicate", DataType.PREDICATE);
+        docType.addField(field);
+        Document doc = new Document(docType, "doc:scheme:");
+        PredicateFieldValue predicate = Mockito.mock(PredicateFieldValue.class);
+        doc.setFieldValue("my_predicate", predicate);
+
+        DocumentSerializerFactory.create42(new GrowableByteBuffer()).write(doc);
+        Mockito.verify(predicate, Mockito.times(1)).serialize(Mockito.same(field), Mockito.any(FieldWriter.class));
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/serialization/XmlDocumentWriterTestCase.java b/document/src/test/java/com/yahoo/document/serialization/XmlDocumentWriterTestCase.java
new file mode 100644
index 00000000000..b185250cbd8
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/serialization/XmlDocumentWriterTestCase.java
@@ -0,0 +1,29 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.serialization;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.Document;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.Field;
+import com.yahoo.document.datatypes.PredicateFieldValue;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a>
+ */
+public class XmlDocumentWriterTestCase {
+
+    @Test
+    public void requireThatPredicateFieldValuesAreSerializedAsString() {
+        DocumentType docType = new DocumentType("my_type");
+        Field field = new Field("my_predicate", DataType.PREDICATE);
+        docType.addField(field);
+        Document doc = new Document(docType, "doc:scheme:");
+        PredicateFieldValue predicate = Mockito.mock(PredicateFieldValue.class);
+        doc.setFieldValue("my_predicate", predicate);
+
+        new XmlDocumentWriter().write(doc);
+        Mockito.verify(predicate, Mockito.times(1)).printXml(Mockito.any(XmlStream.class));
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/serialization/XmlStreamTestCase.java b/document/src/test/java/com/yahoo/document/serialization/XmlStreamTestCase.java
new file mode 100644
index 00000000000..57a7fc76427
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/serialization/XmlStreamTestCase.java
@@ -0,0 +1,104 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.serialization;
+
+import com.yahoo.document.serialization.XmlStream;
+
+import java.io.IOException;
+
+/**
+ * Test for XmlStream used in document XML serialization.
+ *
+ * @author <a href="humbe@yahoo-inc.com>Haakon Humberset</a>
+ */
+public class XmlStreamTestCase extends junit.framework.TestCase {
+
+    public XmlStreamTestCase(String name) {
+        super(name);
+    }
+
+    /** A catch all test checking that regular usage looks good. */
+    public void testNormalUsage() {
+        XmlStream xml = new XmlStream();
+        xml.setIndent("  ");
+        xml.beginTag("foo");
+        xml.addAttribute("bar", "foo");
+        xml.addAttribute("foo", "bar");
+        xml.addContent("foo");
+        xml.beginTag("bar");
+        xml.endTag();
+        xml.beginTag("foo");
+        xml.beginTag("bar");
+        xml.addAttribute("foo", "bar");
+        xml.addContent("bar");
+        xml.endTag();
+        xml.endTag();
+        xml.addContent("bar");
+        xml.beginTag("foo");
+        xml.addContent("foo");
+        xml.addContent("bar");
+        xml.endTag();
+        xml.endTag();
+        String expected =
+              "<foo bar=\"foo\" foo=\"bar\">\n"
+            + "  foo\n"
+            + "  <bar/>\n"
+            + "  <foo>\n"
+            + "    <bar foo=\"bar\">bar</bar>\n"
+            + "  </foo>\n"
+            + "  bar\n"
+            + "  <foo>foobar</foo>\n"
+            + "</foo>\n";
+        assertEquals(expected, xml.toString());
+    }
+
+    /**
+     * Test that XML tag and attribute names are checked for validity.
+     * Only the obvious illegal characters are tested currently.
+     */
+    public void testIllegalAttributeNames() {
+        String illegalNames[] = {">foo", "foo<bar", " foo", "bar ", "foo bar", "foo\"bar", "&foo"};
+        XmlStream xml = new XmlStream();
+        for (String name : illegalNames) {
+            try {
+                xml.beginTag(name);
+                assertTrue(false);
+            } catch (IllegalArgumentException e) {
+                assertTrue(e.getMessage().indexOf("cannot be used as an XML tag name") != -1);
+            }
+        }
+        xml.beginTag("test");
+        for (String name : illegalNames) {
+            try {
+                xml.addAttribute(name, "");
+                assertTrue(false);
+            } catch (IllegalArgumentException e) {
+                assertTrue(e.getMessage().indexOf("cannot be used as an XML attribute name") != -1);
+            }
+        }
+    }
+
+    /** Test that XML attribute values are XML escaped. */
+    public void testAttributeEscaping() {
+        //String badString = "\"&<>'\n\r\u0000";
+        String badString = "\"&<>'";
+        XmlStream xml = new XmlStream();
+        xml.beginTag("foo");
+        xml.addAttribute("bar", badString);
+        xml.endTag();
+        String expected = "<foo bar=\"&quot;&amp;&lt;&gt;'\"/>\n";
+        assertEquals(expected, xml.toString());
+    }
+
+    /** Test that content is XML escaped. */
+    public void testContentEscaping() {
+        //String badString = "\"&<>'\n\r\u0000";
+        String badString = "\"&<>'";
+        XmlStream xml = new XmlStream();
+        xml.beginTag("foo");
+        xml.addContent(badString);
+        xml.endTag();
+        String expected = "<foo>\"&amp;&lt;&gt;'</foo>\n";
+        assertEquals(expected, xml.toString());
+    }
+
+}
diff --git a/document/src/test/java/com/yahoo/document/update/FieldUpdateTestCase.java b/document/src/test/java/com/yahoo/document/update/FieldUpdateTestCase.java
new file mode 100644
index 00000000000..e14922eb2df
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/update/FieldUpdateTestCase.java
@@ -0,0 +1,368 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.update;
+
+import com.yahoo.document.*;
+import com.yahoo.document.datatypes.*;
+import com.yahoo.document.serialization.DocumentDeserializerFactory;
+import com.yahoo.document.serialization.DocumentSerializer;
+import com.yahoo.document.serialization.DocumentSerializerFactory;
+
+import java.util.List;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class FieldUpdateTestCase extends junit.framework.TestCase {
+    Field strfoo;
+    Field strarray;
+    Field strws;
+    Field strws2;
+    DocumentTypeManager docman = new DocumentTypeManager();
+
+    public void setUp() {
+        docman = new DocumentTypeManager();
+
+        DocumentType testType = new DocumentType("foobar");
+        testType.addField(new Field("strfoo", DataType.STRING));
+        testType.addField(new Field("strarray", DataType.getArray(DataType.STRING)));
+        testType.addField(new Field("strws", DataType.getWeightedSet(DataType.STRING)));
+        testType.addField(new Field("strws2", DataType.getWeightedSet(DataType.STRING, true, true)));
+        docman.registerDocumentType(testType);
+
+
+        strfoo = docman.getDocumentType("foobar").getField("strfoo");
+        strarray = docman.getDocumentType("foobar").getField("strarray");
+        strws = docman.getDocumentType("foobar").getField("strws");
+        strws2 = docman.getDocumentType("foobar").getField("strws2");
+    }
+
+    public void testInstantiationExceptions() {
+        //add(field, value)
+        try {
+            FieldUpdate.createAdd(strfoo, new StringFieldValue("banana"));
+            fail("Should have gotten exception");
+        } catch (UnsupportedOperationException uoe) {}
+        FieldUpdate.createAdd(strarray, new StringFieldValue("banana"));
+        try {
+            FieldUpdate.createAdd(strarray, new Array(DataType.getArray(DataType.STRING)));
+                    fail("Should have gotten exception");
+        } catch (IllegalArgumentException iae) {}
+        FieldUpdate.createAdd(strws, new StringFieldValue("banana"));
+        try {
+            FieldUpdate.createAdd(strws, new Array(DataType.getArray(DataType.STRING)));
+            fail("Should have gotten exception");
+        } catch (IllegalArgumentException iae) {}
+
+
+        //add(field, key, weight)
+        try {
+            FieldUpdate.createAdd(strfoo, new StringFieldValue("apple"), 5);
+            fail("Should have gotten exception");
+        } catch (UnsupportedOperationException uoe) {}
+        FieldUpdate.createAdd(strarray, new StringFieldValue("apple"), 5);
+        FieldUpdate.createAdd(strws, new StringFieldValue("apple"), 5);
+        try {
+            FieldUpdate.createAdd(strws, new Array(DataType.getArray(DataType.STRING)), 50);
+            fail("Should have gotten exception");
+        } catch (IllegalArgumentException iae) {}
+
+
+        Array<StringFieldValue> fruitList = new Array<>(DataType.getArray(DataType.STRING));
+        fruitList.add(new StringFieldValue("kiwi"));
+        fruitList.add(new StringFieldValue("mango"));
+        Array<Raw> rawList = new Array<>(DataType.getArray(DataType.RAW));
+        rawList.add(new Raw());
+        rawList.add(new Raw());
+        rawList.add(new Raw());
+
+
+        //addall(field, list)
+        try {
+            FieldUpdate.createAddAll(strfoo, fruitList);
+            fail("Should have gotten exception");
+        } catch (UnsupportedOperationException uoe) {}
+        FieldUpdate.createAddAll(strarray, fruitList);
+        try {
+            FieldUpdate.createAddAll(strarray, rawList);
+            fail("Should have gotten exception");
+        } catch (IllegalArgumentException iae) {}
+        FieldUpdate.createAddAll(strws, fruitList);
+
+        WeightedSet fruitWs = new WeightedSet(DataType.getWeightedSet(DataType.STRING));
+        fruitWs.put(new StringFieldValue("pineapple"), 50);
+        fruitWs.put(new StringFieldValue("grape"), 10);
+        WeightedSet rawWs = new WeightedSet(DataType.getWeightedSet(DataType.RAW));
+        rawWs.put(new Raw(), 5);
+        rawWs.put(new Raw(), 5);
+        rawWs.put(new Raw(), 4);
+
+
+        //addall(field, weightedset)
+        try {
+            FieldUpdate.createAddAll(strfoo, fruitWs);
+            fail("Should have gotten exception");
+        } catch (UnsupportedOperationException uoe) {}
+        FieldUpdate.createAddAll(strarray, fruitWs);
+        FieldUpdate.createAddAll(strws, fruitWs);
+        try {
+            FieldUpdate.createAddAll(strws, rawWs);
+            fail("Should have gotten exception");
+        } catch (IllegalArgumentException iae) {}
+
+
+        //assign(field, object)
+        FieldUpdate.createAssign(strfoo, new StringFieldValue("potato"));
+        FieldUpdate.createAssign(strarray, fruitList);
+        FieldUpdate.createAssign(strws, fruitWs);
+        try {
+            FieldUpdate.createAssign(strfoo, new IntegerFieldValue(69));
+            fail("Should have gotten exception");
+        } catch (IllegalArgumentException uoe) {}
+
+
+        //decrement(field, object, decrement)
+        try {
+            FieldUpdate.createDecrement(strfoo, new StringFieldValue("ruccola"), 49d);
+            fail("Should have gotten exception");
+        } catch (UnsupportedOperationException uoe) {}
+        try {
+            FieldUpdate.createDecrement(strarray, new StringFieldValue("ruccola"), 49d);
+            fail("Should have gotten exception");
+        } catch (IllegalArgumentException iae) {}
+        FieldUpdate.createDecrement(strws, new StringFieldValue("ruccola"), 49d);
+        try {
+            FieldUpdate.createDecrement(strws, new Raw(), 48d);
+            fail("Should have gotten exception");
+        } catch (IllegalArgumentException iae) {}
+
+
+        //increment(field, object, increment)
+        try {
+            FieldUpdate.createIncrement(strfoo, new StringFieldValue("ruccola"), 49d);
+            fail("Should have gotten exception");
+        } catch (UnsupportedOperationException uoe) {}
+        try {
+            FieldUpdate.createIncrement(strarray, new StringFieldValue("ruccola"), 49d);
+            fail("Should have gotten exception");
+        } catch (IllegalArgumentException iae) {}
+        FieldUpdate.createIncrement(strws, new StringFieldValue("ruccola"), 49d);
+        try {
+            FieldUpdate.createIncrement(strws, new Raw(), 48d);
+            fail("Should have gotten exception");
+        } catch (IllegalArgumentException iae) {}
+
+
+        //remove(field)
+        FieldUpdate.createClear(strfoo);
+        FieldUpdate.createClear(strarray);
+        FieldUpdate.createClear(strws);
+
+
+        //remove(field, object)
+        try {
+            FieldUpdate.createRemove(strfoo, new StringFieldValue("salad"));
+            fail("Should have gotten exception");
+        } catch (UnsupportedOperationException uoe) {}
+        FieldUpdate.createRemove(strarray, new StringFieldValue("salad"));
+        try {
+            FieldUpdate.createRemove(strarray, new Raw());
+            fail("Should have gotten exception");
+        } catch (IllegalArgumentException iae) {}
+        FieldUpdate.createRemove(strws, new StringFieldValue("salad"));
+        try {
+            FieldUpdate.createRemove(strws, new Raw());
+            fail("Should have gotten exception");
+        } catch (IllegalArgumentException iae) {}
+    }
+
+    // Copy all field updates using serialization to verify that it is supported
+    private FieldUpdate serializedCopy(FieldUpdate source, DocumentType docType) {
+        DocumentSerializer buffer = DocumentSerializerFactory.create42();
+        source.serialize(buffer);
+        buffer.getBuf().flip();
+        FieldUpdate copy = new FieldUpdate(DocumentDeserializerFactory.create42(docman, buffer.getBuf()), docType, Document.SERIALIZED_VERSION);
+        assertEquals(source, copy);
+        return copy;
+    }
+
+    public void testApplyToSingleValue() {
+        Document testDoc = new Document(docman.getDocumentType("foobar"), new DocumentId("doc:test:ballooo"));
+        FieldUpdate alter = FieldUpdate.create(strfoo);
+
+        ValueUpdate assign = ValueUpdate.createAssign(new StringFieldValue("potato"));
+        alter.addValueUpdate(assign);
+        alter = serializedCopy(alter, testDoc.getDataType());
+        alter.applyTo(testDoc);
+        assertEquals(new StringFieldValue("potato"), testDoc.getFieldValue("strfoo"));
+
+        FieldUpdate clear = FieldUpdate.createClearField(strfoo);
+        clear = serializedCopy(clear, testDoc.getDataType());
+        clear.applyTo(testDoc);
+        assertNull(testDoc.getFieldValue("strfoo"));
+    }
+
+    public void testApplyToArray() {
+        Array<StringFieldValue> fruitList = new Array<>(DataType.getArray(DataType.STRING));
+        fruitList.add(new StringFieldValue("kiwi"));
+        fruitList.add(new StringFieldValue("mango"));
+        Document testDoc = new Document(docman.getDocumentType("foobar"), new DocumentId("doc:test:ballooo"));
+        FieldUpdate alter = FieldUpdate.create(strarray);
+
+        alter.addValueUpdate(ValueUpdate.createAdd(new StringFieldValue("banana")));
+        alter = serializedCopy(alter, testDoc.getDataType());
+        alter.applyTo(testDoc);
+        assertEquals(1, ((List) testDoc.getFieldValue("strarray")).size());
+        assertEquals(new StringFieldValue("banana"), ((List) testDoc.getFieldValue("strarray")).get(0));
+
+        alter.clearValueUpdates();
+        alter.addValueUpdates(ValueUpdate.createAddAll(fruitList));
+        alter = serializedCopy(alter, testDoc.getDataType());
+        alter.applyTo(testDoc);
+        assertEquals(3, ((List) testDoc.getFieldValue("strarray")).size());
+        assertEquals(new StringFieldValue("banana"), ((List) testDoc.getFieldValue("strarray")).get(0));
+        assertEquals(new StringFieldValue("kiwi"), ((List) testDoc.getFieldValue("strarray")).get(1));
+        assertEquals(new StringFieldValue("mango"), ((List) testDoc.getFieldValue("strarray")).get(2));
+
+        alter.clearValueUpdates();
+        alter.addValueUpdate(ValueUpdate.createAssign(fruitList));
+        alter = serializedCopy(alter, testDoc.getDataType());
+        alter.applyTo(testDoc);
+        System.err.println(testDoc.getFieldValue("strarray"));
+        assertEquals(2, ((List) testDoc.getFieldValue("strarray")).size());
+        assertEquals(new StringFieldValue("kiwi"), ((List) testDoc.getFieldValue("strarray")).get(0));
+        assertEquals(new StringFieldValue("mango"), ((List) testDoc.getFieldValue("strarray")).get(1));
+
+        alter.clearValueUpdates();
+        alter.addValueUpdate(ValueUpdate.createRemove(new StringFieldValue("kiwi")));
+        alter = serializedCopy(alter, testDoc.getDataType());
+        alter.applyTo(testDoc);
+        assertEquals(1, ((List) testDoc.getFieldValue("strarray")).size());
+        assertEquals(new StringFieldValue("mango"), ((List) testDoc.getFieldValue("strarray")).get(0));
+
+        FieldUpdate clear = FieldUpdate.createClearField(strarray);
+        clear = serializedCopy(clear, testDoc.getDataType());
+        clear.applyTo(testDoc);
+        assertNull(testDoc.getFieldValue("strarray"));
+    }
+
+    public void testApplyToWs() {
+        WeightedSet fruitWs = new WeightedSet(DataType.getWeightedSet(DataType.STRING));
+        fruitWs.put(new StringFieldValue("pineapple"), 50);
+        fruitWs.put(new StringFieldValue("apple"), 10);
+        Document testDoc = new Document(docman.getDocumentType("foobar"), new DocumentId("doc:test:ballooo"));
+        FieldUpdate alter = FieldUpdate.create(strws);
+        FieldUpdate alter2 = FieldUpdate.create(strws2);
+
+
+        alter.addValueUpdate(ValueUpdate.createAdd(new StringFieldValue("banana")));
+        alter = serializedCopy(alter, testDoc.getDataType());
+        alter.applyTo(testDoc);
+        assertEquals(1, ((WeightedSet) testDoc.getFieldValue("strws")).size());
+        assertEquals(1, (int) ((WeightedSet) testDoc.getFieldValue("strws")).get(new StringFieldValue("banana")));
+
+        alter.clearValueUpdates();
+        alter.addValueUpdate(ValueUpdate.createAdd(new StringFieldValue("apple"), 5));
+        alter = serializedCopy(alter, testDoc.getDataType());
+        alter.applyTo(testDoc);
+        assertEquals(2, ((WeightedSet) testDoc.getFieldValue("strws")).size());
+        assertEquals(1, (int) ((WeightedSet) testDoc.getFieldValue("strws")).get(new StringFieldValue("banana")));
+        assertEquals(5, (int) ((WeightedSet) testDoc.getFieldValue("strws")).get(new StringFieldValue("apple")));
+
+        alter.clearValueUpdates();
+        alter.addValueUpdates(ValueUpdate.createAddAll(fruitWs));
+        alter = serializedCopy(alter, testDoc.getDataType());
+        alter.applyTo(testDoc);
+        assertEquals(3, ((WeightedSet) testDoc.getFieldValue("strws")).size());
+        assertEquals(1, (int) ((WeightedSet) testDoc.getFieldValue("strws")).get(new StringFieldValue("banana")));
+        assertEquals(10, (int) ((WeightedSet) testDoc.getFieldValue("strws")).get(new StringFieldValue("apple")));
+        assertEquals(50, (int) ((WeightedSet) testDoc.getFieldValue("strws")).get(new StringFieldValue("pineapple")));
+
+        alter.clearValueUpdates();
+        alter.addValueUpdate(ValueUpdate.createAssign(fruitWs));
+        alter = serializedCopy(alter, testDoc.getDataType());
+        alter.applyTo(testDoc);
+        assertEquals(2, ((WeightedSet) testDoc.getFieldValue("strws")).size());
+        assertEquals(50, (int) ((WeightedSet) testDoc.getFieldValue("strws")).get(new StringFieldValue("pineapple")));
+        assertEquals(10, (int) ((WeightedSet) testDoc.getFieldValue("strws")).get(new StringFieldValue("apple")));
+
+        alter.clearValueUpdates();
+        alter.addValueUpdate(ValueUpdate.createDecrement(new StringFieldValue("pineapple"), 3d));
+        alter = serializedCopy(alter, testDoc.getDataType());
+        alter.applyTo(testDoc);
+        assertEquals(2, ((WeightedSet) testDoc.getFieldValue("strws")).size());
+        assertEquals(47, (int) ((WeightedSet) testDoc.getFieldValue("strws")).get(new StringFieldValue("pineapple")));
+        assertEquals(10, (int) ((WeightedSet) testDoc.getFieldValue("strws")).get(new StringFieldValue("apple")));
+
+        alter.clearValueUpdates();
+        alter.addValueUpdate(ValueUpdate.createIncrement(new StringFieldValue("pineapple"), 13d));
+        alter = serializedCopy(alter, testDoc.getDataType());
+        alter.applyTo(testDoc);
+        assertEquals(2, ((WeightedSet) testDoc.getFieldValue("strws")).size());
+        assertEquals(60, (int) ((WeightedSet) testDoc.getFieldValue("strws")).get(new StringFieldValue("pineapple")));
+        assertEquals(10, (int) ((WeightedSet) testDoc.getFieldValue("strws")).get(new StringFieldValue("apple")));
+
+        alter.clearValueUpdates();
+        alter.addValueUpdate(ValueUpdate.createRemove(new StringFieldValue("apple")));
+        alter = serializedCopy(alter, testDoc.getDataType());
+        alter.applyTo(testDoc);
+        assertEquals(1, ((WeightedSet) testDoc.getFieldValue("strws")).size());
+        assertEquals(60, (int) ((WeightedSet) testDoc.getFieldValue("strws")).get(new StringFieldValue("pineapple")));
+
+        // Test createifnonexistant
+        alter.clearValueUpdates();
+        alter.addValueUpdate(ValueUpdate.createIncrement(new StringFieldValue("apple"), 1));
+        alter = serializedCopy(alter, testDoc.getDataType());
+        alter.applyTo(testDoc);
+        assertNull(((WeightedSet)testDoc.getFieldValue("strws")).get(new StringFieldValue("apple")));
+
+        alter2.addValueUpdate(ValueUpdate.createIncrement(new StringFieldValue("apple"), 1));
+        alter2 = serializedCopy(alter2, testDoc.getDataType());
+        alter2.applyTo(testDoc);
+        assertEquals(1, ((WeightedSet) testDoc.getFieldValue("strws2")).size());
+        assertEquals(1, (int) ((WeightedSet) testDoc.getFieldValue("strws2")).get(new StringFieldValue("apple")));
+
+        // Test removeifzero
+        alter.clearValueUpdates();
+        alter.addValueUpdate(ValueUpdate.createAdd(new StringFieldValue("banana")));
+        alter = serializedCopy(alter, testDoc.getDataType());
+        alter.applyTo(testDoc);
+        assertEquals(1, (int) ((WeightedSet)testDoc.getFieldValue("strws")).get(new StringFieldValue("banana")));
+        alter.clearValueUpdates();
+        alter.addValueUpdate(ValueUpdate.createDecrement(new StringFieldValue("banana"), 1));
+        alter = serializedCopy(alter, testDoc.getDataType());
+        alter.applyTo(testDoc);
+        assertEquals(0, (int) ((WeightedSet)testDoc.getFieldValue("strws")).get(new StringFieldValue("banana")));
+
+        alter2.clearValueUpdates();
+        alter2.addValueUpdate(ValueUpdate.createAdd(new StringFieldValue("banana")));
+        alter2 = serializedCopy(alter2, testDoc.getDataType());
+        alter2.applyTo(testDoc);
+        assertEquals(1, (int) ((WeightedSet)testDoc.getFieldValue("strws2")).get(new StringFieldValue("banana")));
+
+        alter2.clearValueUpdates();
+        alter2.addValueUpdate(ValueUpdate.createDecrement(new StringFieldValue("banana"), 1));
+        alter2 = serializedCopy(alter2, testDoc.getDataType());
+        alter2.applyTo(testDoc);
+        assertNull(((WeightedSet)testDoc.getFieldValue("strws2")).get(new StringFieldValue("banana")));
+
+        FieldUpdate clear = FieldUpdate.createClearField(strws);
+        clear = serializedCopy(clear, testDoc.getDataType());
+        clear.applyTo(testDoc);
+        assertNull(testDoc.getFieldValue("strws"));
+    }
+
+    public void testArithmeticUpdatesOnAutoCreatedWSetItemsAreZeroBased() {
+        Document testDoc = new Document(
+                docman.getDocumentType("foobar"),
+                new DocumentId("doc:test:ballooo"));
+        // strws2 is fixture weightedset type with create-if-non-existing
+        // and remove-if-zero attributes set.
+        FieldUpdate update = FieldUpdate.create(strws2);
+        StringFieldValue key = new StringFieldValue("apple");
+        update.addValueUpdate(ValueUpdate.createIncrement(key, 1));
+        update.applyTo(testDoc);
+        assertEquals(1, ((WeightedSet)testDoc.getFieldValue("strws2")).size());
+        assertEquals(1, (int)((WeightedSet) testDoc.getFieldValue("strws2")).get(key));
+    }
+
+}
diff --git a/document/src/test/java/com/yahoo/document/update/SerializationTestCase.java b/document/src/test/java/com/yahoo/document/update/SerializationTestCase.java
new file mode 100644
index 00000000000..0d73138e077
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/update/SerializationTestCase.java
@@ -0,0 +1,57 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.update;
+
+import com.yahoo.document.*;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.document.serialization.*;
+
+import java.io.FileOutputStream;
+
+/**
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public class SerializationTestCase extends junit.framework.TestCase {
+
+    private DocumentType documentType;
+
+    private Field field;
+
+    public SerializationTestCase(String name) {
+        super(name);
+    }
+
+    public void setUp() {
+        documentType=new DocumentType("document1");
+        field=new Field("field1", DataType.getArray(DataType.STRING));
+        documentType.addField(field);
+    }
+
+    public void testAddSerialization() {
+        FieldUpdate update=FieldUpdate.createAdd(field, new StringFieldValue("value1"));
+        DocumentSerializer buffer = DocumentSerializerFactory.create42();
+        update.serialize(buffer);
+
+        buffer.getBuf().rewind();
+
+        try{
+            FileOutputStream fos = new FileOutputStream("src/test/files/addfieldser.dat");
+            fos.write(buffer.getBuf().array(), 0, buffer.getBuf().remaining());
+            fos.close();
+        } catch (Exception e) {}
+
+        FieldUpdate deserializedUpdate = new FieldUpdate(DocumentDeserializerFactory.create42(new DocumentTypeManager(), buffer.getBuf()), documentType, Document.SERIALIZED_VERSION);
+        assertEquals("'field1' [add value1 1]", deserializedUpdate.toString());
+    }
+
+    public void testClearSerialization() {
+        FieldUpdate update=FieldUpdate.createClear(field);
+        DocumentSerializer buffer = DocumentSerializerFactory.create42();
+        update.serialize(buffer);
+
+        buffer.getBuf().rewind();
+        FieldUpdate deserializedUpdate = new FieldUpdate(DocumentDeserializerFactory.create42(new DocumentTypeManager(), buffer.getBuf()), documentType, Document.SERIALIZED_VERSION);
+
+        assertEquals("'field1' [clear]", deserializedUpdate.toString());
+    }
+
+}
diff --git a/document/src/test/java/com/yahoo/document/update/ValueUpdateTestCase.java b/document/src/test/java/com/yahoo/document/update/ValueUpdateTestCase.java
new file mode 100644
index 00000000000..d197f77669c
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/update/ValueUpdateTestCase.java
@@ -0,0 +1,23 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.update;
+
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.StringFieldValue;
+
+/**
+ * Test case for ValueUpdate class.
+ *
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class ValueUpdateTestCase extends junit.framework.TestCase {
+    public void testUpdateSimple() {
+        /** We cannot test much on this level anyway, most stuff in ValueUpdate is package
+         * private. Better tests exist in FieldUpdateTestCase. */
+        AssignValueUpdate upd = (AssignValueUpdate) ValueUpdate.createAssign(new StringFieldValue("newvalue"));
+
+        assertEquals(ValueUpdate.ValueUpdateClassID.ASSIGN, upd.getValueUpdateClassID());
+
+        FieldValue newValue = upd.getValue();
+        assertEquals(new StringFieldValue("newvalue"), newValue);
+    }
+}
diff --git a/document/src/test/java/com/yahoo/vespaxmlparser/PositionParserTestCase.java b/document/src/test/java/com/yahoo/vespaxmlparser/PositionParserTestCase.java
new file mode 100644
index 00000000000..a40e2a5a4a3
--- /dev/null
+++ b/document/src/test/java/com/yahoo/vespaxmlparser/PositionParserTestCase.java
@@ -0,0 +1,48 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespaxmlparser;
+
+import com.yahoo.document.Document;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.DocumentTypeManager;
+import com.yahoo.document.PositionDataType;
+import com.yahoo.document.datatypes.Struct;
+import org.junit.Test;
+
+import java.util.Iterator;
+
+import static org.junit.Assert.*;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class PositionParserTestCase {
+
+    @Test
+    public void requireThatPositionStringsCanBeParsed() throws Exception {
+        DocumentTypeManager mgr = new DocumentTypeManager();
+        mgr.register(PositionDataType.INSTANCE);
+        DocumentType docType = new DocumentType("my_doc");
+        docType.addField("my_pos", PositionDataType.INSTANCE);
+        mgr.registerDocumentType(docType);
+
+        VespaXMLFeedReader parser = new VespaXMLFeedReader("src/test/vespaxmlparser/test_position.xml", mgr);
+        Iterator<VespaXMLFeedReader.Operation> it = parser.readAll().iterator();
+        assertTrue(it.hasNext());
+        assertDocument(PositionDataType.valueOf(1, 2), it.next());
+        assertTrue(it.hasNext());
+        assertDocument(PositionDataType.fromString("E3;N4"), it.next());
+        assertTrue(it.hasNext());
+        assertDocument(PositionDataType.fromString("5;6"), it.next());
+        assertTrue(it.hasNext());
+        assertDocument(PositionDataType.fromString("7;8"), it.next());
+        assertFalse(it.hasNext());
+    }
+
+    private static void assertDocument(Struct expected, VespaXMLFeedReader.Operation operation) {
+        assertNotNull(operation);
+        assertEquals(VespaXMLFeedReader.OperationType.DOCUMENT, operation.getType());
+        Document doc = operation.getDocument();
+        assertNotNull(doc);
+        assertEquals(expected, doc.getFieldValue("my_pos"));
+    }
+}
diff --git a/document/src/test/java/com/yahoo/vespaxmlparser/UriParserTestCase.java b/document/src/test/java/com/yahoo/vespaxmlparser/UriParserTestCase.java
new file mode 100644
index 00000000000..2b5bf0f2b3c
--- /dev/null
+++ b/document/src/test/java/com/yahoo/vespaxmlparser/UriParserTestCase.java
@@ -0,0 +1,81 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespaxmlparser;
+
+import com.yahoo.document.*;
+import com.yahoo.document.datatypes.Array;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.document.update.AddValueUpdate;
+import com.yahoo.document.update.FieldUpdate;
+import com.yahoo.document.update.ValueUpdate;
+import org.junit.Test;
+
+import java.util.Iterator;
+
+import static org.junit.Assert.*;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class UriParserTestCase {
+
+    @Test
+    public void requireThatUriFieldsCanBeParsed() throws Exception {
+        DocumentTypeManager mgr = new DocumentTypeManager();
+        DocumentType docType = new DocumentType("my_doc");
+        docType.addField("my_uri", DataType.URI);
+        docType.addField("my_arr", DataType.getArray(DataType.URI));
+        mgr.registerDocumentType(docType);
+
+        VespaXMLFeedReader parser = new VespaXMLFeedReader("src/test/vespaxmlparser/test_uri.xml", mgr);
+        Iterator<VespaXMLFeedReader.Operation> it = parser.readAll().iterator();
+
+        Document doc = nextDocument(it);
+        assertNotNull(doc);
+        assertEquals(new StringFieldValue("scheme://host"), doc.getFieldValue("my_uri"));
+        assertNull(doc.getFieldValue("my_arr"));
+
+        assertNotNull(doc = nextDocument(it));
+        assertNull(doc.getFieldValue("my_uri"));
+        FieldValue val = doc.getFieldValue("my_arr");
+        assertNotNull(val);
+        assertTrue(val instanceof Array);
+        Array arr = (Array)val;
+        assertEquals(1, arr.size());
+        assertEquals(new StringFieldValue("scheme://host"), arr.get(0));
+
+        DocumentUpdate upd = nextUpdate(it);
+        assertNotNull(upd);
+        assertEquals(1, upd.getFieldUpdates().size());
+        FieldUpdate fieldUpd = upd.getFieldUpdate(0);
+        assertNotNull(fieldUpd);
+        assertEquals(docType.getField("my_arr"), fieldUpd.getField());
+        assertEquals(1, fieldUpd.getValueUpdates().size());
+        ValueUpdate valueUpd = fieldUpd.getValueUpdate(0);
+        assertNotNull(valueUpd);
+        assertTrue(valueUpd instanceof AddValueUpdate);
+        assertEquals(new StringFieldValue("scheme://host"), valueUpd.getValue());
+
+        assertFalse(it.hasNext());
+    }
+
+    private static Document nextDocument(Iterator<VespaXMLFeedReader.Operation> it) {
+        assertTrue(it.hasNext());
+        VespaXMLFeedReader.Operation op = it.next();
+        assertNotNull(op);
+        assertEquals(VespaXMLFeedReader.OperationType.DOCUMENT, op.getType());
+        Document doc = op.getDocument();
+        assertNotNull(doc);
+        return doc;
+    }
+
+    private static DocumentUpdate nextUpdate(Iterator<VespaXMLFeedReader.Operation> it) {
+        assertTrue(it.hasNext());
+        VespaXMLFeedReader.Operation op = it.next();
+        assertNotNull(op);
+        assertEquals(VespaXMLFeedReader.OperationType.UPDATE, op.getType());
+        DocumentUpdate upd = op.getDocumentUpdate();
+        assertNotNull(upd);
+        return upd;
+    }
+}
diff --git a/document/src/test/java/com/yahoo/vespaxmlparser/VespaXMLReaderTestCase.java b/document/src/test/java/com/yahoo/vespaxmlparser/VespaXMLReaderTestCase.java
new file mode 100755
index 00000000000..19f753484d3
--- /dev/null
+++ b/document/src/test/java/com/yahoo/vespaxmlparser/VespaXMLReaderTestCase.java
@@ -0,0 +1,894 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespaxmlparser;
+
+import com.yahoo.document.*;
+import com.yahoo.document.datatypes.*;
+import com.yahoo.document.fieldpathupdate.AddFieldPathUpdate;
+import com.yahoo.document.fieldpathupdate.AssignFieldPathUpdate;
+import com.yahoo.document.fieldpathupdate.RemoveFieldPathUpdate;
+import com.yahoo.document.serialization.DeserializationException;
+import com.yahoo.document.update.AddValueUpdate;
+import com.yahoo.document.update.MapValueUpdate;
+import com.yahoo.document.update.RemoveValueUpdate;
+import com.yahoo.document.update.ValueUpdate;
+import com.yahoo.text.Utf8;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.ByteArrayInputStream;
+import java.util.List;
+
+import static org.junit.Assert.*;
+
+/**
+ * Simple test case for Vespa XML parser.
+ *
+ * @author sveina
+ */
+public class VespaXMLReaderTestCase {
+
+    private final DocumentTypeManager manager = new DocumentTypeManager();
+
+    //NOTE: You generally want to extend com.yahoo.vespaxmlparser.test.documentxmltests.VespaXMLParserTestCase
+    //instead  -- and do remember to update the C++ test case also
+
+    @Before
+    public void setUp() {
+        DocumentTypeManagerConfigurer.configure(manager, "file:src/test/vespaxmlparser/documentmanager2.cfg");
+    }
+
+    @Test
+    public void testMapNoKey() throws Exception {
+        try {
+            VespaXMLFeedReader parser = new VespaXMLFeedReader("src/test/vespaxmlparser/testmapnokey.xml", manager);
+            parser.readAll();
+            assertTrue(false);
+        } catch (Exception e) {
+            System.out.println(e);
+        }
+    }
+
+    @Test
+    public void testMapNoValue() throws Exception {
+        try {
+            VespaXMLFeedReader parser = new VespaXMLFeedReader("src/test/vespaxmlparser/testmapnovalue.xml", manager);
+            parser.readAll();
+            assertTrue(false);
+        } catch (Exception e) {
+            System.out.println(e);
+        }
+    }
+
+    @Test
+    public void testNews1() throws Exception {
+        VespaXMLFeedReader parser = new VespaXMLFeedReader("src/test/vespaxmlparser/testalltypes.xml", manager);
+        VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation();
+        parser.read(op);
+
+        assertTrue(VespaXMLFeedReader.OperationType.INVALID != op.getType());
+        Document doc = op.getDocument();
+        assertEquals(new StringFieldValue("testUrl"), doc.getFieldValue("url"));
+        assertEquals(new StringFieldValue("testTitle"), doc.getFieldValue("title"));
+        assertEquals(new IntegerFieldValue(1), doc.getFieldValue("last_downloaded"));
+        assertEquals(new LongFieldValue(2), doc.getFieldValue("value_long"));
+        assertEquals("foobar", Utf8.toString(((Raw)doc.getFieldValue("value_raw")).getByteBuffer()));
+
+        Array strArr = (Array)doc.getFieldValue("stringarr");
+        assertEquals(new StringFieldValue("stringarrItem1"), strArr.get(0));
+        assertEquals(new StringFieldValue("stringarrItem2"), strArr.get(1));
+
+        Array intArr = (Array)doc.getFieldValue("intarr");
+        assertEquals(new IntegerFieldValue(-1311224359), intArr.get(0));
+        assertEquals(new IntegerFieldValue(-1311224358), intArr.get(1));
+        assertEquals(new IntegerFieldValue(-1), intArr.get(2));
+        assertEquals(new IntegerFieldValue(-2147483648), intArr.get(3));
+
+        Array longArr = (Array)doc.getFieldValue("longarr");
+        assertEquals(new LongFieldValue(5L), longArr.get(0));
+        assertEquals(new LongFieldValue(6L), longArr.get(1));
+
+        Array byteArr = (Array)doc.getFieldValue("bytearr");
+        assertEquals(new ByteFieldValue(7), byteArr.get(0));
+        assertEquals(new ByteFieldValue(8), byteArr.get(1));
+
+        Array floatArr = (Array)doc.getFieldValue("floatarr");
+        assertEquals(new FloatFieldValue(9.0f), floatArr.get(0));
+        assertEquals(new FloatFieldValue(10.0f), floatArr.get(1));
+
+        WeightedSet intWset = (WeightedSet)doc.getFieldValue("weightedsetint");
+        assertEquals(new Integer(11), intWset.get(new IntegerFieldValue(11)));
+        assertEquals(new Integer(12), intWset.get(new IntegerFieldValue(12)));
+
+        WeightedSet strWset = (WeightedSet)doc.getFieldValue("weightedsetstring");
+        assertEquals(new Integer(13), strWset.get(new StringFieldValue("string13")));
+        assertEquals(new Integer(14), strWset.get(new StringFieldValue("string14")));
+
+        MapFieldValue strMap = (MapFieldValue)doc.getFieldValue("stringmap");
+        assertEquals(new StringFieldValue("slovakia"), strMap.get(new StringFieldValue("italia")));
+        assertEquals(new StringFieldValue("japan"), strMap.get(new StringFieldValue("danmark")));
+        assertEquals(new StringFieldValue("new zealand"), strMap.get(new StringFieldValue("paraguay")));
+
+        Struct struct = (Struct)doc.getFieldValue("structfield");
+        assertEquals(new StringFieldValue("star wars"), struct.getFieldValue("title"));
+        assertEquals(new StringFieldValue("dummy"), struct.getFieldValue("structfield"));
+
+        List structArr = (List)doc.getFieldValue("structarr");
+        assertEquals(2, structArr.size());
+        assertEquals(new StringFieldValue("title1"), ((Struct)structArr.get(0)).getFieldValue("title"));
+        assertEquals(new StringFieldValue("title2"), ((Struct)structArr.get(1)).getFieldValue("title"));
+        assertEquals(new StringFieldValue("value1"),
+                     ((MapFieldValue)((Struct)structArr.get(0)).getFieldValue("mymap")).get(
+                             new StringFieldValue("key1")));
+        assertEquals(new StringFieldValue("value2"),
+                     ((MapFieldValue)((Struct)structArr.get(0)).getFieldValue("mymap")).get(
+                             new StringFieldValue("key2")));
+        assertEquals(new StringFieldValue("value1.1"),
+                     ((MapFieldValue)((Struct)structArr.get(1)).getFieldValue("mymap")).get(
+                             new StringFieldValue("key1.1")));
+        assertEquals(new StringFieldValue("value1.2"),
+                     ((MapFieldValue)((Struct)structArr.get(1)).getFieldValue("mymap")).get(
+                             new StringFieldValue("key1.2")));
+
+        MapFieldValue arrMap = (MapFieldValue)doc.getFieldValue("arrmap");
+        assertEquals(2, arrMap.size());
+        Array arr = (Array)arrMap.get(new StringFieldValue("foo"));
+        assertEquals(3, arr.size());
+        assertEquals(new StringFieldValue("hei1"), arr.get(0));
+        assertEquals(new StringFieldValue("hei2"), arr.get(1));
+        assertEquals(new StringFieldValue("hei3"), arr.get(2));
+        arr = (Array)arrMap.get(new StringFieldValue("bar"));
+        assertEquals(3, arr.size());
+        assertEquals(new StringFieldValue("hei4"), arr.get(0));
+        assertEquals(new StringFieldValue("hei5"), arr.get(1));
+        assertEquals(new StringFieldValue("hei6"), arr.get(2));
+    }
+
+    @Test
+    public void testNews3() throws Exception {
+        // Updating all elements in a documentType
+        VespaXMLFeedReader parser = new VespaXMLFeedReader("src/test/vespaxmlparser/test03.xml", manager);
+        VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation();
+        parser.read(op);
+
+        assertEquals(VespaXMLFeedReader.OperationType.UPDATE, op.getType());
+
+        DocumentUpdate docUpdate = op.getDocumentUpdate();
+
+        //url
+        assertEquals(new StringFieldValue("assignUrl"), docUpdate.getFieldUpdate("url").getValueUpdate(0).getValue());
+
+        //title
+        assertEquals(new StringFieldValue("assignTitle"),
+                     docUpdate.getFieldUpdate("title").getValueUpdate(0).getValue());
+
+        //last_downloaded
+        assertEquals(new IntegerFieldValue(1),
+                     docUpdate.getFieldUpdate("last_downloaded").getValueUpdate(0).getValue());
+
+        //value_long
+        assertEquals(new LongFieldValue((long)2), docUpdate.getFieldUpdate("value_long").getValueUpdate(0).getValue());
+
+        //stringarr
+        List stringarr = (List)docUpdate.getFieldUpdate("stringarr").getValueUpdate(0).getValue();
+        assertEquals(new StringFieldValue("assignString1"), stringarr.get(0));
+        assertEquals(new StringFieldValue("assignString2"), stringarr.get(1));
+
+        //intarr
+        List intarr = (List)docUpdate.getFieldUpdate("intarr").getValueUpdate(0).getValue();
+        assertEquals(new IntegerFieldValue(3), intarr.get(0));
+        assertEquals(new IntegerFieldValue(4), intarr.get(1));
+
+        //longarr
+        List longarr = (List)docUpdate.getFieldUpdate("longarr").getValueUpdate(0).getValue();
+        assertEquals(new LongFieldValue((long)5), longarr.get(0));
+        assertEquals(new LongFieldValue((long)6), longarr.get(1));
+
+        //bytearr
+        List bytearr = (List)docUpdate.getFieldUpdate("bytearr").getValueUpdate(0).getValue();
+        assertEquals(new ByteFieldValue((byte)7), bytearr.get(0));
+        assertEquals(new ByteFieldValue((byte)8), bytearr.get(1));
+
+        //floatarr
+        List floatarr = (List)docUpdate.getFieldUpdate("floatarr").getValueUpdate(0).getValue();
+        assertEquals(new FloatFieldValue((float)9), floatarr.get(0));
+        assertEquals(new FloatFieldValue((float)10), floatarr.get(1));
+
+        //weightedsetint
+        WeightedSet weightedsetint =
+                (WeightedSet)docUpdate.getFieldUpdate("weightedsetint").getValueUpdate(0).getValue();
+        assertEquals(new Integer(11), weightedsetint.get(new IntegerFieldValue(11)));
+        assertEquals(new Integer(12), weightedsetint.get(new IntegerFieldValue(12)));
+
+        //weightedsetstring
+        WeightedSet weightedsetstring =
+                (WeightedSet)docUpdate.getFieldUpdate("weightedsetstring").getValueUpdate(0).getValue();
+        assertEquals(new Integer(13), weightedsetstring.get(new StringFieldValue("assign13")));
+        assertEquals(new Integer(14), weightedsetstring.get(new StringFieldValue("assign14")));
+
+    }
+
+    @Test
+    public void testNews4() throws Exception {
+        // Test on adding just a few fields to a DocumentUpdate (implies other fields to null)
+        VespaXMLFeedReader parser = new VespaXMLFeedReader("src/test/vespaxmlparser/test04.xml", manager);
+
+        VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation();
+        parser.read(op);
+
+        assertEquals(VespaXMLFeedReader.OperationType.UPDATE, op.getType());
+
+        DocumentUpdate docUpdate = op.getDocumentUpdate();
+        //url
+        assertEquals(new StringFieldValue("assignUrl"), docUpdate.getFieldUpdate("url").getValueUpdate(0).getValue());
+
+        //title
+        assertNull(docUpdate.getFieldUpdate("title"));
+
+        //last_downloaded
+        assertNull(docUpdate.getFieldUpdate("last_downloaded"));
+
+        //value_long
+        assertEquals(new LongFieldValue((long)2), docUpdate.getFieldUpdate("value_long").getValueUpdate(0).getValue());
+
+        //value_content
+        assertNull(docUpdate.getFieldUpdate("value_content"));
+
+        //stringarr
+        List stringarr = (List)docUpdate.getFieldUpdate("stringarr").getValueUpdate(0).getValue();
+        assertEquals(new StringFieldValue("assignString1"), stringarr.get(0));
+        assertEquals(new StringFieldValue("assignString2"), stringarr.get(1));
+
+        //intarr
+        List intarr = (List)docUpdate.getFieldUpdate("intarr").getValueUpdate(0).getValue();
+        assertEquals(new IntegerFieldValue(3), intarr.get(0));
+        assertEquals(new IntegerFieldValue(4), intarr.get(1));
+
+        //longarr
+        assertNull(docUpdate.getFieldUpdate("longarr"));
+
+        //bytearr
+        assertNull(docUpdate.getFieldUpdate("bytearr"));
+
+        //floatarr
+        assertNull(docUpdate.getFieldUpdate("floatarr"));
+
+        //weightedsetint
+        WeightedSet weightedsetint =
+                (WeightedSet)docUpdate.getFieldUpdate("weightedsetint").getValueUpdate(0).getValue();
+        assertEquals(new Integer(11), weightedsetint.get(new IntegerFieldValue(11)));
+        assertEquals(new Integer(12), weightedsetint.get(new IntegerFieldValue(12)));
+
+        //weightedsetstring
+        assertNull(docUpdate.getFieldUpdate("weightedsetstring"));
+    }
+
+    @Test
+    public void testNews5() throws Exception {
+        // Adding a few new fields to a Document using different syntax
+        VespaXMLFeedReader parser = new VespaXMLFeedReader("src/test/vespaxmlparser/test05.xml", manager);
+
+        VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation();
+        parser.read(op);
+
+        assertEquals(VespaXMLFeedReader.OperationType.UPDATE, op.getType());
+
+        DocumentUpdate docUpdate = op.getDocumentUpdate();
+
+        //url
+        assertNull(docUpdate.getFieldUpdate("url"));
+
+        //title
+        assertNull(docUpdate.getFieldUpdate("title"));
+
+        //last_downloaded
+        assertNull(docUpdate.getFieldUpdate("last_downloaded"));
+
+        //value_long
+        //assertNull(docUpdate.getFieldUpdate("value_long"));
+
+        //value_content
+        assertNull(docUpdate.getFieldUpdate("value_content"));
+
+        //stringarr
+        List stringarr = docUpdate.getFieldUpdate("stringarr").getValueUpdates();//.getValueUpdate(0).getValue();
+        assertEquals(new StringFieldValue("addString1"), ((ValueUpdate)stringarr.get(0)).getValue());
+        assertEquals(new StringFieldValue("addString2"), ((ValueUpdate)stringarr.get(1)).getValue());
+
+        //intarr
+        assertNull(docUpdate.getFieldUpdate("intarr"));
+
+        //longarr
+        List longarr = docUpdate.getFieldUpdate("longarr").getValueUpdates();
+        assertEquals(new LongFieldValue((long)5), ((ValueUpdate)longarr.get(0)).getValue());
+
+        //bytearr
+        assertNull(docUpdate.getFieldUpdate("bytearr"));
+
+        //floatarr
+        assertNull(docUpdate.getFieldUpdate("floatarr"));
+
+        //weightedsetint
+        assertEquals(new IntegerFieldValue(11), docUpdate.getFieldUpdate("weightedsetint").getValueUpdate(0).getValue());
+        assertEquals(new Integer(11),
+                     (Integer)((AddValueUpdate)docUpdate.getFieldUpdate("weightedsetint").getValueUpdate(0))
+                             .getWeight());
+        assertEquals(new IntegerFieldValue(12), docUpdate.getFieldUpdate("weightedsetint").getValueUpdate(1).getValue());
+        assertEquals(new Integer(12),
+                     (Integer)((AddValueUpdate)docUpdate.getFieldUpdate("weightedsetint").getValueUpdate(1))
+                             .getWeight());
+
+        //weightedsetstring
+        assertEquals(new StringFieldValue("add13"), docUpdate.getFieldUpdate("weightedsetstring").getValueUpdate(0).getValue());
+        assertEquals(new Integer(1),
+                     (Integer)((AddValueUpdate)docUpdate.getFieldUpdate("weightedsetstring").getValueUpdate(0))
+                             .getWeight());
+    }
+
+    @Test
+    public void testNews6() throws Exception {
+        // A document containing fields with invalid values. Different variants are used All of the updates specified in
+        // XML-file should be skipped and not added to queue Except the three of them
+        VespaXMLFeedReader parser = new VespaXMLFeedReader("src/test/vespaxmlparser/test06.xml", manager);
+
+        // long value with txt
+        try {
+            parser.read(new VespaXMLFeedReader.Operation());
+            fail();
+        } catch (Exception e) {
+            System.out.println(e.getMessage());
+        }
+
+        // empty string
+        VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation();
+        parser.read(op);
+        assertEquals("doc:news:http://news6b", op.getDocument().getId().toString());
+
+        // int array with text
+        try {
+            parser.read(new VespaXMLFeedReader.Operation());
+            fail();
+        } catch (Exception e) {
+            System.out.println(e.getMessage());
+        }
+
+        // long array with whitespace
+        try {
+            parser.read(new VespaXMLFeedReader.Operation());
+            fail();
+        } catch (Exception e) {
+            System.out.println(e.getMessage());
+        }
+
+        // byte array with value
+        try {
+            parser.read(new VespaXMLFeedReader.Operation());
+            fail();
+        } catch (Exception e) {
+            System.out.println(e.getMessage());
+        }
+
+        // float array with string
+        try {
+            parser.read(new VespaXMLFeedReader.Operation());
+            fail();
+        } catch (Exception e) {
+            System.out.println(e.getMessage());
+        }
+
+        // weighted set of int with string
+        try {
+            parser.read(new VespaXMLFeedReader.Operation());
+            fail();
+        } catch (Exception e) {
+            System.out.println(e.getMessage());
+        }
+
+        // weighted set of int with string as weight
+        try {
+            parser.read(new VespaXMLFeedReader.Operation());
+            fail();
+        } catch (Exception e) {
+            System.out.println(e.getMessage());
+        }
+
+        // weighted set of string with string as weight
+        try {
+            parser.read(new VespaXMLFeedReader.Operation());
+            fail();
+        } catch (Exception e) {
+            System.out.println(e.getMessage());
+        }
+
+        parser.read(op = new VespaXMLFeedReader.Operation());
+        assertEquals("doc:news:http://news6j", op.getDocument().getId().toString());
+
+        parser.read(op = new VespaXMLFeedReader.Operation());
+        assertEquals(VespaXMLFeedReader.OperationType.INVALID, op.getType());
+    }
+
+    @Test
+    public void testNews7() throws Exception {
+        // Testing different variants of increment/decrement/multiply/divide among with alterupdate. In test07.xml there
+        // are also some updates that will fail (be skipped).
+        VespaXMLFeedReader parser = new VespaXMLFeedReader("src/test/vespaxmlparser/test07.xml", manager);
+
+        VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation();
+        parser.read(op);
+
+        assertEquals(VespaXMLFeedReader.OperationType.UPDATE, op.getType());
+
+        DocumentUpdate docUpdate = op.getDocumentUpdate();
+
+        List<ValueUpdate> vuList = docUpdate.getFieldUpdate("last_downloaded").getValueUpdates();
+        assertEquals(new DoubleFieldValue(2.0), vuList.get(0).getValue());
+        assertEquals(new DoubleFieldValue(3.0), vuList.get(1).getValue());
+        assertEquals(new DoubleFieldValue(4.0), vuList.get(2).getValue());
+        assertEquals(new DoubleFieldValue(5.0), vuList.get(3).getValue());
+
+        assertEquals(new IntegerFieldValue(7), docUpdate.getFieldUpdate("weightedsetint").getValueUpdate(0).getValue());
+        assertEquals(new DoubleFieldValue(6.0), ((MapValueUpdate)docUpdate.getFieldUpdate("weightedsetint").getValueUpdate(0))
+                .getUpdate().getValue());
+
+        assertEquals(new IntegerFieldValue(9), docUpdate.getFieldUpdate("weightedsetint").getValueUpdate(1).getValue());
+        assertEquals(new DoubleFieldValue(8.0), ((MapValueUpdate)docUpdate.getFieldUpdate("weightedsetint").getValueUpdate(1))
+                .getUpdate().getValue());
+
+        assertEquals(new IntegerFieldValue(11), docUpdate.getFieldUpdate("intarr").getValueUpdate(0).getValue());
+        assertEquals(new DoubleFieldValue(10.0),
+                     ((MapValueUpdate)docUpdate.getFieldUpdate("intarr").getValueUpdate(0)).getUpdate().getValue());
+
+        assertEquals(new IntegerFieldValue(13), docUpdate.getFieldUpdate("floatarr").getValueUpdate(0).getValue());
+        assertEquals(new DoubleFieldValue(12.0),
+                     ((MapValueUpdate)docUpdate.getFieldUpdate("floatarr").getValueUpdate(0)).getUpdate().getValue());
+
+        assertEquals(new IntegerFieldValue(15), docUpdate.getFieldUpdate("floatarr").getValueUpdate(1).getValue());
+        assertEquals(new DoubleFieldValue(14.0),
+                     ((MapValueUpdate)docUpdate.getFieldUpdate("floatarr").getValueUpdate(1)).getUpdate().getValue());
+
+        // Trying arithmetic on string (b)
+        try {
+            parser.read(new VespaXMLFeedReader.Operation());
+            fail();
+        } catch (Exception e) {
+            System.out.println(e.getMessage());
+        }
+
+        // "By" as string (c)
+        try {
+            parser.read(new VespaXMLFeedReader.Operation());
+            fail();
+        } catch (Exception e) {
+            System.out.println(e.getMessage());
+        }
+
+        // Empty key in weighted set of int (d)
+        try {
+            parser.read(new VespaXMLFeedReader.Operation());
+            fail();
+        } catch (Exception e) {
+            System.out.println(e.getMessage());
+        }
+
+        // No "by" attribute (e)
+        try {
+            parser.read(new VespaXMLFeedReader.Operation());
+            fail();
+        } catch (Exception e) {
+            System.out.println(e.getMessage());
+        }
+
+        // Float key as string (f)
+        try {
+            parser.read(new VespaXMLFeedReader.Operation());
+            fail();
+        } catch (Exception e) {
+            System.out.println(e.getMessage());
+        }
+    }
+
+    @Test
+    public void testNews8() throws Exception {
+        VespaXMLFeedReader parser = new VespaXMLFeedReader("src/test/vespaxmlparser/test08.xml", manager);
+
+        VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation();
+        parser.read(op);
+
+        assertEquals(VespaXMLFeedReader.OperationType.UPDATE, op.getType());
+
+        DocumentUpdate docUpdate = op.getDocumentUpdate();
+
+        //stringarr
+        List<ValueUpdate> vuList = docUpdate.getFieldUpdate("stringarr").getValueUpdates();
+        assertTrue(vuList.get(0) instanceof RemoveValueUpdate);
+        assertEquals(new StringFieldValue("removeString1"), vuList.get(0).getValue());
+        assertTrue(vuList.get(1) instanceof RemoveValueUpdate);
+        assertEquals(new StringFieldValue("removeString2"), vuList.get(1).getValue());
+
+        //weightedsetint
+        vuList = docUpdate.getFieldUpdate("weightedsetint").getValueUpdates();
+        assertEquals(2, vuList.size());
+        assertTrue(vuList.contains(new RemoveValueUpdate(new IntegerFieldValue(5))));
+        assertTrue(vuList.contains(new RemoveValueUpdate(new IntegerFieldValue(4))));
+    }
+
+    @Test
+    public void testNews9() throws Exception {
+        VespaXMLFeedReader parser = new VespaXMLFeedReader("src/test/vespaxmlparser/test09.xml", manager);
+
+        {
+            VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation();
+            parser.read(op);
+
+            assertEquals(VespaXMLFeedReader.OperationType.REMOVE, op.getType());
+            assertEquals("doc:news:http://news9a", op.getRemove().toString());
+        }
+        {
+            VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation();
+            parser.read(op);
+
+            assertEquals(VespaXMLFeedReader.OperationType.REMOVE, op.getType());
+            assertEquals("doc:news:http://news9b", op.getRemove().toString());
+        }
+        {
+            // Remove without documentid. Not supported.
+            VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation();
+            try {
+                parser.read(op);
+                fail();
+            } catch (Exception e) {
+                System.out.println(e.getMessage());
+            }
+        }
+    }
+
+    @Test
+    public void testNews10() throws Exception {
+        VespaXMLFeedReader parser = new VespaXMLFeedReader("src/test/vespaxmlparser/test10.xml", manager);
+        {
+            VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation();
+            parser.read(op);
+            Document doc = op.getDocument();
+
+            assertEquals(new StringFieldValue("testUrl"), doc.getFieldValue("url"));
+            assertEquals(new StringFieldValue("testTitle"), doc.getFieldValue("title"));
+            assertEquals(new IntegerFieldValue(1), doc.getFieldValue("last_downloaded"));
+            assertEquals(new LongFieldValue(2), doc.getFieldValue("value_long"));
+
+            Array strArr = (Array)doc.getFieldValue("stringarr");
+            assertEquals(new StringFieldValue("stringarrItem1"), strArr.get(0));
+            assertEquals(new StringFieldValue("stringarrItem2"), strArr.get(1));
+
+            Array intArr = (Array)doc.getFieldValue("intarr");
+            assertEquals(new IntegerFieldValue(3), intArr.get(0));
+            assertEquals(new IntegerFieldValue(4), intArr.get(1));
+
+            Array longArr = (Array)doc.getFieldValue("longarr");
+            assertEquals(new LongFieldValue(5L), longArr.get(0));
+            assertEquals(new LongFieldValue(6L), longArr.get(1));
+
+            Array byteArr = (Array)doc.getFieldValue("bytearr");
+            assertEquals(new ByteFieldValue((byte)7), byteArr.get(0));
+            assertEquals(new ByteFieldValue((byte)8), byteArr.get(1));
+
+            Array floatArr = (Array)doc.getFieldValue("floatarr");
+            assertEquals(new FloatFieldValue(9.0f), floatArr.get(0));
+            assertEquals(new FloatFieldValue(10.0f), floatArr.get(1));
+
+            WeightedSet intWset = (WeightedSet)doc.getFieldValue("weightedsetint");
+            assertEquals(new Integer(11), intWset.get(new IntegerFieldValue(11)));
+            assertEquals(new Integer(12), intWset.get(new IntegerFieldValue(12)));
+
+            WeightedSet strWset = (WeightedSet)doc.getFieldValue("weightedsetstring");
+            assertEquals(new Integer(13), strWset.get(new StringFieldValue("string13")));
+            assertEquals(new Integer(14), strWset.get(new StringFieldValue("string14")));
+        }
+        {
+            VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation();
+            parser.read(op);
+            Document doc = op.getDocument();
+            assertNotNull(doc);
+            assertEquals(new StringFieldValue("testUrl2"), doc.getFieldValue("url"));
+        }
+        {
+            VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation();
+            parser.read(op);
+            DocumentUpdate upd = op.getDocumentUpdate();
+
+            assertNull(upd.getFieldUpdate("url"));
+            assertNull(upd.getFieldUpdate("title"));
+            assertNull(upd.getFieldUpdate("last_downloaded"));
+            assertNull(upd.getFieldUpdate("value_long"));
+            assertNull(upd.getFieldUpdate("value_content"));
+
+            List<ValueUpdate> lst = upd.getFieldUpdate("stringarr").getValueUpdates();
+            assertEquals(new StringFieldValue("addString1"), lst.get(0).getValue());
+            assertEquals(new StringFieldValue("addString2"), lst.get(1).getValue());
+
+            assertNull(upd.getFieldUpdate("intarr"));
+
+            lst = upd.getFieldUpdate("longarr").getValueUpdates();
+            assertEquals(new LongFieldValue((long)5), lst.get(0).getValue());
+
+            assertNull(upd.getFieldUpdate("bytearr"));
+            assertNull(upd.getFieldUpdate("floatarr"));
+
+            assertEquals(new IntegerFieldValue(11), upd.getFieldUpdate("weightedsetint").getValueUpdate(0).getValue());
+            assertEquals(new Integer(11),
+                         (Integer)((AddValueUpdate)upd.getFieldUpdate("weightedsetint").getValueUpdate(0))
+                                 .getWeight());
+            assertEquals(new IntegerFieldValue(12), upd.getFieldUpdate("weightedsetint").getValueUpdate(1).getValue());
+            assertEquals(new Integer(12),
+                         (Integer)((AddValueUpdate)upd.getFieldUpdate("weightedsetint").getValueUpdate(1))
+                                 .getWeight());
+
+            assertEquals(new StringFieldValue("add13"), upd.getFieldUpdate("weightedsetstring").getValueUpdate(0).getValue());
+            assertEquals(new Integer(1),
+                         (Integer)((AddValueUpdate)upd.getFieldUpdate("weightedsetstring").getValueUpdate(0))
+                                 .getWeight());
+        }
+        {
+            VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation();
+            parser.read(op);
+            DocumentUpdate upd = op.getDocumentUpdate();
+
+            assertEquals(new StringFieldValue("assignUrl"),
+                         upd.getFieldUpdate("url").getValueUpdate(0).getValue());
+
+            assertNull(upd.getFieldUpdate("title"));
+            assertNull(upd.getFieldUpdate("last_downloaded"));
+            assertEquals(new LongFieldValue(2),
+                         upd.getFieldUpdate("value_long").getValueUpdate(0).getValue());
+            assertNull(upd.getFieldUpdate("value_content"));
+
+            Array strArr = (Array)upd.getFieldUpdate("stringarr").getValueUpdate(0).getValue();
+            assertEquals(new StringFieldValue("assignString1"), strArr.get(0));
+            assertEquals(new StringFieldValue("assignString2"), strArr.get(1));
+
+            Array intArr = (Array)upd.getFieldUpdate("intarr").getValueUpdate(0).getValue();
+            assertEquals(new IntegerFieldValue(3), intArr.get(0));
+            assertEquals(new IntegerFieldValue(4), intArr.get(1));
+
+            assertNull(upd.getFieldUpdate("longarr"));
+            assertNull(upd.getFieldUpdate("bytearr"));
+            assertNull(upd.getFieldUpdate("floatarr"));
+
+            WeightedSet intWset = (WeightedSet)upd.getFieldUpdate("weightedsetint").getValueUpdate(0).getValue();
+            assertEquals(new Integer(11), intWset.get(new IntegerFieldValue(11)));
+            assertEquals(new Integer(12), intWset.get(new IntegerFieldValue(12)));
+
+            assertNull(upd.getFieldUpdate("weightedsetstring"));
+        }
+        {
+            VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation();
+            parser.read(op);
+            assertEquals("doc:news:http://news10e", op.getRemove().toString());
+        }
+        {
+            // Illegal remove without documentid attribute
+            try {
+                VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation();
+                parser.read(op);
+                fail();
+            } catch (Exception e) {
+                System.out.println(e.getMessage());
+            }
+        }
+    }
+
+    @Test
+    public void testFieldPathUpdates() throws Exception {
+        // Adding a few new fields to a Document using different syntax
+        VespaXMLFeedReader parser = new VespaXMLFeedReader("src/tests/vespaxml/fieldpathupdates.xml", manager);
+
+        VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation();
+        parser.read(op);
+
+        assertEquals(VespaXMLFeedReader.OperationType.UPDATE, op.getType());
+
+        DocumentUpdate docUpdate = op.getDocumentUpdate();
+
+        assertEquals(20, docUpdate.getFieldPathUpdates().size());
+
+        {
+            AssignFieldPathUpdate ass = (AssignFieldPathUpdate)docUpdate.getFieldPathUpdates().get(0);
+            assertEquals("url", ass.getOriginalFieldPath());
+            assertEquals(new StringFieldValue("assignUrl"), ass.getNewValue());
+        }
+        {
+            AssignFieldPathUpdate ass = (AssignFieldPathUpdate)docUpdate.getFieldPathUpdates().get(1);
+            assertEquals("title", ass.getOriginalFieldPath());
+            assertEquals(new StringFieldValue("assignTitle"), ass.getNewValue());
+        }
+        {
+            AssignFieldPathUpdate ass = (AssignFieldPathUpdate)docUpdate.getFieldPathUpdates().get(2);
+            assertEquals("last_downloaded", ass.getOriginalFieldPath());
+            assertEquals("1", ass.getExpression());
+        }
+        {
+            AssignFieldPathUpdate ass = (AssignFieldPathUpdate)docUpdate.getFieldPathUpdates().get(3);
+            assertEquals("value_long", ass.getOriginalFieldPath());
+            assertEquals("2", ass.getExpression());
+        }
+        {
+            AssignFieldPathUpdate ass = (AssignFieldPathUpdate)docUpdate.getFieldPathUpdates().get(5);
+            assertEquals("stringarr", ass.getOriginalFieldPath());
+            assertEquals("[assignString1, assignString2]", ass.getNewValue().toString());
+        }
+        {
+            AssignFieldPathUpdate ass = (AssignFieldPathUpdate)docUpdate.getFieldPathUpdates().get(6);
+            assertEquals("intarr", ass.getOriginalFieldPath());
+            assertEquals("[3, 4]", ass.getNewValue().toString());
+        }
+        {
+            AssignFieldPathUpdate ass = (AssignFieldPathUpdate)docUpdate.getFieldPathUpdates().get(7);
+            assertEquals("longarr", ass.getOriginalFieldPath());
+            assertEquals("[5, 6]", ass.getNewValue().toString());
+        }
+        {
+            AssignFieldPathUpdate ass = (AssignFieldPathUpdate)docUpdate.getFieldPathUpdates().get(8);
+            assertEquals("bytearr", ass.getOriginalFieldPath());
+            assertEquals("[7, 8]", ass.getNewValue().toString());
+        }
+        {
+            AssignFieldPathUpdate ass = (AssignFieldPathUpdate)docUpdate.getFieldPathUpdates().get(9);
+            assertEquals("floatarr", ass.getOriginalFieldPath());
+            assertEquals("[9.0, 10.0]", ass.getNewValue().toString());
+        }
+        {
+            AssignFieldPathUpdate ass = (AssignFieldPathUpdate)docUpdate.getFieldPathUpdates().get(10);
+            assertEquals("weightedsetint", ass.getOriginalFieldPath());
+            WeightedSet set = (WeightedSet)ass.getNewValue();
+            assertEquals(new Integer(11), set.get(new IntegerFieldValue(11)));
+            assertEquals(new Integer(12), set.get(new IntegerFieldValue(12)));
+        }
+        {
+            AssignFieldPathUpdate ass = (AssignFieldPathUpdate)docUpdate.getFieldPathUpdates().get(11);
+            assertEquals("weightedsetstring", ass.getOriginalFieldPath());
+            WeightedSet set = (WeightedSet)ass.getNewValue();
+            assertEquals(new Integer(13), set.get(new StringFieldValue("assign13")));
+            assertEquals(new Integer(14), set.get(new StringFieldValue("assign14")));
+        }
+        {
+            AddFieldPathUpdate ass = (AddFieldPathUpdate)docUpdate.getFieldPathUpdates().get(12);
+            assertEquals("stringarr", ass.getOriginalFieldPath());
+            assertEquals("[addString1, addString2]", ass.getNewValues().toString());
+        }
+        {
+            AddFieldPathUpdate ass = (AddFieldPathUpdate)docUpdate.getFieldPathUpdates().get(13);
+            assertEquals("longarr", ass.getOriginalFieldPath());
+            assertEquals("[5]", ass.getNewValues().toString());
+        }
+        {
+            AssignFieldPathUpdate ass = (AssignFieldPathUpdate)docUpdate.getFieldPathUpdates().get(14);
+            assertEquals("weightedsetint{13}", ass.getOriginalFieldPath());
+            assertEquals("13", ass.getExpression());
+        }
+        {
+            AssignFieldPathUpdate ass = (AssignFieldPathUpdate)docUpdate.getFieldPathUpdates().get(15);
+            assertEquals("weightedsetint{14}", ass.getOriginalFieldPath());
+            assertEquals("14", ass.getExpression());
+        }
+        {
+            AssignFieldPathUpdate ass = (AssignFieldPathUpdate)docUpdate.getFieldPathUpdates().get(16);
+            assertEquals("weightedsetstring{add13}", ass.getOriginalFieldPath());
+            assertEquals("1", ass.getExpression());
+        }
+        {
+            AssignFieldPathUpdate ass = (AssignFieldPathUpdate)docUpdate.getFieldPathUpdates().get(17);
+            assertEquals("weightedsetstring{assign13}", ass.getOriginalFieldPath());
+            assertEquals("130", ass.getExpression());
+        }
+        {
+            RemoveFieldPathUpdate ass = (RemoveFieldPathUpdate)docUpdate.getFieldPathUpdates().get(18);
+            assertEquals("weightedsetstring{assign14}", ass.getOriginalFieldPath());
+        }
+        {
+            RemoveFieldPathUpdate ass = (RemoveFieldPathUpdate)docUpdate.getFieldPathUpdates().get(19);
+            assertEquals("bytearr", ass.getOriginalFieldPath());
+        }
+        Document doc = new Document(manager.getDocumentType("news"), new DocumentId("doc:test:test:test"));
+        docUpdate.applyTo(doc);
+    }
+
+    @Test
+    public void testDocInDoc() throws Exception {
+        DocumentTypeManager m = new DocumentTypeManager();
+        DocumentTypeManagerConfigurer
+                .configure(m, "file:src/test/java/com/yahoo/document/documentmanager.docindoc.cfg");
+
+        VespaXMLFeedReader parser = new VespaXMLFeedReader("src/test/vespaxmlparser/test_docindoc.xml", m);
+        List<VespaXMLFeedReader.Operation> ops = parser.readAll();
+
+        assertEquals(1, ops.size());
+        VespaXMLFeedReader.Operation op = ops.get(0);
+        System.err.println(op);
+
+        assertEquals(VespaXMLFeedReader.OperationType.DOCUMENT, op.getType());
+        assertNull(op.getRemove());
+        assertNull(op.getDocumentUpdate());
+        assertNull(op.getFeedOperation());
+        assertNotNull(op.getDocument());
+
+        Document doc = op.getDocument();
+
+        assertEquals("outerdoc", doc.getDataType().getName());
+        assertEquals("doc:outer:this:is:outer:doc", doc.getId().toString());
+        assertEquals(1, doc.getFieldCount());
+
+        Array lst = (Array)doc.getFieldValue("innerdocuments");
+        assertNotNull(lst);
+        assertEquals(3, lst.size());
+
+        Document child = (Document)lst.get(0);
+        assertEquals(2, child.getFieldCount());
+        assertEquals("Peter Sellers", child.getFieldValue("name").toString());
+        assertEquals("Comedian", child.getFieldValue("content").toString());
+
+        child = (Document)lst.get(1);
+        assertEquals(2, child.getFieldCount());
+        assertEquals("Ole Olsen", child.getFieldValue("name").toString());
+        assertEquals("Common man", child.getFieldValue("content").toString());
+
+        child = (Document)lst.get(2);
+        assertEquals(2, child.getFieldCount());
+        assertEquals("Stein Nilsen", child.getFieldValue("name").toString());
+        assertEquals("Worker", child.getFieldValue("content").toString());
+    }
+
+    @Test(expected = DeserializationException.class)
+    public void testBinaryEncodingStrings() throws Exception {
+        DocumentTypeManager dtm = new DocumentTypeManager();
+
+        DocumentType type = new DocumentType("foo");
+        type.addField(new Field("title", DataType.STRING));
+
+        dtm.registerDocumentType(type);
+
+        String input =
+                "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n" +
+                        "<vespafeed>\n" +
+                        "    <document documenttype=\"foo\" documentid=\"doc:foo:bar:baz\"> \n" +
+                        "        <title binaryencoding=\"base64\">testTitle</title>\n" +
+                        "    </document>\n" +
+                        "</vespafeed>\n";
+
+        VespaXMLFeedReader parser = new VespaXMLFeedReader(new ByteArrayInputStream(Utf8.toBytes(input)), dtm);
+        parser.readAll();
+    }
+
+    @Test(expected = DeserializationException.class)
+    public void testIllegalCharacterInStrings() throws Exception {
+        DocumentTypeManager dtm = new DocumentTypeManager();
+
+        DocumentType type = new DocumentType("foo");
+        type.addField(new Field("title", DataType.STRING));
+
+        dtm.registerDocumentType(type);
+
+        String input =
+                "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n" +
+                        "<vespafeed>\n" +
+                        "    <document documenttype=\"foo\" documentid=\"doc:foo:bar:baz\"> \n" +
+                        "        <title>test\uFDDFTitle</title>\n" +
+                        "    </document>\n" +
+                        "</vespafeed>\n";
+
+        VespaXMLFeedReader parser = new VespaXMLFeedReader(new ByteArrayInputStream(Utf8.toBytes(input)), dtm);
+        parser.readAll();
+    }
+
+    @Test
+    public void testTestAndSetConditionAttribute() throws Exception {
+        VespaXMLFeedReader parser = new VespaXMLFeedReader("src/test/vespaxmlparser/testandset.xml", manager);
+        final int NUM_OPERATIONS_IN_FEED = 3;
+
+        for (int i = 0; i < NUM_OPERATIONS_IN_FEED; i++) {
+            VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation();
+            parser.read(op);
+
+            assertTrue("Missing test and set condition", op.getCondition().isPresent());
+            assertEquals("Condition is not the same as in xml feed", "news.value_long == 1", op.getCondition().getSelection());
+        }
+    }
+
+}
diff --git a/document/src/test/java/com/yahoo/vespaxmlparser/VespaXmlFieldReaderTestCase.java b/document/src/test/java/com/yahoo/vespaxmlparser/VespaXmlFieldReaderTestCase.java
new file mode 100644
index 00000000000..3cfbcac5b62
--- /dev/null
+++ b/document/src/test/java/com/yahoo/vespaxmlparser/VespaXmlFieldReaderTestCase.java
@@ -0,0 +1,181 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespaxmlparser;
+
+import com.yahoo.document.*;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.IntegerFieldValue;
+import com.yahoo.document.datatypes.PredicateFieldValue;
+import com.yahoo.document.datatypes.Struct;
+import com.yahoo.document.predicate.BinaryFormat;
+import com.yahoo.document.predicate.Conjunction;
+import com.yahoo.document.predicate.FeatureRange;
+import com.yahoo.document.predicate.FeatureSet;
+import com.yahoo.document.predicate.Predicate;
+import com.yahoo.document.serialization.DeserializationException;
+import org.apache.commons.codec.binary.Base64;
+import org.junit.Test;
+
+import javax.xml.stream.XMLStreamReader;
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.mockito.Mockito.*;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a>
+ */
+public class VespaXmlFieldReaderTestCase {
+
+    @Test
+    public void requireThatPredicateFieldValuesCanBeRead() throws Exception {
+        assertReadable(new Conjunction(new FeatureSet("foo", "bar"),
+                                       new FeatureRange("baz", 6L, 9L)));
+    }
+
+    @Test
+    public void requireThatArrayItemDeserializeExceptionIncludesFieldName() throws Exception {
+        assertThrows(new Field("my_field", DataType.getArray(DataType.BYTE)),
+                     "<item>-129</item>",
+                     "Field 'my_field': Invalid byte \"-129\". (at line 1, column 70)");
+    }
+
+    @Test
+    public void requireThatMapKeyDeserializeExceptionIncludesFieldName() throws Exception {
+        assertThrows(new Field("my_field", DataType.getMap(DataType.BYTE, DataType.STRING)),
+                     "<item><key>-129</key><value>foo</value></item>",
+                     "Field 'my_field': Invalid byte \"-129\". (at line 1, column 74)");
+    }
+
+    @Test
+    public void requireThatMapValueDeserializeExceptionIncludesFieldName() throws Exception {
+        assertThrows(new Field("my_field", DataType.getMap(DataType.STRING, DataType.BYTE)),
+                     "<item><key>foo</key><value>-129</value></item>",
+                     "Field 'my_field': Invalid byte \"-129\". (at line 1, column 92)");
+    }
+
+    @Test
+    public void requireThatStructFieldDeserializeExceptionIncludesFieldName() throws Exception {
+        StructDataType structType = new StructDataType("my_struct");
+        structType.addField(new Field("my_byte", DataType.BYTE));
+        assertThrows(new Field("my_field", structType),
+                     "<my_byte>-129</my_byte>",
+                     "Field 'my_byte': Invalid byte \"-129\". (at line 1, column 76)");
+    }
+
+    @Test
+    public void requireThatWSetItemDeserializeExceptionIncludesFieldName() throws Exception {
+        assertThrows(new Field("my_field", DataType.getWeightedSet(DataType.BYTE)),
+                     "<item>-129</item>",
+                     "Field 'my_field': Invalid byte \"-129\". (at line 1, column 70)");
+    }
+
+    @Test
+    public void requireThatPutsForTensorFieldsAreNotSupported() throws Exception {
+        assertThrows(new Field("my_tensor", DataType.TENSOR), "",
+                "Field 'my_tensor': XML input for fields of type TENSOR is not supported. Please use JSON input instead.");
+    }
+
+    private class MockedReaderFixture {
+        public DocumentTypeManager mgr;
+        public DocumentType docType;
+        public XMLStreamReader xmlReader;
+        public VespaXMLFieldReader fieldReader;
+
+        public MockedReaderFixture() {
+            mgr = new DocumentTypeManager();
+            mgr.register(PositionDataType.INSTANCE);
+            docType = new DocumentType("my_doc");
+            docType.addField("my_pos", PositionDataType.INSTANCE);
+            mgr.registerDocumentType(docType);
+
+            xmlReader = mock(XMLStreamReader.class);
+            fieldReader = new VespaXMLFieldReader(xmlReader, mgr);
+        }
+
+        public void assertReadPositionEquals(int x, int y) {
+            Struct pos = new Struct(PositionDataType.INSTANCE);
+            fieldReader.read(docType.getField("my_pos"), pos);
+
+            assertEquals(new IntegerFieldValue(x), pos.getFieldValue(PositionDataType.FIELD_X));
+            assertEquals(new IntegerFieldValue(y), pos.getFieldValue(PositionDataType.FIELD_Y));
+        }
+    }
+
+    @Test
+    public void requireThatPositionFieldCanBeReadInSingleEvent() throws Exception {
+        MockedReaderFixture fixture = new MockedReaderFixture();
+        XMLStreamReader xmlReader = fixture.xmlReader;
+
+        when(xmlReader.getAttributeCount()).thenReturn(0);
+        when(xmlReader.hasNext()).thenReturn(true, true, false);
+        when(xmlReader.next()).thenReturn(
+                XMLStreamReader.CHARACTERS, XMLStreamReader.END_ELEMENT);
+        when(xmlReader.getText()).thenReturn("E3;N4");
+
+        fixture.assertReadPositionEquals(3000000, 4000000);
+    }
+
+    @Test
+    public void requireThatPositionFieldCanBeReadAcrossMultipleEvents() throws Exception {
+        MockedReaderFixture fixture = new MockedReaderFixture();
+        XMLStreamReader xmlReader = fixture.xmlReader;
+
+        when(xmlReader.getAttributeCount()).thenReturn(0);
+        when(xmlReader.hasNext()).thenReturn(true, true, true, true, false);
+        when(xmlReader.next()).thenReturn(
+                XMLStreamReader.CHARACTERS, XMLStreamReader.CHARACTERS,
+                XMLStreamReader.CHARACTERS, XMLStreamReader.END_ELEMENT);
+        when(xmlReader.getText()).thenReturn("E3;", "N", "4");
+
+        fixture.assertReadPositionEquals(3000000, 4000000);
+    }
+
+    private static void assertThrows(Field field, String fieldXml, String expected) throws Exception {
+        DocumentTypeManager docManager = new DocumentTypeManager();
+        DocumentType docType = new DocumentType("my_type");
+        docType.addField(field);
+        docManager.register(docType);
+
+        String documentXml = "<document id='doc:scheme:' type='my_type'><" + field.getName() + ">" +
+                             fieldXml + "</" + field.getName() + "></document>";
+        InputStream in = new ByteArrayInputStream(documentXml.getBytes(StandardCharsets.UTF_8));
+        Document doc = new Document(docType, "doc:scheme:");
+        try {
+            new VespaXMLFieldReader(in, docManager).read(null, doc);
+            fail();
+        } catch (DeserializationException e) {
+            assertEquals(expected, e.getMessage());
+        }
+    }
+
+    private static void assertReadable(Predicate predicate) throws Exception {
+        assertRead(predicate,
+                   "<document id='doc:scheme:' type='my_type'>" +
+                   "  <my_predicate>" + predicate + "</my_predicate>" +
+                   "</document>");
+        assertRead(predicate,
+                   "<document id='doc:scheme:' type='my_type'>" +
+                   "  <my_predicate binaryencoding='base64'>" +
+                   Base64.encodeBase64String(BinaryFormat.encode(predicate)) +
+                   "  </my_predicate>" +
+                   "</document>");
+    }
+
+    private static void assertRead(Predicate expected, String documentXml) throws Exception {
+        DocumentTypeManager docManager = new DocumentTypeManager();
+        DocumentType docType = new DocumentType("my_type");
+        docType.addField("my_predicate", DataType.PREDICATE);
+        docManager.register(docType);
+
+        InputStream in = new ByteArrayInputStream(documentXml.getBytes(StandardCharsets.UTF_8));
+        Document doc = new Document(docType, "doc:scheme:");
+        new VespaXMLFieldReader(in, docManager).read(null, doc);
+        FieldValue value = doc.getFieldValue("my_predicate");
+        assertTrue(value instanceof PredicateFieldValue);
+        assertEquals(expected, ((PredicateFieldValue)value).getPredicate());
+    }
+}
diff --git a/document/src/test/java/com/yahoo/vespaxmlparser/VespaXmlUpdateReaderTestCase.java b/document/src/test/java/com/yahoo/vespaxmlparser/VespaXmlUpdateReaderTestCase.java
new file mode 100644
index 00000000000..8730265c80d
--- /dev/null
+++ b/document/src/test/java/com/yahoo/vespaxmlparser/VespaXmlUpdateReaderTestCase.java
@@ -0,0 +1,262 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespaxmlparser;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.DocumentTypeManager;
+import com.yahoo.document.DocumentUpdate;
+import com.yahoo.document.Field;
+import com.yahoo.document.StructDataType;
+import com.yahoo.document.serialization.DeserializationException;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.InputStream;
+import java.io.PrintStream;
+import java.nio.charset.StandardCharsets;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a>
+ */
+public class VespaXmlUpdateReaderTestCase {
+
+    @Test
+    @Ignore
+    public void requireThatArithmeticDeserializationValidateValue() throws Exception {
+        // tracked in ticket 6675085
+        // problem caused by VespaXMLUpdateReader#readArithmetic() parsing value as double
+        Field field = new Field("my_field", DataType.BYTE);
+        assertThrows(field,
+                     "<increment field='my_field' by='-129' />",
+                     "Field 'my_field': Invalid byte \"-129\". (at line 1, column X)");
+        assertThrows(field,
+                     "<decrement field='my_field' by='-129' />",
+                     "Field 'my_field': Invalid byte \"-129\". (at line 1, column X)");
+        assertThrows(field,
+                     "<multiply field='my_field' by='-129' />",
+                     "Field 'my_field': Invalid byte \"-129\". (at line 1, column X)");
+        assertThrows(field,
+                     "<divide field='my_field' by='-129' />",
+                     "Field 'my_field': Invalid byte \"-129\". (at line 1, column X)");
+        assertThrows(field,
+                     "<alter field='my_field'><increment by='-129' /></alter>",
+                     "Field 'my_field': Invalid byte \"-129\". (at line 1, column X)");
+    }
+
+    @Test
+    @Ignore
+    public void requireThatAssignNumericFieldPathValidatesFieldValue() throws Exception {
+        // tracked in ticket 6675089
+        // problem caused by VespaXMLUpdateReader#read(AssignFieldPathUpdate)
+        assertThrows(new Field("my_field", DataType.BYTE),
+                     "<assign fieldpath='my_field'>-129</assign>",
+                     "Field 'my_field': Invalid byte \"-129\". (at line 1, column X)");
+    }
+
+    @Test
+    @Ignore
+    public void requireThatFieldPathWhereClauseIsValidated() throws Exception {
+        // tracked in ticket 6675091
+        // problem caused by VespaXMLUpdateReader#read(FieldPathUpdate) not validating where clause
+        assertThrows(new Field("my_field", DataType.getArray(DataType.BYTE)),
+                     "<remove fieldpath='my_field[$x]' where='my_field[$x] == -129' />",
+                     "Field 'my_field': Invalid byte \"-129\". (at line 1, column X)");
+        assertThrows(new Field("my_field", DataType.getMap(DataType.STRING, DataType.BYTE)),
+                     "<remove fieldpath='my_field{$x}' where='my_field{$x} == -129' />",
+                     "Field 'my_field': Invalid byte \"-129\". (at line 1, column 109)");
+    }
+
+    @Test
+    public void requireThatDeserializeExceptionIncludesFieldName() throws Exception {
+        assertThrows(new Field("my_field", DataType.BYTE),
+                     "<assign field='my_field'>-129</assign>",
+                     "Field 'my_field': Invalid byte \"-129\". (at line 1, column 79)");
+    }
+
+    @Test
+    public void requireThatArrayItemDeserializeExceptionIncludesFieldName() throws Exception {
+        Field field = new Field("my_field", DataType.getArray(DataType.BYTE));
+        assertThrows(field,
+                     "<assign field='my_field'><item>-129</item></assign>",
+                     "Field 'my_field': Invalid byte \"-129\". (at line 1, column 83)");
+        assertThrows(field,
+                     "<assign fieldpath='my_field'><item>-129</item></assign>",
+                     "Field 'my_field': Invalid byte \"-129\". (at line 1, column 87)");
+        assertThrows(field,
+                     "<add field='my_field'><item>-129</item></add>",
+                     "Field 'my_field': Invalid byte \"-129\". (at line 1, column 80)");
+        assertThrows(field,
+                     "<add fieldpath='my_field'><item>-129</item></add>",
+                     "Field 'my_field': Invalid byte \"-129\". (at line 1, column 84)");
+        assertThrows(field,
+                     "<remove field='my_field'><item>-129</item></remove>",
+                     "Field 'my_field': Invalid byte \"-129\". (at line 1, column 83)");
+    }
+
+    @Test
+    public void requireThatMapKeyDeserializeExceptionIncludesFieldName() throws Exception {
+        Field field = new Field("my_field", DataType.getMap(DataType.BYTE, DataType.STRING));
+        assertThrows(field,
+                     "<assign field='my_field'><item><key>-129</key><value>foo</value></item></assign>",
+                     "Field 'my_field': Invalid byte \"-129\". (at line 1, column 87)");
+        assertThrows(field,
+                     "<assign fieldpath='my_field'><item><key>-129</key><value>foo</value></item></assign>",
+                     "Field 'my_field': Invalid byte \"-129\". (at line 1, column 91)");
+        assertThrows(field,
+                     "<add field='my_field'><item><key>-129</key><value>foo</value></item></add>",
+                     "Field 'my_field': Invalid byte \"-129\". (at line 1, column 84)");
+        assertThrows(field,
+                     "<add fieldpath='my_field'><item><key>-129</key><value>foo</value></item></add>",
+                     "Field 'my_field': Invalid byte \"-129\". (at line 1, column 88)");
+        assertThrows(field,
+                     "<remove field='my_field'><item><key>-129</key><value>foo</value></item></remove>",
+                     "Field 'my_field': Invalid byte \"-129\". (at line 1, column 87)");
+        try {
+            readUpdate(field, "<remove fieldpath='my_field{-129}' />");
+            fail();
+        } catch (NumberFormatException e) {
+
+        }
+    }
+
+    @Test
+    public void requireThatMapValueDeserializeExceptionIncludesFieldName() throws Exception {
+        Field field = new Field("my_field", DataType.getMap(DataType.STRING, DataType.BYTE));
+        assertThrows(field,
+                     "<assign field='my_field'><item><key>foo</key><value>-129</value></item></assign>",
+                     "Field 'my_field': Invalid byte \"-129\". (at line 1, column 105)");
+        assertThrows(field,
+                     "<assign fieldpath='my_field'><item><key>foo</key><value>-129</value></item></assign>",
+                     "Field 'my_field': Invalid byte \"-129\". (at line 1, column 109)");
+        assertThrows(field,
+                     "<add field='my_field'><item><key>foo</key><value>-129</value></item></add>",
+                     "Field 'my_field': Invalid byte \"-129\". (at line 1, column 102)");
+        assertThrows(field,
+                     "<add fieldpath='my_field'><item><key>foo</key><value>-129</value></item></add>",
+                     "Field 'my_field': Invalid byte \"-129\". (at line 1, column 106)");
+        assertThrows(field,
+                     "<remove field='my_field'><item><key>foo</key><value>-129</value></item></remove>",
+                     "Field 'my_field': Invalid byte \"-129\". (at line 1, column 105)");
+    }
+
+    @Test
+    public void requireThatStructFieldDeserializeExceptionIncludesFieldName() throws Exception {
+        StructDataType structType = new StructDataType("my_struct");
+        structType.addField(new Field("my_byte", DataType.BYTE));
+        Field field = new Field("my_field", structType);
+        assertThrows(field,
+                     "<assign field='my_field'><my_byte>-129</my_byte></assign>",
+                     "Field 'my_byte': Invalid byte \"-129\". (at line 1, column 89)");
+        assertThrows(field,
+                     "<assign fieldpath='my_field'><my_byte>-129</my_byte></assign>",
+                     "Field 'my_byte': Invalid byte \"-129\". (at line 1, column 93)");
+        assertThrows(field,
+                     "<add field='my_field'><my_byte>-129</my_byte></add>",
+                     "Field 'my_byte': Invalid byte \"-129\". (at line 1, column 86)");
+        assertThrows(field,
+                     "<add fieldpath='my_field'><my_byte>-129</my_byte></add>",
+                     "Field 'my_byte': Invalid byte \"-129\". (at line 1, column 90)");
+        assertThrows(field,
+                     "<remove field='my_field'><my_byte>-129</my_byte></remove>",
+                     "Field 'my_byte': Invalid byte \"-129\". (at line 1, column 89)");
+    }
+
+    @Test
+    public void requireThatWSetItemDeserializeExceptionIncludesFieldName() throws Exception {
+        Field field = new Field("my_field", DataType.getWeightedSet(DataType.BYTE));
+        assertThrows(field,
+                     "<assign field='my_field'><item>-129</item></assign>",
+                     "Field 'my_field': Invalid byte \"-129\". (at line 1, column 83)");
+        assertThrows(field,
+                     "<assign fieldpath='my_field'><item>-129</item></assign>",
+                     "Field 'my_field': Invalid byte \"-129\". (at line 1, column 87)");
+        assertThrows(field,
+                     "<add field='my_field'><item>-129</item></add>",
+                     "Field 'my_field': Invalid byte \"-129\". (at line 1, column 80)");
+        assertThrows(field,
+                     "<add fieldpath='my_field'><item>-129</item></add>",
+                     "Field 'my_field': Invalid byte \"-129\". (at line 1, column 84)");
+        assertThrows(field,
+                     "<remove field='my_field'><item>-129</item></remove>",
+                     "Field 'my_field': Invalid byte \"-129\". (at line 1, column 83)");
+        try {
+            readUpdate(field, "<remove fieldpath='my_field{-129}' />");
+            fail();
+        } catch (NumberFormatException e) {
+
+        }
+    }
+
+    @Test
+    public void requireThatCreateIfNonExistentFlagCanBeSpecified() throws Exception {
+        {
+            assertTrue(readUpdate(true).getCreateIfNonExistent());
+            assertFalse(readUpdate(false).getCreateIfNonExistent());
+        }
+    }
+
+    @Test
+    public void requireThatCreateIfNonExistentFlagIsValidated() throws Exception {
+        String documentXml = "<update id='doc:scheme:' type='my_type' create-if-non-existent='illegal'></update>";
+        try {
+            readUpdateHelper(null, documentXml);
+            fail();
+        } catch (DeserializationException e) {
+            assertEquals(printStackTrace(e), "'create-if-non-existent' must be either 'true' or 'false', was 'illegal' (at line 1, column 74)", e.getMessage());
+        }
+    }
+
+    @Test
+    public void requireThatUpdatesForTensorFieldsAreNotSupported() throws Exception {
+        assertThrows(new Field("my_tensor", DataType.TENSOR), "<assign field='my_tensor'></assign>",
+                "Field 'my_tensor': XML input for fields of type TENSOR is not supported. Please use JSON input instead.");
+    }
+
+    private static void assertThrows(Field field, String fieldXml, String expected) throws Exception {
+        try {
+            readUpdate(field, fieldXml);
+            fail();
+        } catch (DeserializationException e) {
+            assertEquals(printStackTrace(e), expected, e.getMessage());
+        }
+    }
+
+    private static DocumentUpdate readUpdate(Field field, String fieldXml) throws Exception {
+        String documentXml = "<update id='doc:scheme:' type='my_type'>" + fieldXml + "</update>";
+        return readUpdateHelper(field, documentXml);
+    }
+
+    private static DocumentUpdate readUpdate(boolean createIfNonExistent) throws Exception {
+        String documentXml = "<update id='doc:scheme:' type='my_type' create-if-non-existent='" + (createIfNonExistent ? "true" : "false") + "'></update>";
+        return readUpdateHelper(null, documentXml);
+    }
+
+    private static DocumentUpdate readUpdateHelper(Field field, String documentXml) throws Exception {
+        DocumentTypeManager docManager = new DocumentTypeManager();
+        DocumentType docType = new DocumentType("my_type");
+        if (field != null) {
+            docType.addField(field);
+        }
+        docManager.register(docType);
+
+        InputStream in = new ByteArrayInputStream(documentXml.getBytes(StandardCharsets.UTF_8));
+        DocumentUpdate doc = new DocumentUpdate(docType, "doc:scheme:");
+        VespaXMLUpdateReader reader = new VespaXMLUpdateReader(in, docManager);
+        reader.reader.next(); // initialize reader
+        reader.read(doc);
+        return doc;
+    }
+
+    private static String printStackTrace(Throwable t) {
+        ByteArrayOutputStream out = new ByteArrayOutputStream();
+        t.printStackTrace(new PrintStream(out));
+        return new String(out.toByteArray(), StandardCharsets.UTF_8);
+    }
+}
diff --git a/document/src/test/java/com/yahoo/vespaxmlparser/XMLNumericFieldErrorMsgTestCase.java b/document/src/test/java/com/yahoo/vespaxmlparser/XMLNumericFieldErrorMsgTestCase.java
new file mode 100644
index 00000000000..1f74cd650ae
--- /dev/null
+++ b/document/src/test/java/com/yahoo/vespaxmlparser/XMLNumericFieldErrorMsgTestCase.java
@@ -0,0 +1,114 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespaxmlparser;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.Document;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.DocumentTypeManager;
+import com.yahoo.document.serialization.DeserializationException;
+import org.junit.Test;
+
+import java.io.ByteArrayInputStream;
+import java.nio.charset.StandardCharsets;
+
+import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.fail;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ * @since 5.1.29
+ */
+public class XMLNumericFieldErrorMsgTestCase {
+
+    private static DocumentTypeManager setupTypes() {
+        DocumentTypeManager dtm = new DocumentTypeManager();
+        DocumentType docType = new DocumentType("doctype");
+        docType.addField("bytefield", DataType.BYTE);
+        docType.addField("intfield", DataType.INT);
+        docType.addField("longfield", DataType.LONG);
+        docType.addField("floatfield", DataType.FLOAT);
+        docType.addField("doublefield", DataType.DOUBLE);
+        dtm.register(docType);
+        return dtm;
+    }
+
+    @Test
+    public void requireDescriptiveErrorMsgForFloats() throws Exception {
+        DocumentTypeManager dtm = setupTypes();
+        try {
+            VespaXMLDocumentReader documentReader = new VespaXMLDocumentReader(
+                    new ByteArrayInputStream(("<document id=\"doc:foo:bar\" type=\"doctype\">" +
+                                              "  <floatfield></floatfield>" +
+                                              "</document>").getBytes(StandardCharsets.UTF_8)), dtm);
+            new Document(documentReader);
+            fail("Sorry mac");
+        } catch (DeserializationException e) {
+            assertThat(e.getMessage(), e.getMessage().contains("Field 'floatfield': Invalid float \"\""), is(true));
+        }
+    }
+
+    @Test
+    public void requireDescriptiveErrorMsgForDoubles() throws Exception {
+        DocumentTypeManager dtm = setupTypes();
+        try {
+            VespaXMLDocumentReader documentReader = new VespaXMLDocumentReader(
+                    new ByteArrayInputStream(("<document id=\"doc:foo:bar\" type=\"doctype\">" +
+                                              "  <doublefield></doublefield>" +
+                                              "</document>").getBytes(StandardCharsets.UTF_8)), dtm);
+            new Document(documentReader);
+            fail("Sorry mac");
+        } catch (DeserializationException e) {
+            assertThat(e.getMessage(), e.getMessage().contains("Field 'doublefield': Invalid double \"\""), is(true));
+        }
+    }
+
+    @Test
+    public void requireDescriptiveErrorMsgForLongs() throws Exception {
+        DocumentTypeManager dtm = setupTypes();
+        try {
+            VespaXMLDocumentReader documentReader = new VespaXMLDocumentReader(
+                    new ByteArrayInputStream(("<document id=\"doc:foo:bar\" type=\"doctype\">" +
+                                              "  <longfield></longfield>" +
+                                              "</document>").getBytes(StandardCharsets.UTF_8)), dtm);
+            new Document(documentReader);
+            fail("Sorry mac");
+        } catch (DeserializationException e) {
+            assertThat(e.getMessage(), e.getMessage().contains("Field 'longfield': Invalid long \"\""), is(true));
+        }
+    }
+
+    @Test
+    public void requireDescriptiveErrorMsgForIntegers() throws Exception {
+        DocumentTypeManager dtm = setupTypes();
+        try {
+            VespaXMLDocumentReader documentReader = new VespaXMLDocumentReader(
+                    new ByteArrayInputStream(("<document id=\"doc:foo:bar\" type=\"doctype\">" +
+                                              "  <intfield></intfield>" +
+                                              "</document>").getBytes(StandardCharsets.UTF_8)), dtm);
+            new Document(documentReader);
+            fail("Sorry mac");
+        } catch (DeserializationException e) {
+            assertThat(e.getMessage(), e.getMessage().contains("Field 'intfield': Invalid integer \"\""), is(true));
+        }
+    }
+
+    @Test
+    public void requireDescriptiveErrorMsgForBytes() throws Exception {
+        DocumentTypeManager dtm = setupTypes();
+        try {
+            VespaXMLDocumentReader documentReader = new VespaXMLDocumentReader(
+                    new ByteArrayInputStream(("<document id=\"doc:foo:bar\" type=\"doctype\">" +
+                                              "  <bytefield></bytefield>" +
+                                              "</document>").getBytes(StandardCharsets.UTF_8)), dtm);
+            new Document(documentReader);
+            fail("Sorry mac");
+        } catch (DeserializationException e) {
+            assertThat(e.getMessage(), e.getMessage().contains("Field 'bytefield': Invalid byte \"\""), is(true));
+        }
+    }
+
+
+
+}
+
diff --git a/document/src/test/resources/predicates/false__cpp b/document/src/test/resources/predicates/false__cpp
new file mode 100644
index 00000000000..00a71d5fe73
--- /dev/null
+++ b/document/src/test/resources/predicates/false__cpp
diff --git a/document/src/test/resources/predicates/false__java b/document/src/test/resources/predicates/false__java
new file mode 100644
index 00000000000..00a71d5fe73
--- /dev/null
+++ b/document/src/test/resources/predicates/false__java
diff --git a/document/src/test/resources/predicates/foo_in_6_9__cpp b/document/src/test/resources/predicates/foo_in_6_9__cpp
new file mode 100644
index 00000000000..f9f7249e65b
--- /dev/null
+++ b/document/src/test/resources/predicates/foo_in_6_9__cpp
diff --git a/document/src/test/resources/predicates/foo_in_6_9__java b/document/src/test/resources/predicates/foo_in_6_9__java
new file mode 100644
index 00000000000..d18b937bead
--- /dev/null
+++ b/document/src/test/resources/predicates/foo_in_6_9__java
diff --git a/document/src/test/resources/predicates/foo_in_6_x__cpp b/document/src/test/resources/predicates/foo_in_6_x__cpp
new file mode 100644
index 00000000000..0ccf8e9794b
--- /dev/null
+++ b/document/src/test/resources/predicates/foo_in_6_x__cpp
diff --git a/document/src/test/resources/predicates/foo_in_6_x__java b/document/src/test/resources/predicates/foo_in_6_x__java
new file mode 100644
index 00000000000..b8ae334882e
--- /dev/null
+++ b/document/src/test/resources/predicates/foo_in_6_x__java
diff --git a/document/src/test/resources/predicates/foo_in_bar__cpp b/document/src/test/resources/predicates/foo_in_bar__cpp
new file mode 100644
index 00000000000..a2761427c6a
--- /dev/null
+++ b/document/src/test/resources/predicates/foo_in_bar__cpp
diff --git a/document/src/test/resources/predicates/foo_in_bar__java b/document/src/test/resources/predicates/foo_in_bar__java
new file mode 100644
index 00000000000..aeb10e2b5d7
--- /dev/null
+++ b/document/src/test/resources/predicates/foo_in_bar__java
diff --git a/document/src/test/resources/predicates/foo_in_bar_and_baz_in_cox__cpp b/document/src/test/resources/predicates/foo_in_bar_and_baz_in_cox__cpp
new file mode 100644
index 00000000000..8a2705e6f62
--- /dev/null
+++ b/document/src/test/resources/predicates/foo_in_bar_and_baz_in_cox__cpp
diff --git a/document/src/test/resources/predicates/foo_in_bar_and_baz_in_cox__java b/document/src/test/resources/predicates/foo_in_bar_and_baz_in_cox__java
new file mode 100644
index 00000000000..8a2705e6f62
--- /dev/null
+++ b/document/src/test/resources/predicates/foo_in_bar_and_baz_in_cox__java
diff --git a/document/src/test/resources/predicates/foo_in_bar_baz__cpp b/document/src/test/resources/predicates/foo_in_bar_baz__cpp
new file mode 100644
index 00000000000..b4cf08bbf57
--- /dev/null
+++ b/document/src/test/resources/predicates/foo_in_bar_baz__cpp
diff --git a/document/src/test/resources/predicates/foo_in_bar_baz__java b/document/src/test/resources/predicates/foo_in_bar_baz__java
new file mode 100644
index 00000000000..ea3314d9bd7
--- /dev/null
+++ b/document/src/test/resources/predicates/foo_in_bar_baz__java
diff --git a/document/src/test/resources/predicates/foo_in_bar_or_baz_in_cox__cpp b/document/src/test/resources/predicates/foo_in_bar_or_baz_in_cox__cpp
new file mode 100644
index 00000000000..4d5474c24e4
--- /dev/null
+++ b/document/src/test/resources/predicates/foo_in_bar_or_baz_in_cox__cpp
diff --git a/document/src/test/resources/predicates/foo_in_bar_or_baz_in_cox__java b/document/src/test/resources/predicates/foo_in_bar_or_baz_in_cox__java
new file mode 100644
index 00000000000..4d5474c24e4
--- /dev/null
+++ b/document/src/test/resources/predicates/foo_in_bar_or_baz_in_cox__java
diff --git a/document/src/test/resources/predicates/foo_in_x_9__cpp b/document/src/test/resources/predicates/foo_in_x_9__cpp
new file mode 100644
index 00000000000..29218b9e944
--- /dev/null
+++ b/document/src/test/resources/predicates/foo_in_x_9__cpp
diff --git a/document/src/test/resources/predicates/foo_in_x_9__java b/document/src/test/resources/predicates/foo_in_x_9__java
new file mode 100644
index 00000000000..017a610a7d5
--- /dev/null
+++ b/document/src/test/resources/predicates/foo_in_x_9__java
diff --git a/document/src/test/resources/predicates/foo_in_x__cpp b/document/src/test/resources/predicates/foo_in_x__cpp
new file mode 100644
index 00000000000..bd2916835c6
--- /dev/null
+++ b/document/src/test/resources/predicates/foo_in_x__cpp
diff --git a/document/src/test/resources/predicates/foo_in_x__java b/document/src/test/resources/predicates/foo_in_x__java
new file mode 100644
index 00000000000..6537cc6bdeb
--- /dev/null
+++ b/document/src/test/resources/predicates/foo_in_x__java
diff --git a/document/src/test/resources/predicates/foo_in_x_x__cpp b/document/src/test/resources/predicates/foo_in_x_x__cpp
new file mode 100644
index 00000000000..e0d8282c46d
--- /dev/null
+++ b/document/src/test/resources/predicates/foo_in_x_x__cpp
diff --git a/document/src/test/resources/predicates/foo_in_x_x__java b/document/src/test/resources/predicates/foo_in_x_x__java
new file mode 100644
index 00000000000..5060718417a
--- /dev/null
+++ b/document/src/test/resources/predicates/foo_in_x_x__java
diff --git a/document/src/test/resources/predicates/not_foo_in_bar__cpp b/document/src/test/resources/predicates/not_foo_in_bar__cpp
new file mode 100644
index 00000000000..b654de6d53e
--- /dev/null
+++ b/document/src/test/resources/predicates/not_foo_in_bar__cpp
diff --git a/document/src/test/resources/predicates/not_foo_in_bar__java b/document/src/test/resources/predicates/not_foo_in_bar__java
new file mode 100644
index 00000000000..b654de6d53e
--- /dev/null
+++ b/document/src/test/resources/predicates/not_foo_in_bar__java
diff --git a/document/src/test/resources/predicates/true__cpp b/document/src/test/resources/predicates/true__cpp
new file mode 100644
index 00000000000..2b5da7409d5
--- /dev/null
+++ b/document/src/test/resources/predicates/true__cpp
diff --git a/document/src/test/resources/predicates/true__java b/document/src/test/resources/predicates/true__java
new file mode 100644
index 00000000000..2b5da7409d5
--- /dev/null
+++ b/document/src/test/resources/predicates/true__java
diff --git a/document/src/test/resources/tensor/empty_tensor__cpp b/document/src/test/resources/tensor/empty_tensor__cpp
new file mode 100644
index 00000000000..365182b14eb
--- /dev/null
+++ b/document/src/test/resources/tensor/empty_tensor__cpp
diff --git a/document/src/test/resources/tensor/empty_tensor__java b/document/src/test/resources/tensor/empty_tensor__java
new file mode 100644
index 00000000000..365182b14eb
--- /dev/null
+++ b/document/src/test/resources/tensor/empty_tensor__java
diff --git a/document/src/test/resources/tensor/multi_cell_tensor__cpp b/document/src/test/resources/tensor/multi_cell_tensor__cpp
new file mode 100644
index 00000000000..090e1c57959
--- /dev/null
+++ b/document/src/test/resources/tensor/multi_cell_tensor__cpp
diff --git a/document/src/test/resources/tensor/multi_cell_tensor__java b/document/src/test/resources/tensor/multi_cell_tensor__java
new file mode 100644
index 00000000000..a202c1a09ab
--- /dev/null
+++ b/document/src/test/resources/tensor/multi_cell_tensor__java
diff --git a/document/src/test/resources/tensor/non_existing_tensor__cpp b/document/src/test/resources/tensor/non_existing_tensor__cpp
new file mode 100644
index 00000000000..08cbcac6dd3
--- /dev/null
+++ b/document/src/test/resources/tensor/non_existing_tensor__cpp
diff --git a/document/src/test/resources/tensor/non_existing_tensor__java b/document/src/test/resources/tensor/non_existing_tensor__java
new file mode 100644
index 00000000000..08cbcac6dd3
--- /dev/null
+++ b/document/src/test/resources/tensor/non_existing_tensor__java
diff --git a/document/src/test/scala/com/yahoo/document/annotation/.gitignore b/document/src/test/scala/com/yahoo/document/annotation/.gitignore
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/document/src/test/scala/com/yahoo/document/annotation/.gitignore
diff --git a/document/src/test/serializeddocuments/.gitignore b/document/src/test/serializeddocuments/.gitignore
new file mode 100644
index 00000000000..7ef24eb7f2c
--- /dev/null
+++ b/document/src/test/serializeddocuments/.gitignore
@@ -0,0 +1,2 @@
+document-java-currentversion-lz4-9.dat
+document-java-currentversion-uncompressed.dat
diff --git a/document/src/test/serializeddocuments/document-java-v8-uncompressed.dat b/document/src/test/serializeddocuments/document-java-v8-uncompressed.dat
new file mode 100644
index 00000000000..8f3ea65be2a
--- /dev/null
+++ b/document/src/test/serializeddocuments/document-java-v8-uncompressed.dat
diff --git a/document/src/test/serializeddocuments/java-6/README b/document/src/test/serializeddocuments/java-6/README
new file mode 100644
index 00000000000..6dc8465bcbe
--- /dev/null
+++ b/document/src/test/serializeddocuments/java-6/README
@@ -0,0 +1 @@
+A document serialized from Java, at serialized version 6.
diff --git a/document/src/test/serializeddocuments/java-6/java-6.cfg b/document/src/test/serializeddocuments/java-6/java-6.cfg
new file mode 100644
index 00000000000..eb11f9c9c04
--- /dev/null
+++ b/document/src/test/serializeddocuments/java-6/java-6.cfg
@@ -0,0 +1,124 @@
+datatype[9]
+datatype[0].id -260050933
+datatype[0].arraytype[0]
+datatype[0].weightedsettype[0]
+datatype[0].structtype[1]
+datatype[0].structtype[0].name serializetest.header
+datatype[0].structtype[0].version 0
+datatype[0].structtype[0].field[5]
+datatype[0].structtype[0].field[0].name floatfield
+datatype[0].structtype[0].field[0].id[0]
+datatype[0].structtype[0].field[0].datatype 1
+datatype[0].structtype[0].field[1].name stringfield
+datatype[0].structtype[0].field[1].id[0]
+datatype[0].structtype[0].field[1].datatype 2
+datatype[0].structtype[0].field[2].name longfield
+datatype[0].structtype[0].field[2].id[0]
+datatype[0].structtype[0].field[2].datatype 4
+datatype[0].structtype[0].field[3].name timestampfield
+datatype[0].structtype[0].field[3].id[0]
+datatype[0].structtype[0].field[3].datatype 9
+datatype[0].structtype[0].field[4].name urifield
+datatype[0].structtype[0].field[4].id[0]
+datatype[0].structtype[0].field[4].datatype 10
+datatype[0].documenttype[0]
+datatype[1].id 1001
+datatype[1].arraytype[1]
+datatype[1].arraytype[0].datatype 1
+datatype[1].weightedsettype[0]
+datatype[1].structtype[0]
+datatype[1].documenttype[0]
+datatype[2].id 2001
+datatype[2].arraytype[1]
+datatype[2].arraytype[0].datatype 1001
+datatype[2].weightedsettype[0]
+datatype[2].structtype[0]
+datatype[2].documenttype[0]
+datatype[3].id 437829
+datatype[3].arraytype[0]
+datatype[3].weightedsettype[1]
+datatype[3].weightedsettype[0].datatype 2
+datatype[3].weightedsettype[0].createifnonexistant false
+datatype[3].weightedsettype[0].removeifzero false
+datatype[3].structtype[0]
+datatype[3].documenttype[0]
+datatype[4].id 1026122976
+datatype[4].arraytype[0]
+datatype[4].weightedsettype[0]
+datatype[4].structtype[1]
+datatype[4].structtype[0].name serializetest.body
+datatype[4].structtype[0].version 0
+datatype[4].structtype[0].field[11]
+datatype[4].structtype[0].field[0].name intfield
+datatype[4].structtype[0].field[0].id[0]
+datatype[4].structtype[0].field[0].datatype 0
+datatype[4].structtype[0].field[1].name rawfield
+datatype[4].structtype[0].field[1].id[0]
+datatype[4].structtype[0].field[1].datatype 3
+datatype[4].structtype[0].field[2].name doublefield
+datatype[4].structtype[0].field[2].id[0]
+datatype[4].structtype[0].field[2].datatype 5
+datatype[4].structtype[0].field[3].name exactstringfield
+datatype[4].structtype[0].field[3].id[0]
+datatype[4].structtype[0].field[3].datatype 11
+datatype[4].structtype[0].field[4].name contentfield
+datatype[4].structtype[0].field[4].id[0]
+datatype[4].structtype[0].field[4].datatype 12
+datatype[4].structtype[0].field[5].name termboostfield
+datatype[4].structtype[0].field[5].id[0]
+datatype[4].structtype[0].field[5].datatype 15
+datatype[4].structtype[0].field[6].name bytefield
+datatype[4].structtype[0].field[6].id[0]
+datatype[4].structtype[0].field[6].datatype 16
+datatype[4].structtype[0].field[7].name arrayoffloatfield
+datatype[4].structtype[0].field[7].id[0]
+datatype[4].structtype[0].field[7].datatype 1001
+datatype[4].structtype[0].field[8].name arrayofarrayoffloatfield
+datatype[4].structtype[0].field[8].id[0]
+datatype[4].structtype[0].field[8].datatype 2001
+datatype[4].structtype[0].field[9].name docfield
+datatype[4].structtype[0].field[9].id[0]
+datatype[4].structtype[0].field[9].datatype 8
+datatype[4].structtype[0].field[10].name wsfield
+datatype[4].structtype[0].field[10].id[0]
+datatype[4].structtype[0].field[10].datatype 437829
+datatype[4].documenttype[0]
+datatype[5].id 1306012852
+datatype[5].arraytype[0]
+datatype[5].weightedsettype[0]
+datatype[5].structtype[0]
+datatype[5].documenttype[1]
+datatype[5].documenttype[0].name serializetest
+datatype[5].documenttype[0].version 0
+datatype[5].documenttype[0].inherits[0]
+datatype[5].documenttype[0].headerstruct -260050933
+datatype[5].documenttype[0].bodystruct 1026122976
+datatype[6].id -1686125086
+datatype[6].arraytype[0]
+datatype[6].weightedsettype[0]
+datatype[6].structtype[1]
+datatype[6].structtype[0].name docindoc.header
+datatype[6].structtype[0].version 0
+datatype[6].structtype[0].field[0]
+datatype[6].documenttype[0]
+datatype[7].id 2030224503
+datatype[7].arraytype[0]
+datatype[7].weightedsettype[0]
+datatype[7].structtype[1]
+datatype[7].structtype[0].name docindoc.body
+datatype[7].structtype[0].version 0
+datatype[7].structtype[0].field[1]
+datatype[7].structtype[0].field[0].name stringindocfield
+datatype[7].structtype[0].field[0].id[0]
+datatype[7].structtype[0].field[0].datatype 2
+datatype[7].documenttype[0]
+datatype[8].id 1447635645
+datatype[8].arraytype[0]
+datatype[8].weightedsettype[0]
+datatype[8].structtype[0]
+datatype[8].documenttype[1]
+datatype[8].documenttype[0].name docindoc
+datatype[8].documenttype[0].version 0
+datatype[8].documenttype[0].inherits[0]
+datatype[8].documenttype[0].headerstruct -1686125086
+datatype[8].documenttype[0].bodystruct 2030224503
diff --git a/document/src/test/serializeddocuments/java-6/java-6.dat b/document/src/test/serializeddocuments/java-6/java-6.dat
new file mode 100644
index 00000000000..723754892a2
--- /dev/null
+++ b/document/src/test/serializeddocuments/java-6/java-6.dat
diff --git a/document/src/test/vespaxmlparser/alltypes.cfg b/document/src/test/vespaxmlparser/alltypes.cfg
new file mode 100644
index 00000000000..5d89611d24b
--- /dev/null
+++ b/document/src/test/vespaxmlparser/alltypes.cfg
@@ -0,0 +1,101 @@
+datatype[5]
+datatype[0].id -240642363
+datatype[0].arraytype[0]
+datatype[0].weightedsettype[0]
+datatype[0].structtype[1]
+datatype[0].structtype[0].name alltypes.header
+datatype[0].structtype[0].version 0
+datatype[0].structtype[0].field[0]
+datatype[0].documenttype[0]
+datatype[1].id 1000002
+datatype[1].arraytype[1]
+datatype[1].arraytype[0].datatype 2
+datatype[1].weightedsettype[0]
+datatype[1].structtype[0]
+datatype[1].documenttype[0]
+datatype[2].id 2000001
+datatype[2].arraytype[0]
+datatype[2].weightedsettype[1]
+datatype[2].weightedsettype[0].datatype 2
+datatype[2].weightedsettype[0].createifnonexistant false
+datatype[2].weightedsettype[0].removeifzero false
+datatype[2].structtype[0]
+datatype[2].documenttype[0]
+datatype[3].id 163574298
+datatype[3].arraytype[0]
+datatype[3].weightedsettype[0]
+datatype[3].structtype[1]
+datatype[3].structtype[0].name alltypes.body
+datatype[3].structtype[0].version 0
+datatype[3].structtype[0].field[20]
+datatype[3].structtype[0].field[0].name stringval
+datatype[3].structtype[0].field[0].id[0]
+datatype[3].structtype[0].field[0].datatype 2
+datatype[3].structtype[0].field[1].name intval1
+datatype[3].structtype[0].field[1].id[0]
+datatype[3].structtype[0].field[1].datatype 0
+datatype[3].structtype[0].field[2].name intval2
+datatype[3].structtype[0].field[2].id[0]
+datatype[3].structtype[0].field[2].datatype 0
+datatype[3].structtype[0].field[3].name intval3
+datatype[3].structtype[0].field[3].id[0]
+datatype[3].structtype[0].field[3].datatype 0
+datatype[3].structtype[0].field[4].name longval1
+datatype[3].structtype[0].field[4].id[0]
+datatype[3].structtype[0].field[4].datatype 4
+datatype[3].structtype[0].field[5].name longval2
+datatype[3].structtype[0].field[5].id[0]
+datatype[3].structtype[0].field[5].datatype 4
+datatype[3].structtype[0].field[6].name longval3
+datatype[3].structtype[0].field[6].id[0]
+datatype[3].structtype[0].field[6].datatype 4
+datatype[3].structtype[0].field[7].name byteval1
+datatype[3].structtype[0].field[7].id[0]
+datatype[3].structtype[0].field[7].datatype 16
+datatype[3].structtype[0].field[8].name byteval2
+datatype[3].structtype[0].field[8].id[0]
+datatype[3].structtype[0].field[8].datatype 16
+datatype[3].structtype[0].field[9].name byteval3
+datatype[3].structtype[0].field[9].id[0]
+datatype[3].structtype[0].field[9].datatype 16
+datatype[3].structtype[0].field[10].name floatval
+datatype[3].structtype[0].field[10].id[0]
+datatype[3].structtype[0].field[10].datatype 1
+datatype[3].structtype[0].field[11].name doubleval
+datatype[3].structtype[0].field[11].id[0]
+datatype[3].structtype[0].field[11].datatype 5
+datatype[3].structtype[0].field[12].name rawval1
+datatype[3].structtype[0].field[12].id[0]
+datatype[3].structtype[0].field[12].datatype 3
+datatype[3].structtype[0].field[13].name rawval2
+datatype[3].structtype[0].field[13].id[0]
+datatype[3].structtype[0].field[13].datatype 3
+datatype[3].structtype[0].field[14].name urival
+datatype[3].structtype[0].field[14].id[0]
+datatype[3].structtype[0].field[14].datatype 10
+datatype[3].structtype[0].field[15].name contentval1
+datatype[3].structtype[0].field[15].id[0]
+datatype[3].structtype[0].field[15].datatype 12
+datatype[3].structtype[0].field[16].name contentval2
+datatype[3].structtype[0].field[16].id[0]
+datatype[3].structtype[0].field[16].datatype 12
+datatype[3].structtype[0].field[17].name arrayofstringval
+datatype[3].structtype[0].field[17].id[0]
+datatype[3].structtype[0].field[17].datatype 1000002
+datatype[3].structtype[0].field[18].name weightedsetofstringval
+datatype[3].structtype[0].field[18].id[0]
+datatype[3].structtype[0].field[18].datatype 2000001
+datatype[3].structtype[0].field[19].name tagval
+datatype[3].structtype[0].field[19].id[0]
+datatype[3].structtype[0].field[19].datatype 18
+datatype[3].documenttype[0]
+datatype[4].id -1126644934
+datatype[4].arraytype[0]
+datatype[4].weightedsettype[0]
+datatype[4].structtype[0]
+datatype[4].documenttype[1]
+datatype[4].documenttype[0].name alltypes
+datatype[4].documenttype[0].version 0
+datatype[4].documenttype[0].inherits[0]
+datatype[4].documenttype[0].headerstruct -240642363
+datatype[4].documenttype[0].bodystruct 163574298
diff --git a/document/src/test/vespaxmlparser/documentmanager.cfg b/document/src/test/vespaxmlparser/documentmanager.cfg
new file mode 100644
index 00000000000..6662f5caab5
--- /dev/null
+++ b/document/src/test/vespaxmlparser/documentmanager.cfg
@@ -0,0 +1,109 @@
+datatype[10]
+datatype[0].id 1002
+datatype[0].arraytype[1]
+datatype[0].arraytype[0].datatype 2
+datatype[0].weightedsettype[0]
+datatype[0].structtype[0]
+datatype[0].documenttype[0]
+datatype[1].id 1000
+datatype[1].arraytype[1]
+datatype[1].arraytype[0].datatype 0
+datatype[1].weightedsettype[0]
+datatype[1].structtype[0]
+datatype[1].documenttype[0]
+datatype[2].id 1004
+datatype[2].arraytype[1]
+datatype[2].arraytype[0].datatype 4
+datatype[2].weightedsettype[0]
+datatype[2].structtype[0]
+datatype[2].documenttype[0]
+datatype[3].id 1016
+datatype[3].arraytype[1]
+datatype[3].arraytype[0].datatype 16
+datatype[3].weightedsettype[0]
+datatype[3].structtype[0]
+datatype[3].documenttype[0]
+datatype[4].id 1001
+datatype[4].arraytype[1]
+datatype[4].arraytype[0].datatype 1
+datatype[4].weightedsettype[0]
+datatype[4].structtype[0]
+datatype[4].documenttype[0]
+datatype[5].id 2001
+datatype[5].arraytype[0]
+datatype[5].weightedsettype[1]
+datatype[5].weightedsettype[0].datatype 0
+datatype[5].weightedsettype[0].createifnonexistant false
+datatype[5].weightedsettype[0].removeifzero false
+datatype[5].structtype[0]
+datatype[5].documenttype[0]
+datatype[6].id 2002
+datatype[6].arraytype[0]
+datatype[6].weightedsettype[1]
+datatype[6].weightedsettype[0].datatype 2
+datatype[6].weightedsettype[0].createifnonexistant false
+datatype[6].weightedsettype[0].removeifzero false
+datatype[6].structtype[0]
+datatype[6].documenttype[0]
+datatype[7].id -628990518
+datatype[7].arraytype[0]
+datatype[7].weightedsettype[0]
+datatype[7].structtype[1]
+datatype[7].structtype[0].name news.header
+datatype[7].structtype[0].version 0
+datatype[7].structtype[0].field[12]
+datatype[7].structtype[0].field[0].name url
+datatype[7].structtype[0].field[0].id[0]
+datatype[7].structtype[0].field[0].datatype 10
+datatype[7].structtype[0].field[1].name title
+datatype[7].structtype[0].field[1].id[0]
+datatype[7].structtype[0].field[1].datatype 2
+datatype[7].structtype[0].field[2].name last_downloaded
+datatype[7].structtype[0].field[2].id[0]
+datatype[7].structtype[0].field[2].datatype 0
+datatype[7].structtype[0].field[3].name value_long
+datatype[7].structtype[0].field[3].id[0]
+datatype[7].structtype[0].field[3].datatype 4
+datatype[7].structtype[0].field[4].name value_content
+datatype[7].structtype[0].field[4].id[0]
+datatype[7].structtype[0].field[4].datatype 12
+datatype[7].structtype[0].field[5].name stringarr
+datatype[7].structtype[0].field[5].id[0]
+datatype[7].structtype[0].field[5].datatype 1002
+datatype[7].structtype[0].field[6].name intarr
+datatype[7].structtype[0].field[6].id[0]
+datatype[7].structtype[0].field[6].datatype 1000
+datatype[7].structtype[0].field[7].name longarr
+datatype[7].structtype[0].field[7].id[0]
+datatype[7].structtype[0].field[7].datatype 1004
+datatype[7].structtype[0].field[8].name bytearr
+datatype[7].structtype[0].field[8].id[0]
+datatype[7].structtype[0].field[8].datatype 1016
+datatype[7].structtype[0].field[9].name floatarr
+datatype[7].structtype[0].field[9].id[0]
+datatype[7].structtype[0].field[9].datatype 1001
+datatype[7].structtype[0].field[10].name weightedsetint
+datatype[7].structtype[0].field[10].id[0]
+datatype[7].structtype[0].field[10].datatype 2001
+datatype[7].structtype[0].field[11].name weightedsetstring
+datatype[7].structtype[0].field[11].id[0]
+datatype[7].structtype[0].field[11].datatype 2002
+datatype[7].documenttype[0]
+datatype[8].id 538588767
+datatype[8].arraytype[0]
+datatype[8].weightedsettype[0]
+datatype[8].structtype[1]
+datatype[8].structtype[0].name news.body
+datatype[8].structtype[0].version 0
+datatype[8].structtype[0].field[0]
+datatype[8].documenttype[0]
+datatype[9].id -1048827947
+datatype[9].arraytype[0]
+datatype[9].weightedsettype[0]
+datatype[9].structtype[0]
+datatype[9].documenttype[1]
+datatype[9].documenttype[0].name news
+datatype[9].documenttype[0].version 0
+datatype[9].documenttype[0].inherits[0]
+datatype[9].documenttype[0].headerstruct -628990518
+datatype[9].documenttype[0].bodystruct 538588767
diff --git a/document/src/test/vespaxmlparser/documentmanager2.cfg b/document/src/test/vespaxmlparser/documentmanager2.cfg
new file mode 100644
index 00000000000..578fa740a6c
--- /dev/null
+++ b/document/src/test/vespaxmlparser/documentmanager2.cfg
@@ -0,0 +1,220 @@
+enablecompression false
+datatype[16]
+datatype[0].id -1486737430
+datatype[0].arraytype[1]
+datatype[0].arraytype[0].datatype 2
+datatype[0].maptype[0]
+datatype[0].weightedsettype[0]
+datatype[0].structtype[0]
+datatype[0].annotationreftype[0]
+datatype[0].documenttype[0]
+datatype[1].id -1245117006
+datatype[1].arraytype[1]
+datatype[1].arraytype[0].datatype 0
+datatype[1].maptype[0]
+datatype[1].weightedsettype[0]
+datatype[1].structtype[0]
+datatype[1].annotationreftype[0]
+datatype[1].documenttype[0]
+datatype[2].id 58874399
+datatype[2].arraytype[1]
+datatype[2].arraytype[0].datatype 4
+datatype[2].maptype[0]
+datatype[2].weightedsettype[0]
+datatype[2].structtype[0]
+datatype[2].annotationreftype[0]
+datatype[2].documenttype[0]
+datatype[3].id 49942803
+datatype[3].arraytype[1]
+datatype[3].arraytype[0].datatype 16
+datatype[3].maptype[0]
+datatype[3].weightedsettype[0]
+datatype[3].structtype[0]
+datatype[3].annotationreftype[0]
+datatype[3].documenttype[0]
+datatype[4].id 1650586661
+datatype[4].arraytype[1]
+datatype[4].arraytype[0].datatype 1
+datatype[4].maptype[0]
+datatype[4].weightedsettype[0]
+datatype[4].structtype[0]
+datatype[4].annotationreftype[0]
+datatype[4].documenttype[0]
+datatype[5].id 519906144
+datatype[5].arraytype[0]
+datatype[5].maptype[0]
+datatype[5].weightedsettype[1]
+datatype[5].weightedsettype[0].datatype 0
+datatype[5].weightedsettype[0].createifnonexistant false
+datatype[5].weightedsettype[0].removeifzero false
+datatype[5].structtype[0]
+datatype[5].annotationreftype[0]
+datatype[5].documenttype[0]
+datatype[6].id 1328286588
+datatype[6].arraytype[0]
+datatype[6].maptype[0]
+datatype[6].weightedsettype[1]
+datatype[6].weightedsettype[0].datatype 2
+datatype[6].weightedsettype[0].createifnonexistant false
+datatype[6].weightedsettype[0].removeifzero false
+datatype[6].structtype[0]
+datatype[6].annotationreftype[0]
+datatype[6].documenttype[0]
+datatype[7].id 339965458
+datatype[7].arraytype[0]
+datatype[7].maptype[1]
+datatype[7].maptype[0].keytype 2
+datatype[7].maptype[0].valtype 2
+datatype[7].weightedsettype[0]
+datatype[7].structtype[0]
+datatype[7].annotationreftype[0]
+datatype[7].documenttype[0]
+datatype[8].id -2092985853
+datatype[8].arraytype[0]
+datatype[8].maptype[0]
+datatype[8].weightedsettype[0]
+datatype[8].structtype[1]
+datatype[8].structtype[0].name "mystruct"
+datatype[8].structtype[0].version 0
+datatype[8].structtype[0].field[4]
+datatype[8].structtype[0].field[0].name "bytearr"
+datatype[8].structtype[0].field[0].datatype 49942803
+datatype[8].structtype[0].field[0].id[0]
+datatype[8].structtype[0].field[1].name "mymap"
+datatype[8].structtype[0].field[1].datatype 339965458
+datatype[8].structtype[0].field[1].id[0]
+datatype[8].structtype[0].field[2].name "title"
+datatype[8].structtype[0].field[2].datatype 2
+datatype[8].structtype[0].field[2].id[0]
+datatype[8].structtype[0].field[3].name "structfield"
+datatype[8].structtype[0].field[3].datatype 2
+datatype[8].structtype[0].field[3].id[0]
+datatype[8].structtype[0].inherits[0]
+datatype[8].annotationreftype[0]
+datatype[8].documenttype[0]
+datatype[9].id 1901258752
+datatype[9].arraytype[0]
+datatype[9].maptype[1]
+datatype[9].maptype[0].keytype 0
+datatype[9].maptype[0].valtype -2092985853
+datatype[9].weightedsettype[0]
+datatype[9].structtype[0]
+datatype[9].annotationreftype[0]
+datatype[9].documenttype[0]
+datatype[10].id 759956026
+datatype[10].arraytype[1]
+datatype[10].arraytype[0].datatype -2092985853
+datatype[10].maptype[0]
+datatype[10].weightedsettype[0]
+datatype[10].structtype[0]
+datatype[10].annotationreftype[0]
+datatype[10].documenttype[0]
+datatype[11].id -1220861393
+datatype[11].arraytype[0]
+datatype[11].maptype[1]
+datatype[11].maptype[0].keytype 2
+datatype[11].maptype[0].valtype -1486737430
+datatype[11].weightedsettype[0]
+datatype[11].structtype[0]
+datatype[11].annotationreftype[0]
+datatype[11].documenttype[0]
+datatype[12].id 69621385
+datatype[12].arraytype[1]
+datatype[12].arraytype[0].datatype 339965458
+datatype[12].maptype[0]
+datatype[12].weightedsettype[0]
+datatype[12].structtype[0]
+datatype[12].annotationreftype[0]
+datatype[12].documenttype[0]
+datatype[13].id -628990518
+datatype[13].arraytype[0]
+datatype[13].maptype[0]
+datatype[13].weightedsettype[0]
+datatype[13].structtype[1]
+datatype[13].structtype[0].name "news.header"
+datatype[13].structtype[0].version 0
+datatype[13].structtype[0].field[19]
+datatype[13].structtype[0].field[0].name "url"
+datatype[13].structtype[0].field[0].datatype 10
+datatype[13].structtype[0].field[0].id[0]
+datatype[13].structtype[0].field[1].name "title"
+datatype[13].structtype[0].field[1].datatype 2
+datatype[13].structtype[0].field[1].id[0]
+datatype[13].structtype[0].field[2].name "last_downloaded"
+datatype[13].structtype[0].field[2].datatype 0
+datatype[13].structtype[0].field[2].id[0]
+datatype[13].structtype[0].field[3].name "value_long"
+datatype[13].structtype[0].field[3].datatype 4
+datatype[13].structtype[0].field[3].id[0]
+datatype[13].structtype[0].field[4].name "value_content"
+datatype[13].structtype[0].field[4].datatype 2
+datatype[13].structtype[0].field[4].id[0]
+datatype[13].structtype[0].field[5].name "value_raw"
+datatype[13].structtype[0].field[5].datatype 3
+datatype[13].structtype[0].field[5].id[0]
+datatype[13].structtype[0].field[6].name "stringarr"
+datatype[13].structtype[0].field[6].datatype -1486737430
+datatype[13].structtype[0].field[6].id[0]
+datatype[13].structtype[0].field[7].name "intarr"
+datatype[13].structtype[0].field[7].datatype -1245117006
+datatype[13].structtype[0].field[7].id[0]
+datatype[13].structtype[0].field[8].name "longarr"
+datatype[13].structtype[0].field[8].datatype 58874399
+datatype[13].structtype[0].field[8].id[0]
+datatype[13].structtype[0].field[9].name "bytearr"
+datatype[13].structtype[0].field[9].datatype 49942803
+datatype[13].structtype[0].field[9].id[0]
+datatype[13].structtype[0].field[10].name "floatarr"
+datatype[13].structtype[0].field[10].datatype 1650586661
+datatype[13].structtype[0].field[10].id[0]
+datatype[13].structtype[0].field[11].name "weightedsetint"
+datatype[13].structtype[0].field[11].datatype 519906144
+datatype[13].structtype[0].field[11].id[0]
+datatype[13].structtype[0].field[12].name "weightedsetstring"
+datatype[13].structtype[0].field[12].datatype 1328286588
+datatype[13].structtype[0].field[12].id[0]
+datatype[13].structtype[0].field[13].name "stringmap"
+datatype[13].structtype[0].field[13].datatype 339965458
+datatype[13].structtype[0].field[13].id[0]
+datatype[13].structtype[0].field[14].name "structfield"
+datatype[13].structtype[0].field[14].datatype -2092985853
+datatype[13].structtype[0].field[14].id[0]
+datatype[13].structtype[0].field[15].name "structmap"
+datatype[13].structtype[0].field[15].datatype 1901258752
+datatype[13].structtype[0].field[15].id[0]
+datatype[13].structtype[0].field[16].name "structarr"
+datatype[13].structtype[0].field[16].datatype 759956026
+datatype[13].structtype[0].field[16].id[0]
+datatype[13].structtype[0].field[17].name "arrmap"
+datatype[13].structtype[0].field[17].datatype -1220861393
+datatype[13].structtype[0].field[17].id[0]
+datatype[13].structtype[0].field[18].name "maparr"
+datatype[13].structtype[0].field[18].datatype 69621385
+datatype[13].structtype[0].field[18].id[0]
+datatype[13].structtype[0].inherits[0]
+datatype[13].annotationreftype[0]
+datatype[13].documenttype[0]
+datatype[14].id 538588767
+datatype[14].arraytype[0]
+datatype[14].maptype[0]
+datatype[14].weightedsettype[0]
+datatype[14].structtype[1]
+datatype[14].structtype[0].name "news.body"
+datatype[14].structtype[0].version 0
+datatype[14].structtype[0].field[0]
+datatype[14].structtype[0].inherits[0]
+datatype[14].annotationreftype[0]
+datatype[14].documenttype[0]
+datatype[15].id -1048827947
+datatype[15].arraytype[0]
+datatype[15].maptype[0]
+datatype[15].weightedsettype[0]
+datatype[15].structtype[0]
+datatype[15].annotationreftype[0]
+datatype[15].documenttype[1]
+datatype[15].documenttype[0].name "news"
+datatype[15].documenttype[0].version 0
+datatype[15].documenttype[0].headerstruct -628990518
+datatype[15].documenttype[0].bodystruct 538588767
+datatype[15].documenttype[0].inherits[0]
+annotationtype[0]
diff --git a/document/src/test/vespaxmlparser/test01.xml b/document/src/test/vespaxmlparser/test01.xml
new file mode 100644
index 00000000000..88bbf434533
--- /dev/null
+++ b/document/src/test/vespaxmlparser/test01.xml
@@ -0,0 +1,45 @@
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<!--
+    NOTE: see documentmanager.cfg for document type during test
+    a new document, adding all possible fields available to this documenttype 
+-->
+
+<vespafeed>
+    
+    <document documenttype="news" documentid="doc:news:http://news1"> 
+        <url>testUrl</url>
+        <title>testTitle</title>
+        <last_downloaded>1</last_downloaded>
+        <value_long>2</value_long>
+        <value_content>testValueContent</value_content>
+        <stringarr>
+            <item>stringarrItem1</item>
+            <item>stringarrItem2</item>
+        </stringarr>
+        <intarr>
+            <item>3</item>
+            <item>4</item>
+        </intarr>
+        <longarr>
+            <item>5</item>
+            <item>6</item>
+        </longarr>
+        <bytearr>
+            <item>7</item>
+            <item>8</item>
+        </bytearr>
+        <floatarr>
+            <item>9</item>
+            <item>10</item>
+        </floatarr>
+        <weightedsetint>
+            <item weight="11">11</item>
+            <item weight="12">12</item>
+        </weightedsetint>
+        <weightedsetstring>
+            <item weight="13">string13</item>
+            <item weight="14">string14</item>
+        </weightedsetstring>
+    </document>
+    
+</vespafeed>
diff --git a/document/src/test/vespaxmlparser/test02.xml b/document/src/test/vespaxmlparser/test02.xml
new file mode 100644
index 00000000000..d230733b276
--- /dev/null
+++ b/document/src/test/vespaxmlparser/test02.xml
@@ -0,0 +1,17 @@
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<!--
+    NOTE: see documentmanager.cfg for document type during test
+    smaller document adding just a few fields 
+-->
+
+<vespafeed>
+    <document documenttype="news" documentid="doc:news:http://news2"> 
+        <url>testUrl2</url>
+        <title>testTitle2</title>
+        <weightedsetint>
+            <item weight="21">21</item>
+            <item weight="22">22</item>
+        </weightedsetint>
+    </document>
+    
+</vespafeed>
diff --git a/document/src/test/vespaxmlparser/test03.xml b/document/src/test/vespaxmlparser/test03.xml
new file mode 100644
index 00000000000..84da06cc9ae
--- /dev/null
+++ b/document/src/test/vespaxmlparser/test03.xml
@@ -0,0 +1,47 @@
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<!--
+    NOTE: see documentmanager.cfg for document type during test
+    
+    A document update including:
+    
+    - different variants of assign
+    - all fields in this documenttype are assigned new values
+-->
+
+<vespafeed>
+    <update documenttype="news" documentid="doc:news:http://news3"> 
+        <assign field="url">assignUrl</assign>
+        <assign field="title">assignTitle</assign>
+        <assign field="last_downloaded">1</assign>
+        <assign field="value_long">2</assign>
+        <assign field="value_content">assignContent</assign>
+        <assign field="stringarr">
+            <item>assignString1</item>
+            <item>assignString2</item>
+        </assign>
+        <assign field="intarr">
+            <item>3</item>
+            <item>4</item>
+        </assign>
+        <assign field="longarr">
+            <item>5</item>
+            <item>6</item>
+        </assign>
+        <assign field="bytearr">
+            <item>7</item>
+            <item>8</item>
+        </assign>
+        <assign field="floatarr">
+            <item>9</item>
+            <item>10</item>
+        </assign>
+        <assign field="weightedsetint">
+            <item weight="11">11</item>
+            <item weight="12">12</item>
+        </assign>
+        <assign field="weightedsetstring">
+            <item weight="13">assign13</item>
+            <item weight="14">assign14</item>
+        </assign>
+    </update>
+</vespafeed>
diff --git a/document/src/test/vespaxmlparser/test04.xml b/document/src/test/vespaxmlparser/test04.xml
new file mode 100644
index 00000000000..0b1b76c1fc8
--- /dev/null
+++ b/document/src/test/vespaxmlparser/test04.xml
@@ -0,0 +1,28 @@
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<!--
+    NOTE: see documentmanager.cfg for document type during test
+    
+    A document update including:
+    
+    - different variants of assign
+    - Only a few fieldupdates (implies rest is not set/null)
+-->
+
+<vespafeed>
+    <update documenttype="news" documentid="doc:news:http://news4"> 
+        <assign field="url">assignUrl</assign>
+        <assign field="value_long">2</assign>
+        <assign field="stringarr">
+            <item>assignString1</item>
+            <item>assignString2</item>
+        </assign>
+        <assign field="intarr">
+            <item>3</item>
+            <item>4</item>
+        </assign>
+        <assign field="weightedsetint">
+            <item weight="11">11</item>
+            <item weight="12">12</item>
+        </assign>
+    </update>
+</vespafeed>
diff --git a/document/src/test/vespaxmlparser/test05.xml b/document/src/test/vespaxmlparser/test05.xml
new file mode 100644
index 00000000000..5467a6999b8
--- /dev/null
+++ b/document/src/test/vespaxmlparser/test05.xml
@@ -0,0 +1,28 @@
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<!--
+    NOTE: see documentmanager.cfg for document type during test
+    
+    A document update including:
+
+    - Only multivalue fields are supported by "add"
+    - on "weightedsetstring", no given weight implies weight = 1
+-->
+
+<vespafeed>
+    <update documenttype="news" documentid="doc:news:http://news5"> 
+        <add field="stringarr">
+            <item>addString1</item>
+            <item>addString2</item>
+        </add>
+        <add field="longarr">
+            <item>5</item>
+        </add>
+        <add field="weightedsetint">
+            <item weight="11">11</item>
+            <item weight="12">12</item>
+        </add>
+        <add field="weightedsetstring">
+            <item>add13</item>
+        </add>
+    </update>
+</vespafeed>
diff --git a/document/src/test/vespaxmlparser/test06.xml b/document/src/test/vespaxmlparser/test06.xml
new file mode 100644
index 00000000000..d2670b24663
--- /dev/null
+++ b/document/src/test/vespaxmlparser/test06.xml
@@ -0,0 +1,77 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+
+<!--
+    Document   : testXML06.xml
+    Created on : July 26, 2007, 11:18 AM
+    Author     : alimf
+    Description:
+        Several Documents that will fail/be skipped, except the last one.
+-->
+
+<vespafeed>
+    
+    <document documenttype="news" documentid="doc:news:http://news6a"> 
+        <value_long>txt</value_long>
+    </document>
+    
+    <document documenttype="news" documentid="doc:news:http://news6b"> 
+        <stringarr>
+            <item>stringarrItem1</item>
+            <item></item>
+        </stringarr>
+    </document>
+    
+    <document documenttype="news" documentid="doc:news:http://news6c"> 
+        <intarr>
+            <item>1</item>
+            <item>txt</item>
+        </intarr>
+    </document>
+    
+    <document documenttype="news" documentid="doc:news:http://news6d"> 
+        <longarr>
+            <item>1</item>
+            <item>    </item>
+        </longarr>
+    </document>
+    
+    <document documenttype="news" documentid="doc:news:http://news6e"> 
+        <bytearr>
+            <item>1</item>
+            <item>128</item>
+        </bytearr>
+    </document>
+    
+    <document documenttype="news" documentid="doc:news:http://news6f"> 
+        <floatarr>
+            <item>1.0</item>
+            <item>two</item>
+        </floatarr>
+    </document>
+    
+    <document documenttype="news" documentid="doc:news:http://news6g"> 
+        <weightedsetint>
+            <item>1</item>
+            <item weight="12">txt</item>
+        </weightedsetint>
+    </document>
+
+    <document documenttype="news" documentid="doc:news:http://news6h"> 
+        <weightedsetint>
+            <item weight="1">2</item>
+            <item weight="three">4</item>
+        </weightedsetint>
+    </document>
+    
+    <document documenttype="news" documentid="doc:news:http://news6i"> 
+        <weightedsetstring>
+            <item weight="txt">aString</item>
+        </weightedsetstring>
+    </document>
+    
+    <document documenttype="news" documentid="doc:news:http://news6j"> 
+        <title>myTitle</title>
+    </document>
+    
+</vespafeed>
diff --git a/document/src/test/vespaxmlparser/test07.xml b/document/src/test/vespaxmlparser/test07.xml
new file mode 100644
index 00000000000..9d67646798a
--- /dev/null
+++ b/document/src/test/vespaxmlparser/test07.xml
@@ -0,0 +1,72 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<!--
+    - Different numeric operations
+
+    - different variants of assign
+    - some fields in this documenttype are assigned invalid values
+-->
+<vespafeed>
+    <update documenttype="news" documentid="doc:news:http://news7a">
+        <alter field="last_downloaded">
+            <increment by="2" />
+            <decrement by="3" />
+            <multiply by="4" />
+            <divide by="5" />
+        </alter>
+
+        <increment field="weightedsetint" by="6">
+            <key>7</key>
+        </increment>
+
+        <decrement field="weightedsetint" by="8">
+            <key>9</key>
+        </decrement>
+        
+        <multiply field="intarr" by="10">
+            <key>11</key>
+        </multiply> 
+
+        <multiply field="floatarr" by="12">
+            <key>13</key>
+        </multiply> 
+        
+        <divide field="floatarr" by="14">
+            <key>15</key>
+        </divide> 
+    </update>
+    
+    <update documenttype="news" documentid="doc:news:http://news7b">
+        <alter field="title">
+            <increment by="2" />
+            <decrement by="3" />
+            <multiply by="4" />
+            <divide by="5" />
+        </alter>
+    </update>
+    
+    <update documenttype="news" documentid="doc:news:http://news7c">
+        <increment field="weightedsetint" by="five">
+            <key>1</key>
+        </increment> 
+    </update>
+    
+    <update documenttype="news" documentid="doc:news:http://news7d">
+        <decrement field="weightedsetint" by="2">
+            <key></key>
+        </decrement> 
+    </update>
+    
+    <update documenttype="news" documentid="doc:news:http://news7e">
+        <multiply field="floatarr">
+            <key>3</key>
+        </multiply> 
+    </update>
+    
+    <update documenttype="news" documentid="doc:news:http://news7f">
+        <divide field="floatarr" by="3">
+            <key>four</key>
+        </divide> 
+    </update>
+    
+</vespafeed>
diff --git a/document/src/test/vespaxmlparser/test08.xml b/document/src/test/vespaxmlparser/test08.xml
new file mode 100644
index 00000000000..e9dd3d31940
--- /dev/null
+++ b/document/src/test/vespaxmlparser/test08.xml
@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+
+<!-- 
+    NOTE: see documentmanager.cfg for document type during test
+    
+    - Different types of remove
+    - All are valid
+    
+-->
+
+<vespafeed>
+    <update documenttype="news" documentid="doc:news:http://news8a"> 
+        <remove field="stringarr">
+            <item>removeString1</item>
+            <item>removeString2</item>
+        </remove>
+        <remove field="weightedsetint">
+            <item>4</item>
+            <item>5</item>
+        </remove>
+    </update>
+</vespafeed>
diff --git a/document/src/test/vespaxmlparser/test09.xml b/document/src/test/vespaxmlparser/test09.xml
new file mode 100644
index 00000000000..1d82f865f73
--- /dev/null
+++ b/document/src/test/vespaxmlparser/test09.xml
@@ -0,0 +1,22 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+
+<!--
+    NOTE: see documentmanager.cfg for document type during test
+    
+    - Different types of remove document
+    - All are valid
+        
+-->
+
+<vespafeed>
+    <remove documentid="doc:news:http://news9a" />
+
+    <remove documentid="doc:news:http://news9b">
+    </remove>
+
+    <remove>
+        <uri>doc:news:http://news9c</uri>
+    </remove>
+    
+</vespafeed>
diff --git a/document/src/test/vespaxmlparser/test10.xml b/document/src/test/vespaxmlparser/test10.xml
new file mode 100644
index 00000000000..855ff4b20e9
--- /dev/null
+++ b/document/src/test/vespaxmlparser/test10.xml
@@ -0,0 +1,94 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+
+<!--
+    Document   : test10.xml
+    Created on : July 27, 2007, 11:37 AM
+    Author     : alimf
+    Description:
+        this feed contains both documents, updates and removes.
+-->
+
+<vespafeed>
+ 
+    <document documenttype="news" documentid="doc:news:http://news10a"> 
+        <url>testUrl</url>
+        <title>testTitle</title>
+        <last_downloaded>1</last_downloaded>
+        <value_long>2</value_long>
+        <value_content>testValueContent</value_content>
+        <stringarr>
+            <item>stringarrItem1</item>
+            <item>stringarrItem2</item>
+        </stringarr>
+        <intarr>
+            <item>3</item>
+            <item>4</item>
+        </intarr>
+        <longarr>
+            <item>5</item>
+            <item>6</item>
+        </longarr>
+        <bytearr>
+            <item>7</item>
+            <item>8</item>
+        </bytearr>
+        <floatarr>
+            <item>9</item>
+            <item>10</item>
+        </floatarr>
+        <weightedsetint>
+            <item weight="11">11</item>
+            <item weight="12">12</item>
+        </weightedsetint>
+        <weightedsetstring>
+            <item weight="13">string13</item>
+            <item weight="14">string14</item>
+        </weightedsetstring>
+    </document>
+    
+    <document documenttype="news" documentid="doc:news:http://news10b"> 
+        <url>testUrl2</url>
+    </document>
+
+    <update documenttype="news" documentid="doc:news:http://news10c"> 
+        <add field="stringarr">
+            <item>addString1</item>
+            <item>addString2</item> 
+        </add>
+        <add field="longarr">
+            <item>5</item> 
+        </add>
+        <add field="weightedsetint">
+            <item weight="11">11</item>
+            <item weight="12">12</item>
+        </add>
+        <add field="weightedsetstring">
+            <item>add13</item>
+        </add>
+    </update>
+
+    <update documenttype="news" documentid="doc:news:http://news10d"> 
+        <assign field="url">assignUrl</assign>
+        <assign field="value_long">2</assign>
+        <assign field="stringarr">
+            <item>assignString1</item>
+            <item>assignString2</item>
+        </assign>
+        <assign field="intarr">
+            <item>3</item>
+            <item>4</item>
+        </assign>
+        <assign field="weightedsetint">
+            <item weight="11">11</item>
+            <item weight="12">12</item>
+        </assign>
+    </update>
+
+    <remove documentid="doc:news:http://news10e"/>
+
+    <remove>
+        <uri>doc:news:http://news10f</uri>
+    </remove>
+    
+</vespafeed>
diff --git a/document/src/test/vespaxmlparser/test12.xml b/document/src/test/vespaxmlparser/test12.xml
new file mode 100644
index 00000000000..349283dd499
--- /dev/null
+++ b/document/src/test/vespaxmlparser/test12.xml
@@ -0,0 +1,43 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<vespafeed>
+ 
+    <!--warning document: -->
+    <vespaadd>
+        <document documenttype="news" documentid="34sdfasva4"> 
+            <value_long>testUrlA</value_long>
+        </document>
+    </vespaadd>
+
+    <!-- bad document: -->
+    <document documenttype="news" documentid="doc:news:http://news12b"> 
+        <value_long>testUrlB</value_long>
+    </document>
+    
+    <!-- OK document -->
+    <document documenttype="news" documentid="doc:news:http://news12c"> 
+        <url>testUrl</url>
+        <title>testTitle</title>
+    </document>
+    
+    <!-- bad document: -->
+    <document documenttype="news" documentid="doc:news:http://news12d"> 
+        <value_long>testUrlC</value_long>
+    </documen>
+
+    <!-- OK document -->
+    <document documenttype="news" documentid="doc:news:http://news12e"> 
+        <url>testUrl</url>
+        <title>testTitle</title>
+    </document>
+    
+    <!-- bad document: -->
+    <document documenttype="news" documentid="doc:news:http://news12f"> 
+        <value_long>testUrlD</value_long>
+    </document>
+    
+    <!--warning document: -->
+    <document documenttype="news" documentid="98svh98sdfh"> 
+        <value_long>testUrlD</value_long>
+    </document>
+</vespafeed>
diff --git a/document/src/test/vespaxmlparser/test13.xml b/document/src/test/vespaxmlparser/test13.xml
new file mode 100644
index 00000000000..9d5bc88a195
--- /dev/null
+++ b/document/src/test/vespaxmlparser/test13.xml
@@ -0,0 +1,37 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<vespafeed>
+    <document documenttype="alltypes" documentid="doc:foobar:http://all.types">
+        <stringval>Banana</stringval>
+        <intval1>2146983647</intval1>
+        <intval2>4294967291</intval2>
+        <intval3>-5</intval3>
+        <longval1>9223372036354454243</longval1>
+        <longval2>18446744073709551029</longval2>
+        <longval3>-587</longval3>
+        <byteval1>126</byteval1>
+        <byteval2>135</byteval2>
+        <byteval3>-121</byteval3>
+        <floatval>4.73</floatval>
+        <doubleval>-9.11</doubleval>
+        <rawval1>ABCDEFGHIJKL</rawval1>
+        <rawval2 binaryencoding="baSe64">RG9uYWxkRHVjawo=</rawval2>
+        <urival>http://www.vespa.vespa/boo/baa#frag</urival>
+        <contentval1 contenttype="text/plain" encoding="utf-8" language="no">This is a test</contentval1>
+        <contentval2 contenttype="text/plain" encoding="utf-8" language="no" binaryencoding="baSE64">VGhpcyBpcyBhIHRlc3QK</contentval2>
+        <arrayofstringval>
+            <item>Boobaa</item>
+            <item>Bibiii</item>
+        </arrayofstringval>
+        <weightedsetofstringval>
+            <item weight="5">Baahabhh</item>
+            <item weight="10">bkasj</item>
+            <item>asdkfjas</item>
+        </weightedsetofstringval>
+        <tagval>
+            <item weight="5">Baahabhh</item>
+            <item weight="10">bkasj</item>
+            <item>asdkfjas</item>
+        </tagval>
+    </document>
+</vespafeed>
diff --git a/document/src/test/vespaxmlparser/testXMLfile.xml b/document/src/test/vespaxmlparser/testXMLfile.xml
new file mode 100644
index 00000000000..59c83698951
--- /dev/null
+++ b/document/src/test/vespaxmlparser/testXMLfile.xml
@@ -0,0 +1,29 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<vespafeed>
+    <document documenttype="music" documentid="doc:music:http://music.yahoo.com/bobdylan/BestOf"> 
+        <url>http://music.yahoo.com/bobdylan/BestOf</url>
+        <songs>Knockin on Heaven's Door; Mr. Tambourine Man</songs>
+        <title>Best of Bob Dylan</title>
+        <tracks>
+            <item>Mr. Tambourine Man</item>
+            <item>Someday Baby</item>
+            <item>Blowin' In The Wind</item>
+        </tracks> 
+        <popularity>
+            <item weight="3">0</item>
+            <item weight="5">1</item>
+            <item weight="30">2</item>
+            <item weight="26">3</item>
+        </popularity>
+    </document>
+    
+    <remove documentid="doc:music:http://music.yahoo.com/BritneySpears/HitMe"/>
+    
+    <update documenttype="music" documentid="doc:music:http://music.yahoo.com/bobdylan/BestOf"> 
+        <assign field="title">The Best of Bob Dylan</assign>
+        <add field="tracks">
+            <item>Man Of Constant Sorrow</item>
+        </add>
+    </update>
+</vespafeed>
diff --git a/document/src/test/vespaxmlparser/test_docindoc.xml b/document/src/test/vespaxmlparser/test_docindoc.xml
new file mode 100644
index 00000000000..3de94107d8e
--- /dev/null
+++ b/document/src/test/vespaxmlparser/test_docindoc.xml
@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<vespafeed>
+
+  <document type="outerdoc" id="doc:outer:this:is:outer:doc">
+    <innerdocuments>
+        <item>
+            <document type="docindoc" id="doc:inner:this:is:inner:doc:a">
+              <name>Peter Sellers</name>
+              <content>Comedian</content>
+            </document>
+        </item>
+        <item>
+            <document type="docindoc" id="doc:inner:this:is:inner:doc:b">
+              <name>Ole Olsen</name>
+              <content>Common man</content>
+            </document>
+        </item>
+        <item>
+            <document type="docindoc" id="doc:inner:this:is:inner:doc:c">
+              <name>Stein Nilsen</name>
+              <content>Worker</content>
+            </document>
+        </item>
+    </innerdocuments>
+  </document>
+
+</vespafeed>
diff --git a/document/src/test/vespaxmlparser/test_position.xml b/document/src/test/vespaxmlparser/test_position.xml
new file mode 100644
index 00000000000..d03bc27480f
--- /dev/null
+++ b/document/src/test/vespaxmlparser/test_position.xml
@@ -0,0 +1,15 @@
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<vespafeed>
+    <document documenttype="my_doc" documentid="doc:scheme:">
+        <my_pos><x>1</x><y>2</y></my_pos>
+    </document>
+    <document documenttype="my_doc" documentid="doc:scheme:">
+        <my_pos>E3;N4</my_pos>
+    </document>
+    <document documenttype="my_doc" documentid="doc:scheme:">
+        <my_pos>5;6</my_pos>
+    </document>
+    <document documenttype="my_doc" documentid="doc:scheme:">
+        <my_pos binaryencoding="base64">Nzs4</my_pos>
+    </document>
+</vespafeed>
diff --git a/document/src/test/vespaxmlparser/test_uri.xml b/document/src/test/vespaxmlparser/test_uri.xml
new file mode 100644
index 00000000000..c261e5e3e31
--- /dev/null
+++ b/document/src/test/vespaxmlparser/test_uri.xml
@@ -0,0 +1,14 @@
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<vespafeed>
+    <document documenttype="my_doc" documentid="doc:scheme:">
+        <my_uri>scheme://host</my_uri>
+    </document>
+    <document documenttype="my_doc" documentid="doc:scheme:">
+        <my_arr><item>scheme://host</item></my_arr>
+    </document>
+    <update documenttype="my_doc" documentid="doc:scheme:">
+        <add field="my_arr">
+            <item>scheme://host</item>
+        </add>
+    </update>
+</vespafeed>
diff --git a/document/src/test/vespaxmlparser/test_url.xml b/document/src/test/vespaxmlparser/test_url.xml
new file mode 100644
index 00000000000..0710ce0d0f3
--- /dev/null
+++ b/document/src/test/vespaxmlparser/test_url.xml
@@ -0,0 +1,20 @@
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<vespafeed>
+    <document documenttype="my_doc" documentid="doc:scheme:">
+        <my_url>
+            <all>scheme://user@host:99/path?query#fragment</all>
+            <scheme>scheme</scheme>
+            <host>host</host>
+            <port>99</port>
+            <path>/path</path>
+            <query>query</query>
+            <fragment>fragment</fragment>
+        </my_url>
+    </document>
+    <document documenttype="my_doc" documentid="doc:scheme:">
+        <my_url>scheme://user@host:99/path?query#fragment</my_url>
+    </document>
+    <document documenttype="my_doc" documentid="doc:scheme:">
+        <my_url binaryencoding="base64">c2NoZW1lOi8vdXNlckBob3N0Ojk5L3BhdGg/cXVlcnkjZnJhZ21lbnQ=</my_url>
+    </document>
+</vespafeed>
diff --git a/document/src/test/vespaxmlparser/testalltypes.xml b/document/src/test/vespaxmlparser/testalltypes.xml
new file mode 100644
index 00000000000..d05cec70b11
--- /dev/null
+++ b/document/src/test/vespaxmlparser/testalltypes.xml
@@ -0,0 +1,136 @@
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<!--
+    NOTE: see documentmanager.cfg for document type during test
+    a new document, adding all possible fields available to this documenttype 
+-->
+
+<vespafeed>
+    
+    <document documenttype="news" documentid="doc:news:http://news1"> 
+        <url>testUrl</url>
+        <title>testTitle</title>
+        <last_downloaded>1</last_downloaded>
+        <value_long>2</value_long>
+
+        <value_raw binaryencoding="base64">Zm9vYmFy</value_raw>
+        <value_content>dGVzdFZhbHVlQ29udGVudA==</value_content>
+
+        <stringarr>
+            <item>stringarrItem1</item>
+            <item>stringarrItem2</item>
+        </stringarr>
+        <intarr>
+            <item>2983742937</item>
+            <item>2983742938</item>
+            <item>4294967295</item>
+            <item>-2147483648</item>
+        </intarr>
+        <longarr>
+            <item>5</item>
+            <item>6</item>
+        </longarr>
+        <bytearr>
+            <item>7</item>
+            <item>8</item>
+        </bytearr>
+        <floatarr>
+            <item>9</item>
+            <item>10</item>
+        </floatarr>
+        <weightedsetint>
+            <item weight="11">11</item>
+            <item weight="12">12</item>
+        </weightedsetint>
+        <weightedsetstring>
+            <item weight="13">string13</item>
+            <item weight="14">string14</item>
+        </weightedsetstring>
+
+        <stringmap>
+            <item><key>italia</key><value>slovakia</value></item>
+            <item><key>danmark</key><value>japan</value></item> 
+            <item><key>paraguay</key><value>new zealand</value></item>
+        </stringmap>
+
+        <structfield>
+            <title>star wars</title>
+            <structfield>dummy</structfield>
+        </structfield>
+
+        <structarr>
+           <item>
+             <title>title1</title>
+             <mymap>
+               <item><key>key1</key><value>value1</value></item>
+               <item><key>key2</key><value>value2</value></item> 
+             </mymap>
+           </item>
+           <item> 
+             <title>title2</title> 
+             <mymap> 
+               <item><key>key1.1</key><value>value1.1</value></item> 
+               <item><key>key1.2</key><value>value1.2</value></item>  
+             </mymap> 
+           </item>
+        </structarr>
+
+        <structmap>
+           <item>
+              <key>32</key>
+              <value>
+                 <title>title1</title>
+                 <mymap>
+                   <item><key>key1</key><value>value1</value></item> 
+                   <item><key>key2</key><value>value2</value></item>  
+                 </mymap>
+              </value>
+           </item>
+           <item> 
+              <key>28</key> 
+              <value> 
+                 <title>title2</title> 
+                 <mymap> 
+                   <item><key>key3</key><value>value3</value></item>  
+                   <item><key>key4</key><value>value4</value></item>   
+                 </mymap> 
+              </value> 
+           </item>
+        </structmap>
+
+        <arrmap>
+           <item>
+              <key>foo</key>
+              <value>
+                 <item>hei1</item>
+                 <item>hei2</item> 
+                 <item>hei3</item> 
+              </value>
+           </item>
+           <item> 
+              <key>bar</key> 
+              <value> 
+                 <item>hei4</item> 
+                 <item>hei5</item>  
+                 <item>hei6</item>  
+              </value> 
+           </item>
+        </arrmap>
+
+        <maparr>
+            <item>
+                <item><key>key1</key><value>value1</value></item>
+                <item><key>key2</key><value>value2</value></item>
+            </item>
+            <item>
+                <item><key>key3</key><value>value3</value></item>
+                <item><key>key4</key><value>value4</value></item>
+            </item>
+            <item>
+                <item><key>key5</key><value>value5</value></item>
+                <item><key>key6</key><value>value6</value></item>
+            </item>
+        </maparr>
+
+    </document>
+    
+</vespafeed>
diff --git a/document/src/test/vespaxmlparser/testandset.xml b/document/src/test/vespaxmlparser/testandset.xml
new file mode 100644
index 00000000000..10b2e45cc46
--- /dev/null
+++ b/document/src/test/vespaxmlparser/testandset.xml
@@ -0,0 +1,22 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+
+<!--
+    Document   : testandset.xml
+    Created on : October 16, 2015
+    Author     : Vegard Sjonfjell
+    Description:
+    This feed contains documents, updates and removes with a test and set condition set
+-->
+
+<vespafeed>
+    <document documenttype="news" documentid="id:news:news::test1" condition="news.value_long == 1">
+        <value_long>2</value_long>
+    </document>
+
+    <update documenttype="news" documentid="id:news:news::test2" condition="news.value_long == 1">
+        <assign field="value_long">2</assign>
+    </update>
+
+    <remove documentid="id:news:news::test3" condition="news.value_long == 1"/>
+</vespafeed>
diff --git a/document/src/test/vespaxmlparser/testmapnokey.xml b/document/src/test/vespaxmlparser/testmapnokey.xml
new file mode 100644
index 00000000000..2d0527f5719
--- /dev/null
+++ b/document/src/test/vespaxmlparser/testmapnokey.xml
@@ -0,0 +1,23 @@
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<!--
+    NOTE: see documentmanager.cfg for document type during test
+    a new document, adding all possible fields available to this documenttype 
+-->
+
+<vespafeed>
+    
+    <document documenttype="news" documentid="doc:news:http://news1">
+
+        <arrmap>
+           <item> 
+              <value> 
+                 <item>hei4</item> 
+                 <item>hei5</item>  
+                 <item>hei6</item>  
+              </value> 
+           </item>
+        </arrmap>
+
+    </document>
+    
+</vespafeed>
diff --git a/document/src/test/vespaxmlparser/testmapnovalue.xml b/document/src/test/vespaxmlparser/testmapnovalue.xml
new file mode 100644
index 00000000000..07478c2d46d
--- /dev/null
+++ b/document/src/test/vespaxmlparser/testmapnovalue.xml
@@ -0,0 +1,19 @@
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<!--
+    NOTE: see documentmanager.cfg for document type during test
+    a new document, adding all possible fields available to this documenttype 
+-->
+
+<vespafeed>
+    
+    <document documenttype="news" documentid="doc:news:http://news1">
+
+        <arrmap>
+           <item> 
+              <key>bar</key> 
+           </item>
+        </arrmap>
+
+    </document>
+    
+</vespafeed>
diff --git a/document/src/testlist.txt b/document/src/testlist.txt
new file mode 100644
index 00000000000..7e2776a2c10
--- /dev/null
+++ b/document/src/testlist.txt
@@ -0,0 +1,10 @@
+tests
+tests/annotation
+tests/base
+tests/datatype
+tests/fieldvalue
+tests/predicate
+tests/repo
+tests/serialization
+tests/struct_anno
+tests/tensor_fieldvalue
diff --git a/document/src/tests/.gitignore b/document/src/tests/.gitignore
new file mode 100644
index 00000000000..3e5d3aea0af
--- /dev/null
+++ b/document/src/tests/.gitignore
@@ -0,0 +1,16 @@
+*.So
+*.core
+.*.swp
+.config.log
+.depend
+Makefile
+cpp-globalids.txt
+indexdir
+mmapbuffertestfile.txt
+mmapq_test
+mytestfile
+mytestfile_clear
+test.vlog
+testrunner
+*_test
+document_testrunner_app
diff --git a/document/src/tests/CMakeLists.txt b/document/src/tests/CMakeLists.txt
new file mode 100644
index 00000000000..95e6b4b9a18
--- /dev/null
+++ b/document/src/tests/CMakeLists.txt
@@ -0,0 +1,36 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(document_testrunner_app
+    SOURCES
+    teststringutil.cpp
+    testbytebuffer.cpp
+    stringtokenizertest.cpp
+    documentcalculatortestcase.cpp
+    buckettest.cpp
+    globalidtest.cpp
+    documentidtest.cpp
+    documenttypetestcase.cpp
+    primitivefieldvaluetest.cpp
+    arrayfieldvaluetest.cpp
+    weightedsetfieldvaluetest.cpp
+    structfieldvaluetest.cpp
+    documenttestcase.cpp
+    testdocmantest.cpp
+    documentupdatetestcase.cpp
+    fieldpathupdatetestcase.cpp
+    documentselectparsertest.cpp
+    bucketselectortest.cpp
+    testxml.cpp
+    forcelinktest.cpp
+    orderingselectortest.cpp
+    testrunner.cpp
+    heapdebuggerother.cpp
+    positiontypetest.cpp
+    urltypetest.cpp
+    fieldsettest.cpp
+    gid_filter_test.cpp
+    DEPENDS
+    document
+    AFTER
+    document_documentconfig
+)
+vespa_add_test(NAME document_testrunner_app COMMAND document_testrunner_app)
diff --git a/document/src/tests/annotation/.gitignore b/document/src/tests/annotation/.gitignore
new file mode 100644
index 00000000000..861906f0e95
--- /dev/null
+++ b/document/src/tests/annotation/.gitignore
@@ -0,0 +1,6 @@
+*.So
+*_test
+.depend
+Makefile
+annotation_test
+document_annotation_test_app
diff --git a/document/src/tests/annotation/CMakeLists.txt b/document/src/tests/annotation/CMakeLists.txt
new file mode 100644
index 00000000000..1ec6a3e3dde
--- /dev/null
+++ b/document/src/tests/annotation/CMakeLists.txt
@@ -0,0 +1,10 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(document_annotation_test_app
+    SOURCES
+    annotation_test.cpp
+    DEPENDS
+    document
+    AFTER
+    document_documentconfig
+)
+vespa_add_test(NAME document_annotation_test_app COMMAND document_annotation_test_app)
diff --git a/document/src/tests/annotation/annotation_test.cpp b/document/src/tests/annotation/annotation_test.cpp
new file mode 100644
index 00000000000..c4976485b8b
--- /dev/null
+++ b/document/src/tests/annotation/annotation_test.cpp
@@ -0,0 +1,295 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Unit tests for annotation.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("annotation_test");
+
+#include <stdlib.h>
+#include <vespa/document/annotation/alternatespanlist.h>
+#include <vespa/document/annotation/annotation.h>
+#include <vespa/document/annotation/span.h>
+#include <vespa/document/annotation/spanlist.h>
+#include <vespa/document/annotation/spantree.h>
+#include <vespa/document/datatype/annotationreferencedatatype.h>
+#include <vespa/document/datatype/annotationtype.h>
+#include <vespa/document/datatype/arraydatatype.h>
+#include <vespa/document/datatype/primitivedatatype.h>
+#include <vespa/document/datatype/structdatatype.h>
+#include <vespa/document/fieldvalue/annotationreferencefieldvalue.h>
+#include <vespa/document/fieldvalue/arrayfieldvalue.h>
+#include <vespa/document/fieldvalue/doublefieldvalue.h>
+#include <vespa/document/fieldvalue/structfieldvalue.h>
+#include <vespa/vespalib/testkit/testapp.h>
+#include <memory>
+
+using std::unique_ptr;
+using namespace document;
+
+namespace {
+
+AnnotationType text_type(1, "text");
+AnnotationType begin_tag(2, "begintag");
+AnnotationType end_tag(3, "endtag");
+AnnotationType body_type(4, "body");
+AnnotationType header_type(5, "header");
+AnnotationType city_type(6, "city");
+AnnotationType markup_type(7, "markup");
+
+template <typename T>
+unique_ptr<T> makeUP(T *p) { return unique_ptr<T>(p); }
+
+TEST("requireThatSpansHaveOrder") {
+    Span span(10, 10);
+    Span before(5, 3);
+    Span overlap_start(8, 10);
+    Span contained(12, 3);
+    Span overlap_end(15, 10);
+    Span after(21, 10);
+    Span overlap_complete(5, 20);
+    Span shorter(10, 5);
+    Span longer(10, 15);
+    EXPECT_TRUE(span > before);
+    EXPECT_TRUE(span > overlap_start);
+    EXPECT_TRUE(span < contained);
+    EXPECT_TRUE(span < overlap_end);
+    EXPECT_TRUE(span < after);
+    EXPECT_TRUE(span > overlap_complete);
+    EXPECT_TRUE(span > shorter);
+    EXPECT_TRUE(span < longer);
+    EXPECT_TRUE(!(span < span));
+}
+
+TEST("requireThatSimpleSpanTreeCanBeBuilt") {
+    SpanList::UP root(new SpanList);
+    root->add(makeUP(new Span(0, 19)));
+    root->add(makeUP(new Span(19, 5)));
+    root->add(makeUP(new Span(24, 21)));
+    root->add(makeUP(new Span(45, 23)));
+    root->add(makeUP(new Span(68, 14)));
+
+    EXPECT_EQUAL(5u, root->size());
+    SpanList::const_iterator it = root->begin();
+    EXPECT_EQUAL(Span(0, 19), *(static_cast<Span *>(*it++)));
+    EXPECT_EQUAL(Span(19, 5), *(static_cast<Span *>(*it++)));
+    EXPECT_EQUAL(Span(24, 21), *(static_cast<Span *>(*it++)));
+    EXPECT_EQUAL(Span(45, 23), *(static_cast<Span *>(*it++)));
+    EXPECT_EQUAL(Span(68, 14), *(static_cast<Span *>(*it++)));
+    EXPECT_TRUE(it == root->end());
+
+    SpanTree tree("html", std::move(root));
+}
+
+TEST("requireThatSpanTreeCanHaveAnnotations") {
+    SpanList::UP root_owner(new SpanList);
+    SpanList *root = root_owner.get();
+    SpanTree tree("html", std::move(root_owner));
+
+    Span &span1 = root->add(makeUP(new Span(0, 19)));
+    tree.annotate(span1, markup_type);
+
+    Span &span2 = root->add(makeUP(new Span(19, 5)));
+    tree.annotate(span2, text_type);
+
+    EXPECT_EQUAL(2u, tree.numAnnotations());
+    SpanTree::const_iterator it = tree.begin();
+
+    EXPECT_EQUAL(Annotation(markup_type), *it++);
+    EXPECT_EQUAL(Annotation(text_type), *it++);
+    EXPECT_TRUE(it == tree.end());
+}
+
+TEST("requireThatSpanTreeCanHaveMultipleLevels") {
+    SpanList::UP root_owner(new SpanList);
+    SpanList *root = root_owner.get();
+    SpanTree tree("html", std::move(root_owner));
+
+    SpanList::UP header(new SpanList);
+    tree.annotate(header->add(makeUP(new Span(6, 6))), begin_tag);
+    tree.annotate(header->add(makeUP(new Span(12, 7))), begin_tag);
+    tree.annotate(header->add(makeUP(new Span(19, 5))), text_type);
+    tree.annotate(header->add(makeUP(new Span(24, 8))), end_tag);
+    tree.annotate(header->add(makeUP(new Span(32, 7))), end_tag);
+    tree.annotate(*header, header_type);
+
+    SpanList::UP body(new SpanList);
+    tree.annotate(body->add(makeUP(new Span(39, 6))), begin_tag);
+    tree.annotate(body->add(makeUP(new Span(45, 23))), text_type);
+    tree.annotate(body->add(makeUP(new Span(68, 7))), end_tag);
+    tree.annotate(*body, body_type);
+
+    tree.annotate(root->add(makeUP(new Span(0, 6))), begin_tag);
+    root->add(std::move(std::move(header)));
+    root->add(std::move(std::move(body)));
+    tree.annotate(root->add(makeUP(new Span(75, 7))), end_tag);
+
+    EXPECT_EQUAL(12u, tree.numAnnotations());
+    SpanTree::const_iterator it = tree.begin();
+    EXPECT_EQUAL(Annotation(begin_tag), *it++);
+    EXPECT_EQUAL(Annotation(begin_tag), *it++);
+    EXPECT_EQUAL(Annotation(text_type), *it++);
+    EXPECT_EQUAL(Annotation(end_tag), *it++);
+    EXPECT_EQUAL(Annotation(end_tag), *it++);
+    EXPECT_EQUAL(Annotation(header_type), *it++);
+    EXPECT_EQUAL(Annotation(begin_tag), *it++);
+    EXPECT_EQUAL(Annotation(text_type), *it++);
+    EXPECT_EQUAL(Annotation(end_tag), *it++);
+    EXPECT_EQUAL(Annotation(body_type), *it++);
+    EXPECT_EQUAL(Annotation(begin_tag), *it++);
+    EXPECT_EQUAL(Annotation(end_tag), *it++);
+    EXPECT_TRUE(it == tree.end());
+}
+
+TEST("requireThatAnnotationsCanHaveValues") {
+    PrimitiveDataType double_type(DataType::T_DOUBLE);
+    StructDataType city_data_type;
+    city_data_type.addField(Field("latitude", 0, double_type, false));
+    city_data_type.addField(Field("longitude", 1, double_type, false));
+
+    StructFieldValue::UP position(new StructFieldValue(city_data_type));
+    position->setValue("latitude", DoubleFieldValue(37.774929));
+    position->setValue("longitude", DoubleFieldValue(-122.419415));
+    StructFieldValue original(*position);
+
+    Annotation city(city_type, std::move(position));
+
+    EXPECT_TRUE(*city.getFieldValue() == original);
+}
+
+TEST("requireThatAnnotationsCanReferenceAnnotations") {
+    SpanList::UP root(new SpanList);
+    SpanTree tree("html", std::move(root));
+    size_t san_index = tree.annotate(makeUP(new Annotation(text_type)));
+    size_t fran_index = tree.annotate(makeUP(new Annotation(text_type)));
+
+    AnnotationReferenceDataType annotation_ref_type(text_type, 101);
+    ArrayDataType array_type(annotation_ref_type);
+    StructDataType city_data_type("name", 42);
+    city_data_type.addField(Field("references", 0, array_type, false));
+
+    StructFieldValue::UP city_data(new StructFieldValue(city_data_type));
+    ArrayFieldValue ref_list(array_type);
+    ref_list.add(AnnotationReferenceFieldValue(annotation_ref_type,
+                                               san_index));
+    ref_list.add(AnnotationReferenceFieldValue(annotation_ref_type,
+                                               fran_index));
+    city_data->setValue("references", ref_list);
+    StructFieldValue original(*city_data);
+
+    Annotation city(city_type, std::move(city_data));
+
+    ASSERT_TRUE(city.getFieldValue());
+    EXPECT_TRUE(city.getFieldValue()->isA(original));
+    EXPECT_EQUAL(original, *city.getFieldValue());
+}
+
+const double prob0 = 0.6;
+const double prob1 = 0.4;
+
+TEST("requireThatAlternateSpanListHoldsMultipleLists") {
+    AlternateSpanList span_list;
+    span_list.add(0, makeUP(new Span(0, 19)));
+    span_list.add(0, makeUP(new Span(19, 5)));
+    span_list.add(1, makeUP(new Span(0, 5)));
+    span_list.add(1, makeUP(new Span(5, 19)));
+    span_list.setProbability(0, prob0);
+    span_list.setProbability(1, prob1);
+
+    EXPECT_EQUAL(2u, span_list.getNumSubtrees());
+    EXPECT_EQUAL(2u, span_list.getSubtree(0).size());
+    EXPECT_EQUAL(2u, span_list.getSubtree(1).size());
+    EXPECT_EQUAL(prob0, span_list.getProbability(0));
+    EXPECT_EQUAL(prob1, span_list.getProbability(1));
+
+    SpanList::const_iterator it = span_list.getSubtree(0).begin();
+    EXPECT_EQUAL(Span(0, 19), *(static_cast<Span *>(*it++)));
+    EXPECT_EQUAL(Span(19, 5), *(static_cast<Span *>(*it++)));
+    EXPECT_TRUE(it == span_list.getSubtree(0).end());
+
+    it = span_list.getSubtree(1).begin();
+    EXPECT_EQUAL(Span(0, 5), *(static_cast<Span *>(*it++)));
+    EXPECT_EQUAL(Span(5, 19), *(static_cast<Span *>(*it++)));
+    EXPECT_TRUE(it == span_list.getSubtree(1).end());
+}
+
+struct MySpanTreeVisitor : SpanTreeVisitor {
+    int span_count;
+    int span_list_count;
+    int alt_span_list_count;
+
+    MySpanTreeVisitor()
+        : span_count(0), span_list_count(0), alt_span_list_count(0) {}
+
+    void visitChildren(const SpanList &node) {
+        for (SpanList::const_iterator
+                 it = node.begin(); it != node.end(); ++it) {
+            (*it)->accept(*this);
+        }
+    }
+
+    void visitChildren(const SimpleSpanList &node) {
+        for (SimpleSpanList::const_iterator
+                 it = node.begin(); it != node.end(); ++it) {
+            (*it).accept(*this);
+        }
+    }
+
+    void visit(const Span &) override { ++span_count; }
+    void visit(const SpanList &node) override {
+        ++span_list_count;
+        visitChildren(node);
+    }
+    void visit(const SimpleSpanList &node) override {
+        ++span_list_count;
+        visitChildren(node);
+    }
+    void visit(const AlternateSpanList &node) override {
+        ++alt_span_list_count;
+        for (size_t i = 0; i < node.getNumSubtrees(); ++i) {
+            visitChildren(node.getSubtree(i));
+        }
+    }
+};
+
+TEST("requireThatSpanTreeCanBeVisited") {
+    SpanList::UP root(new SpanList);
+    root->add(makeUP(new Span(0, 19)));
+    AlternateSpanList::UP alt_list(new AlternateSpanList);
+    alt_list->add(0, makeUP(new Span(19, 5)));
+    alt_list->add(1, makeUP(new Span(24, 21)));
+    root->add(std::move(alt_list));
+
+    SpanTree tree("html", std::move(root));
+
+    MySpanTreeVisitor visitor;
+    tree.accept(visitor);
+
+    EXPECT_EQUAL(3, visitor.span_count);
+    EXPECT_EQUAL(1, visitor.span_list_count);
+    EXPECT_EQUAL(1, visitor.alt_span_list_count);
+}
+
+TEST("requireThatDefaultAnnotationTypesHaveDefaultDataTypes") {
+    ASSERT_TRUE(AnnotationType::TERM->getDataType());
+    EXPECT_EQUAL(*DataType::STRING, *AnnotationType::TERM->getDataType());
+    ASSERT_TRUE(AnnotationType::TOKEN_TYPE->getDataType());
+    EXPECT_EQUAL(*DataType::INT, *AnnotationType::TOKEN_TYPE->getDataType());
+}
+
+TEST("require that SpanTrees can be compared") {
+    SpanList::UP root(new SpanList);
+    root->add(makeUP(new Span(0, 19)));
+    SpanTree tree1("html", std::move(root));
+
+    root.reset(new SpanList);
+    root->add(makeUP(new Span(0, 18)));
+    SpanTree tree2("html", std::move(root));
+
+    EXPECT_EQUAL(0, tree1.compare(tree1));
+    EXPECT_LESS(0, tree1.compare(tree2));
+    EXPECT_GREATER(0, tree2.compare(tree1));
+}
+
+}  // namespace
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/document/src/tests/arrayfieldvaluetest.cpp b/document/src/tests/arrayfieldvaluetest.cpp
new file mode 100644
index 00000000000..136129274b5
--- /dev/null
+++ b/document/src/tests/arrayfieldvaluetest.cpp
@@ -0,0 +1,247 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/document/fieldvalue/fieldvalues.h>
+#include <vespa/document/serialization/vespadocumentdeserializer.h>
+#include <vespa/vdstestlib/cppunit/macros.h>
+#include <vespa/vespalib/objects/nbostream.h>
+
+using vespalib::nbostream;
+
+namespace document {
+
+struct ArrayFieldValueTest : public CppUnit::TestFixture {
+    void setUp() {}
+    void tearDown() {}
+
+    void testArray();
+
+    CPPUNIT_TEST_SUITE(ArrayFieldValueTest);
+    CPPUNIT_TEST(testArray);
+    CPPUNIT_TEST_SUITE_END();
+
+};
+
+CPPUNIT_TEST_SUITE_REGISTRATION(ArrayFieldValueTest);
+
+namespace {
+template <typename T>
+void deserialize(const ByteBuffer &buffer, T &value) {
+    uint16_t version = Document::getNewestSerializationVersion();
+    nbostream stream(buffer.getBufferAtPos(), buffer.getRemaining());
+    DocumentTypeRepo repo;
+    VespaDocumentDeserializer deserializer(repo, stream, version);
+    deserializer.read(value);
+}
+}  // namespace
+
+void ArrayFieldValueTest::testArray()
+{
+    ArrayDataType type(*DataType::INT);
+    ArrayFieldValue value(type);
+
+        // Initially empty
+    CPPUNIT_ASSERT_EQUAL(size_t(0), value.size());
+    CPPUNIT_ASSERT(value.isEmpty());
+    CPPUNIT_ASSERT(!value.contains(IntFieldValue(1)));
+
+    CPPUNIT_ASSERT(value.add(IntFieldValue(1)));
+
+        // Not empty
+    CPPUNIT_ASSERT_EQUAL(size_t(1), value.size());
+    CPPUNIT_ASSERT(!value.isEmpty());
+    CPPUNIT_ASSERT(value.contains(IntFieldValue(1)));
+
+        // Adding some more
+    CPPUNIT_ASSERT(value.add(IntFieldValue(2)));
+    CPPUNIT_ASSERT(value.add(IntFieldValue(3)));
+
+        // Not empty
+    CPPUNIT_ASSERT_EQUAL(size_t(3), value.size());
+    CPPUNIT_ASSERT(!value.isEmpty());
+    CPPUNIT_ASSERT_EQUAL(IntFieldValue(1), (IntFieldValue&) value[0]);
+    CPPUNIT_ASSERT_EQUAL(IntFieldValue(2), (IntFieldValue&) value[1]);
+    CPPUNIT_ASSERT_EQUAL(IntFieldValue(3), (IntFieldValue&) value[2]);
+
+        // Serialize & equality
+    std::unique_ptr<ByteBuffer> buffer(value.serialize());
+    buffer->flip();
+    ArrayFieldValue value2(type);
+    CPPUNIT_ASSERT(value != value2);
+    deserialize(*buffer, value2);
+    CPPUNIT_ASSERT_EQUAL(value, value2);
+
+        // Various ways of removing
+    {
+            // By index
+        buffer->setPos(0);
+        deserialize(*buffer, value2);
+        value2.remove(1);
+        CPPUNIT_ASSERT(!value2.contains(IntFieldValue(2)));
+        CPPUNIT_ASSERT_EQUAL(size_t(2), value2.size());
+
+            // By value
+        buffer->setPos(0);
+        deserialize(*buffer, value2);
+        CPPUNIT_ASSERT(value2.remove(IntFieldValue(1)));
+        CPPUNIT_ASSERT(!value2.contains(IntFieldValue(1)));
+        CPPUNIT_ASSERT_EQUAL(size_t(2), value2.size());
+
+            // By value with multiple present
+        buffer->setPos(0);
+        deserialize(*buffer, value2);
+        value2.add(IntFieldValue(1));
+        CPPUNIT_ASSERT(value2.remove(IntFieldValue(1)));
+        CPPUNIT_ASSERT(!value2.contains(IntFieldValue(1)));
+        CPPUNIT_ASSERT_EQUAL(size_t(2), value2.size());
+
+            // Clearing all
+        buffer->setPos(0);
+        deserialize(*buffer, value2);
+        value2.clear();
+        CPPUNIT_ASSERT(!value2.contains(IntFieldValue(1)));
+        CPPUNIT_ASSERT_EQUAL(size_t(0), value2.size());
+        CPPUNIT_ASSERT(value2.isEmpty());
+    }
+
+        // Updating
+    value2 = value;
+    CPPUNIT_ASSERT_EQUAL(value, value2);
+    value2[1].assign(IntFieldValue(5));
+    CPPUNIT_ASSERT(!value2.contains(IntFieldValue(2)));
+    CPPUNIT_ASSERT_EQUAL(IntFieldValue(5), (IntFieldValue&) value2[1]);
+    CPPUNIT_ASSERT(value != value2);
+    value2.assign(value);
+    CPPUNIT_ASSERT_EQUAL(value, value2);
+    ArrayFieldValue::UP valuePtr(value2.clone());
+    CPPUNIT_ASSERT_EQUAL(value, *valuePtr);
+
+        // Iterating
+    const ArrayFieldValue& constVal(value);
+    for(ArrayFieldValue::const_iterator it = constVal.begin();
+        it != constVal.end(); ++it)
+    {
+        const FieldValue& fval1(*it);
+        (void) fval1;
+        CPPUNIT_ASSERT_EQUAL((uint32_t) IntFieldValue::classId,
+                             it->getClass().id());
+    }
+    value2 = value;
+    for(ArrayFieldValue::iterator it = value2.begin(); it != value2.end(); ++it)
+    {
+        (*it).assign(IntFieldValue(7));
+        it->assign(IntFieldValue(7));
+    }
+    CPPUNIT_ASSERT(value != value2);
+    CPPUNIT_ASSERT(value2.contains(IntFieldValue(7)));
+    CPPUNIT_ASSERT(value2.remove(IntFieldValue(7)));
+    CPPUNIT_ASSERT(value2.isEmpty());
+
+        // Comparison
+    value2 = value;
+    CPPUNIT_ASSERT_EQUAL(0, value.compare(value2));
+    value2.remove(1);
+    CPPUNIT_ASSERT(value.compare(value2) > 0);
+    CPPUNIT_ASSERT(value2.compare(value) < 0);
+    value2 = value;
+    value2[1].assign(IntFieldValue(5));
+    CPPUNIT_ASSERT(value.compare(value2) < 0);
+    CPPUNIT_ASSERT(value2.compare(value) > 0);
+
+        // Output
+    CPPUNIT_ASSERT_EQUAL(std::string("Array(size: 3,\n  1,\n  2,\n  3\n)"),
+                         value.toString(false));
+    CPPUNIT_ASSERT_EQUAL(std::string("Array(size: 3,\n.  1,\n.  2,\n.  3\n.)"),
+                         value.toString(true, "."));
+    CPPUNIT_ASSERT_EQUAL(std::string(
+                            "<value>\n"
+                            "  <item>1</item>\n"
+                            "  <item>2</item>\n"
+                            "  <item>3</item>\n"
+                            "</value>"
+                         ),
+                         value.toXml("  "));
+
+        // Failure situations.
+
+        // Refuse to accept non-array types
+    try{
+        ArrayFieldValue value6(*DataType::TAG);
+        CPPUNIT_FAIL("Didn't complain about non-array type");
+    } catch (std::exception& e) {
+        fprintf(stderr, "Exception = %s\n", e.what());
+        CPPUNIT_ASSERT_CONTAIN(
+                "Cannot generate an array value with non-array type",
+                e.what());
+    }
+
+        // Verify that datatypes are verified
+        // Created almost equal types to try to get it to fail
+    ArrayDataType type1(*DataType::INT);
+    ArrayDataType type2(*DataType::LONG);
+    ArrayDataType type3(static_cast<DataType &>(type1));
+    ArrayDataType type4(static_cast<DataType &>(type2));
+    ArrayFieldValue value3(type3);
+    ArrayFieldValue value4(type4);
+    try{
+        value3 = value4;
+        CPPUNIT_FAIL("Failed to check type equality in operator=");
+    } catch (std::exception& e) {
+        CPPUNIT_ASSERT_CONTAIN("Cannot assign value of type", e.what());
+    }
+    try{
+        value3.assign(value4);
+        CPPUNIT_FAIL("Failed to check type equality in assign()");
+    } catch (std::exception& e) {
+        CPPUNIT_ASSERT_CONTAIN("Cannot assign value of type", e.what());
+    }
+    try{
+        ArrayFieldValue subValue(type2);
+        subValue.add(LongFieldValue(4));
+        value3.add(subValue);
+        CPPUNIT_FAIL("Failed to check type equality in add()");
+    } catch (std::exception& e) {
+        CPPUNIT_ASSERT_CONTAIN("Cannot add value of type",e.what());
+    }
+    try{
+        ArrayFieldValue subValue(type2);
+        subValue.add(LongFieldValue(4));
+        value3.contains(subValue);
+        CPPUNIT_FAIL("Failed to check type equality in contains()");
+    } catch (std::exception& e) {
+        CPPUNIT_ASSERT_CONTAIN("can't possibly be in array of type", e.what());
+    }
+    try{
+        ArrayFieldValue subValue(type2);
+        subValue.add(LongFieldValue(4));
+        value3.remove(subValue);
+        CPPUNIT_FAIL("Failed to check type equality in remove()");
+    } catch (std::exception& e) {
+        CPPUNIT_ASSERT_CONTAIN("can't possibly be in array of type", e.what());
+    }
+        // Verify that compare sees difference
+    {
+        ArrayFieldValue subValue1(type1);
+        ArrayFieldValue subValue2(type2);
+        subValue1.add(IntFieldValue(3));
+        subValue2.add(LongFieldValue(3));
+        value3.clear();
+        value4.clear();
+        value3.add(subValue1);
+        value4.add(subValue2);
+        CPPUNIT_ASSERT(value3.compare(value4) != 0);
+    }
+
+        // Removing non-existing element
+    value2 = value;
+    try{
+        value2.remove(5);
+        CPPUNIT_FAIL("Failed to throw out of bounds exception on remove(int)");
+    } catch (std::exception& e) {
+        CPPUNIT_ASSERT_CONTAIN("Cannot remove index 5 from an array of size 3",
+                               e.what());
+    }
+    CPPUNIT_ASSERT(!value2.remove(IntFieldValue(15)));
+}
+
+} // document
diff --git a/document/src/tests/arrayfieldvaluetest.h b/document/src/tests/arrayfieldvaluetest.h
new file mode 100644
index 00000000000..8118d57021a
--- /dev/null
+++ b/document/src/tests/arrayfieldvaluetest.h
@@ -0,0 +1,29 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/* $Id$*/
+
+#pragma once
+
+#include <cppunit/extensions/HelperMacros.h>
+
+class ArrayFieldValue_Test : public CppUnit::TestFixture {
+    CPPUNIT_TEST_SUITE( ArrayFieldValue_Test);
+    CPPUNIT_TEST(testArray);
+    CPPUNIT_TEST(testArray2);
+    CPPUNIT_TEST(testArrayRaw);
+    CPPUNIT_TEST(testTextSerialize);
+    CPPUNIT_TEST(testArrayRemove);
+    CPPUNIT_TEST_SUITE_END();
+
+public:
+    void setUp();
+
+    void tearDown();
+protected:
+    void testArray();
+    void testArray2();
+    void testArrayRaw();
+    void testTextSerialize();
+    void testArrayRemove();
+};
+
+
diff --git a/document/src/tests/base/.gitignore b/document/src/tests/base/.gitignore
new file mode 100644
index 00000000000..bdb07eae1cb
--- /dev/null
+++ b/document/src/tests/base/.gitignore
@@ -0,0 +1,7 @@
+*.So
+*_test
+*_benchmark
+.depend
+Makefile
+document_documentid_test_app
+document_documentid_benchmark_app
diff --git a/document/src/tests/base/CMakeLists.txt b/document/src/tests/base/CMakeLists.txt
new file mode 100644
index 00000000000..550d77d1340
--- /dev/null
+++ b/document/src/tests/base/CMakeLists.txt
@@ -0,0 +1,19 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(document_documentid_test_app
+    SOURCES
+    documentid_test.cpp
+    DEPENDS
+    document
+    AFTER
+    document_documentconfig
+)
+vespa_add_test(NAME document_documentid_test_app COMMAND document_documentid_test_app)
+vespa_add_executable(document_documentid_benchmark_app
+    SOURCES
+    documentid_benchmark.cpp
+    DEPENDS
+    document
+    AFTER
+    document_documentconfig
+)
+vespa_add_test(NAME document_documentid_benchmark_app COMMAND document_documentid_benchmark_app BENCHMARK)
diff --git a/document/src/tests/base/documentid_benchmark.cpp b/document/src/tests/base/documentid_benchmark.cpp
new file mode 100644
index 00000000000..16f9c9016d7
--- /dev/null
+++ b/document/src/tests/base/documentid_benchmark.cpp
@@ -0,0 +1,21 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/document/base/documentid.h>
+
+using namespace document;
+
+int main(int argc, char *argv[])
+{
+    if (argc < 3) {
+        fprintf(stderr, "Usage %s <docid> <count>\n", argv[0]);
+    }
+    vespalib::string s(argv[1]);
+    uint64_t n = strtoul(argv[2], NULL, 0);
+    printf("Creating documentid '%s' %ld times\n", s.c_str(), n);
+    printf("sizeof(IdString)=%ld,  sizeof(IdIdString)=%ld\n", sizeof(IdString), sizeof(IdIdString));
+    for (uint64_t i=0; i < n; i++) {
+        IdString::UP id = IdString::createIdString(s);
+        id.reset();
+    }
+    return 0;
+}
+
diff --git a/document/src/tests/base/documentid_test.cpp b/document/src/tests/base/documentid_test.cpp
new file mode 100644
index 00000000000..62ce42c0fb6
--- /dev/null
+++ b/document/src/tests/base/documentid_test.cpp
@@ -0,0 +1,228 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Unit tests for documentid.
+
+#include <vespa/log/log.h>
+LOG_SETUP("documentid_test");
+#include <vespa/fastos/fastos.h>
+
+#include <vespa/document/base/documentid.h>
+#include <vespa/vespalib/testkit/testapp.h>
+
+using namespace document;
+using vespalib::string;
+
+namespace {
+
+const string ns = "namespace";
+const string ns_id = "namespaceid";
+const string type = "my_type";
+
+void checkId(const string &id, IdString::Type t,
+             const string &ns_str, const string local_id) {
+    DocumentId doc_id(id);
+    EXPECT_EQUAL(t, doc_id.getScheme().getType());
+    EXPECT_EQUAL(ns_str, doc_id.getScheme().getNamespace());
+    EXPECT_EQUAL(local_id, doc_id.getScheme().getNamespaceSpecific());
+    EXPECT_EQUAL(id, doc_id.getScheme().toString());
+}
+
+template <typename IdType>
+const IdType &getAs(const DocumentId &doc_id) {
+    const IdType *id_str = dynamic_cast<const IdType *>(&doc_id.getScheme());
+    ASSERT_TRUE(id_str);
+    return *id_str;
+}
+
+template <typename IdType>
+void checkUser(const string &id, int64_t user_id) {
+    DocumentId doc_id(id);
+    EXPECT_EQUAL(user_id, getAs<IdType>(doc_id).getUserId());
+}
+
+void checkOrdering(const string &id, uint64_t ordering) {
+    DocumentId doc_id(id);
+    EXPECT_EQUAL(ordering, getAs<OrderDocIdString>(doc_id).getOrdering());
+}
+
+void checkGroup(const string &id, const string &group) {
+    DocumentId doc_id(id);
+    EXPECT_EQUAL(group, getAs<GroupDocIdString>(doc_id).getGroup());
+}
+
+void checkType(const string &id, const string &doc_type) {
+    DocumentId doc_id(id);
+    ASSERT_TRUE(doc_id.hasDocType());
+    EXPECT_EQUAL(doc_type, doc_id.getDocType());
+}
+
+TEST("require that doc id can be parsed") {
+    const string id = "doc:" + ns + ":" + ns_id;
+    checkId(id, IdString::DOC, ns, ns_id);
+}
+
+TEST("require that userdoc id can be parsed") {
+    const string id = "userdoc:" + ns + ":1234:" + ns_id;
+    checkId(id, IdString::USERDOC, ns, ns_id);
+    checkUser<UserDocIdString>(id, 1234);
+}
+
+TEST("require that orderdoc id can be parsed") {
+    const string id = "orderdoc(31,19):" + ns + ":1234:1268182861:" + ns_id;
+    checkId(id, IdString::ORDERDOC, ns, ns_id);
+    checkUser<OrderDocIdString>(id, 1234);
+    checkOrdering(id, 1268182861);
+}
+
+TEST("require that groupdoc id can be parsed") {
+    const string group = "mygroup";
+    const string id = "groupdoc:" + ns + ":" + group + ":" + ns_id;
+    checkId(id, IdString::GROUPDOC, ns, ns_id);
+    checkGroup(id, group);
+}
+
+TEST("require that id id can be parsed") {
+    const string id = "id:" + ns + ":" + type + "::" + ns_id;
+    checkId(id, IdString::ID, ns, ns_id);
+    checkType(id, type);
+}
+
+TEST("require that we allow ':' in namespace specific part") {
+    const string nss=":a:b:c:";
+    string id="id:" + ns + ":" + type + "::" + nss;
+    checkId(id, IdString::ID, ns, nss);
+    checkType(id, type);
+
+    id="doc:" + ns + ":" + nss;
+    checkId(id, IdString::DOC, ns, nss);
+
+    id="userdoc:" + ns + ":123:" + nss;
+    checkId(id, IdString::USERDOC, ns, nss);
+    checkUser<UserDocIdString>(id, 123);
+
+    id="groupdoc:" + ns + ":123:" + nss;
+    checkId(id, IdString::GROUPDOC, ns, nss);
+    checkGroup(id, "123");
+
+    id="orderdoc(5,2):" + ns + ":42:007:" + nss;
+    checkId(id, IdString::ORDERDOC, ns, nss);
+}
+
+TEST("require that id id can specify location") {
+    DocumentId id("id:ns:type:n=12345:foo");
+    EXPECT_EQUAL(12345u, id.getScheme().getLocation());
+    EXPECT_EQUAL(12345u, getAs<IdIdString>(id).getNumber());
+}
+
+TEST("require that id id's n key must be a 64-bit number") {
+    EXPECT_EXCEPTION(DocumentId("id:ns:type:n=abc:foo"), IdParseException,
+                     "'n'-value must be a 64-bit number");
+    DocumentId("id:ns:type:n=18446744073709551615:foo");  // ok
+    EXPECT_EXCEPTION(DocumentId("id:ns:type:n=18446744073709551616:foo"),
+                     IdParseException, "'n'-value out of range");
+}
+
+TEST("require that id id can specify group") {
+    DocumentId id1("id:ns:type:g=mygroup:foo");
+    DocumentId id2("id:ns:type:g=mygroup:bar");
+    DocumentId id3("id:ns:type:g=other group:baz");
+    EXPECT_EQUAL(id1.getScheme().getLocation(), id2.getScheme().getLocation());
+    EXPECT_NOT_EQUAL(id1.getScheme().getLocation(),
+                     id3.getScheme().getLocation());
+    EXPECT_EQUAL("mygroup", getAs<IdIdString>(id1).getGroup());
+}
+
+TEST("require that id id location is specified by local id only by default") {
+    DocumentId id1("id:ns:type::locationspec");
+    DocumentId id2("id:ns:type:g=locationspec:bar");
+    EXPECT_EQUAL("locationspec", id1.getScheme().getNamespaceSpecific());
+    EXPECT_EQUAL("bar", id2.getScheme().getNamespaceSpecific());
+    EXPECT_EQUAL(id1.getScheme().getLocation(), id2.getScheme().getLocation());
+}
+
+TEST("require that local id can be empty") {
+    const string id = "userdoc:" + ns + ":1234:";
+    checkId(id, IdString::USERDOC, ns, "");
+    checkUser<UserDocIdString>(id, 1234);
+}
+
+TEST("require that document ids can be assigned") {
+    DocumentId id1("userdoc:" + ns + ":1234:");
+    DocumentId id2 = id1;
+    checkId(id2.toString(), IdString::USERDOC, ns, "");
+    checkUser<UserDocIdString>(id2.toString(), 1234);
+}
+
+TEST("require that illegal ids fail") {
+    EXPECT_EXCEPTION(DocumentId("idg:foo:bar:baz"), IdParseException,
+                     "No scheme separator ':' found");
+    EXPECT_EXCEPTION(DocumentId("id:"), IdParseException, "too short");
+    EXPECT_EXCEPTION(DocumentId("id:ns"), IdParseException,
+                     "No namespace separator ':' found");
+    EXPECT_EXCEPTION(DocumentId("id:ns:type"), IdParseException,
+                     "No document type separator ':' found");
+    EXPECT_EXCEPTION(DocumentId("id:ns:type:kv_pair"), IdParseException,
+                     "No key/value-pairs separator ':' found");
+    EXPECT_EXCEPTION(DocumentId("id:ns:type:k=foo:bar"), IdParseException,
+                     "Illegal key 'k'");
+    EXPECT_EXCEPTION(DocumentId("id:ns:type:n=0,n=1:bar"), IdParseException,
+                     "Illegal key combination in n=0,n=1");
+    EXPECT_EXCEPTION(DocumentId("id:ns:type:g=foo,g=ba:bar"), IdParseException,
+                     "Illegal key combination in g=foo,g=ba");
+    EXPECT_EXCEPTION(DocumentId("id:ns:type:n=0,g=foo:bar"), IdParseException,
+                     "Illegal key combination in n=0,g=foo");
+}
+
+TEST("require that key-value pairs in id id are preserved") {
+    const string id_str1 = "id:ns:type:n=1:foo";
+    EXPECT_EQUAL(id_str1, DocumentId(id_str1).toString());
+
+    const string id_str2 = "id:ns:type:g=mygroup:foo";
+    EXPECT_EQUAL(id_str2, DocumentId(id_str2).toString());
+}
+
+TEST("require that id strings reports features (hasNumber, hasGroup)") {
+    DocumentId none("id:ns:type::foo");
+    EXPECT_FALSE(none.getScheme().hasNumber());
+    EXPECT_FALSE(none.getScheme().hasGroup());
+    EXPECT_EQUAL("foo", none.getScheme().getNamespaceSpecific());
+
+    none = DocumentId("doc:ns:foo");
+    EXPECT_FALSE(none.getScheme().hasNumber());
+    EXPECT_FALSE(none.getScheme().hasGroup());
+    EXPECT_EQUAL("foo", none.getScheme().getNamespaceSpecific());
+
+    DocumentId user("id:ns:type:n=42:foo");
+    EXPECT_TRUE(user.getScheme().hasNumber());
+    EXPECT_FALSE(user.getScheme().hasGroup());
+    EXPECT_EQUAL(42u, user.getScheme().getNumber());
+    EXPECT_EQUAL("foo", user.getScheme().getNamespaceSpecific());
+
+    user = DocumentId("userdoc:ns:42:foo");
+    EXPECT_TRUE(user.getScheme().hasNumber());
+    EXPECT_FALSE(user.getScheme().hasGroup());
+    EXPECT_EQUAL(42u, user.getScheme().getNumber());
+    EXPECT_EQUAL("foo", user.getScheme().getNamespaceSpecific());
+
+    DocumentId group("id:ns:type:g=mygroup:foo");
+    EXPECT_FALSE(group.getScheme().hasNumber());
+    EXPECT_TRUE(group.getScheme().hasGroup());
+    EXPECT_EQUAL("mygroup", group.getScheme().getGroup());
+    EXPECT_EQUAL("foo", group.getScheme().getNamespaceSpecific());
+
+    group = DocumentId("groupdoc:ns:mygroup:foo");
+    EXPECT_FALSE(group.getScheme().hasNumber());
+    EXPECT_TRUE(group.getScheme().hasGroup());
+    EXPECT_EQUAL("mygroup", group.getScheme().getGroup());
+    EXPECT_EQUAL("foo", group.getScheme().getNamespaceSpecific());
+
+    DocumentId order("orderdoc(5,2):ns:42:007:foo");
+    EXPECT_TRUE(order.getScheme().hasNumber());
+    EXPECT_TRUE(order.getScheme().hasGroup());
+    EXPECT_EQUAL(42u, order.getScheme().getNumber());
+    EXPECT_EQUAL("42", order.getScheme().getGroup());
+    EXPECT_EQUAL("foo", order.getScheme().getNamespaceSpecific());
+}
+
+}  // namespace
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/document/src/tests/bucketselectortest.cpp b/document/src/tests/bucketselectortest.cpp
new file mode 100644
index 00000000000..f04816cba0d
--- /dev/null
+++ b/document/src/tests/bucketselectortest.cpp
@@ -0,0 +1,102 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/document/bucket/bucketselector.h>
+
+#include <vespa/document/bucket/bucketid.h>
+#include <vespa/vdstestlib/cppunit/macros.h>
+#include <vespa/document/select/parser.h>
+#include <vespa/document/base/testdocrepo.h>
+#include <memory>
+
+using document::select::Node;
+using document::select::Parser;
+
+namespace document {
+
+struct BucketSelectorTest : public CppUnit::TestFixture {
+    void testSimple();
+
+    CPPUNIT_TEST_SUITE(BucketSelectorTest);
+    CPPUNIT_TEST(testSimple);
+    CPPUNIT_TEST_SUITE_END();
+};
+
+CPPUNIT_TEST_SUITE_REGISTRATION(BucketSelectorTest);
+
+#define ASSERT_BUCKET_COUNT(expression, count) \
+{ \
+    TestDocRepo testRepo; \
+    BucketIdFactory idfactory; \
+    BucketSelector selector(idfactory); \
+    Parser parser(testRepo.getTypeRepo(), idfactory); \
+    std::unique_ptr<Node> node(parser.parse(expression)); \
+    CPPUNIT_ASSERT(node.get() != 0); \
+    std::unique_ptr<BucketSelector::BucketVector> buckets( \
+            selector.select(*node)); \
+    size_t bcount(buckets.get() ? buckets->size() : 0); \
+    std::ostringstream ost; \
+    ost << "Expression " << expression << " did not contain " << count \
+        << " buckets as expected"; \
+    CPPUNIT_ASSERT_EQUAL_MSG(ost.str(), (size_t) count, bcount); \
+}
+
+#define ASSERT_BUCKET(expression, bucket) \
+{ \
+    TestDocRepo testRepo; \
+    BucketIdFactory idfactory; \
+    BucketSelector selector(idfactory); \
+    Parser parser(testRepo.getTypeRepo(), idfactory); \
+    std::unique_ptr<Node> node(parser.parse(expression)); \
+    CPPUNIT_ASSERT(node.get() != 0); \
+    std::unique_ptr<BucketSelector::BucketVector> buckets( \
+            selector.select(*node)); \
+    std::ostringstream ost; \
+    ost << "Expression " << expression << " did not contain bucket " \
+        << bucket.toString(); \
+    if (buckets.get()) { \
+        ost << ". Buckets: " << std::hex << *buckets; \
+    } else { \
+        ost << ". Matches all buckets"; \
+    } \
+    CPPUNIT_ASSERT_MSG(ost.str(), buckets.get() && \
+                       std::find(buckets->begin(), buckets->end(), \
+                       bucket) != buckets->end()); \
+}
+
+void BucketSelectorTest::testSimple()
+{
+    ASSERT_BUCKET_COUNT("id = \"userdoc:ns:123:foobar\"", 1u);
+    ASSERT_BUCKET_COUNT("id = \"userdoc:ns:123:foo*\"", 0u);
+    ASSERT_BUCKET_COUNT("id == \"userdoc:ns:123:f?oo*\"", 1u);
+    ASSERT_BUCKET_COUNT("id =~ \"userdoc:ns:123:foo*\"", 0u);
+    ASSERT_BUCKET_COUNT("id =~ \"userdoc:ns:123:foo?\"", 0u);
+    ASSERT_BUCKET_COUNT("id.user = 123", 1u);
+    ASSERT_BUCKET_COUNT("id.user == 123", 1u);
+    ASSERT_BUCKET_COUNT("id.group = \"yahoo.com\"", 1u);
+    ASSERT_BUCKET_COUNT("id.group = \"yahoo.com\" or id.user=123", 2u);
+    ASSERT_BUCKET_COUNT("id.group = \"yahoo.com\" and id.user=123", 0u);
+    ASSERT_BUCKET_COUNT(
+            "id.group = \"yahoo.com\" and testdoctype1.hstringval=\"Doe\"", 1u);
+    ASSERT_BUCKET_COUNT("not id.group = \"yahoo.com\"", 0u);
+    ASSERT_BUCKET_COUNT("id.group != \"yahoo.com\"", 0u);
+    ASSERT_BUCKET_COUNT("id.group <= \"yahoo.com\"", 0u);
+
+    ASSERT_BUCKET_COUNT("id.bucket = 0x4000000000003018", 1u); // Bucket 12312
+    ASSERT_BUCKET_COUNT("id.bucket == 0x4000000000000258", 1u); // Bucket 600
+    ASSERT_BUCKET_COUNT("(testdoctype1 and id.bucket=0)", 1u);
+
+    ASSERT_BUCKET_COUNT("searchcolumn.3 = 1", 21845u);
+
+        // Check that the correct buckets is found
+    ASSERT_BUCKET("id = \"userdoc:ns:123:foobar\"",
+                  document::BucketId(58, 123));
+
+    ASSERT_BUCKET("id.bucket == 0x4000000000000258", document::BucketId(16, 600));
+
+    ASSERT_BUCKET("id.user = 123", document::BucketId(32, 123));
+    ASSERT_BUCKET("id.group = \"yahoo.com\"",
+                  document::BucketId(32, 0x9a1acd50));
+}
+
+} // document
diff --git a/document/src/tests/buckettest.cpp b/document/src/tests/buckettest.cpp
new file mode 100644
index 00000000000..2f2f3df9ab6
--- /dev/null
+++ b/document/src/tests/buckettest.cpp
@@ -0,0 +1,258 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <cppunit/extensions/HelperMacros.h>
+#include <vespa/document/bucket/bucketidfactory.h>
+#include <vespa/document/base/documentid.h>
+#include <vespa/vespalib/util/random.h>
+
+namespace document {
+
+class BucketTest : public CppUnit::TestFixture {
+    CPPUNIT_TEST_SUITE(BucketTest);
+    CPPUNIT_TEST(testBucket);
+    CPPUNIT_TEST(testBucketGeneration);
+    CPPUNIT_TEST(testBucketSerialization);
+    CPPUNIT_TEST(testReverseBucket);
+    CPPUNIT_TEST(testContains);
+    CPPUNIT_TEST(testGetBit);
+    CPPUNIT_TEST_SUITE_END();
+
+public:
+    void testBucket();
+    void testBucketGeneration();
+    void testBucketSerialization();
+    void testReverseBucket();
+    void testContains();
+    void testGetBit();
+};
+
+CPPUNIT_TEST_SUITE_REGISTRATION(BucketTest);
+
+struct Hex {
+    BucketId::Type val;
+
+    Hex(BucketId::Type v) : val(v) {}
+    bool operator==(const Hex& h) const { return val == h.val; }
+};
+
+inline std::ostream& operator<<(std::ostream& out, const Hex& h) {
+    out << std::hex << h.val << std::dec;
+    return out;
+}
+
+void BucketTest::testBucket()
+{
+    // Test empty (invalid) buckets
+    BucketId id1;
+    BucketId id2;
+    CPPUNIT_ASSERT_EQUAL(id1, id2);
+    CPPUNIT_ASSERT(!(id1 < id2) && !(id2 < id1));
+    CPPUNIT_ASSERT_EQUAL(Hex(0), Hex(id1.getId()));
+    CPPUNIT_ASSERT_EQUAL(Hex(0), Hex(id1.getRawId()));
+    CPPUNIT_ASSERT_EQUAL(vespalib::string("BucketId(0x0000000000000000)"),
+                         id1.toString());
+    CPPUNIT_ASSERT_EQUAL(0u, id1.getUsedBits());
+
+    // Test bucket with a value
+    id2 = BucketId((BucketId::Type(16) << 58) | 0x123);
+    CPPUNIT_ASSERT(id1 != id2);
+    CPPUNIT_ASSERT((id1 < id2) && !(id2 < id1));
+    CPPUNIT_ASSERT_EQUAL(Hex(0x4000000000000123ull), Hex(id2.getId()));
+    CPPUNIT_ASSERT_EQUAL(Hex(0x4000000000000123ull), Hex(id2.getRawId()));
+    CPPUNIT_ASSERT_EQUAL(vespalib::string("BucketId(0x4000000000000123)"),
+                         id2.toString());
+    CPPUNIT_ASSERT_EQUAL(16u, id2.getUsedBits());
+
+    // Test copy constructor and operator=
+    BucketId id3(id2);
+    CPPUNIT_ASSERT_EQUAL(id2, id3);
+    id3 = id1;
+    CPPUNIT_ASSERT(!(id2 == id3));
+    id3 = id2;
+    CPPUNIT_ASSERT_EQUAL(id2, id3);
+}
+
+void BucketTest::testGetBit()
+{
+    for (uint32_t i = 0; i < 58; ++i) {
+        CPPUNIT_ASSERT_EQUAL(0, (int)document::BucketId(16, 0).getBit(i));
+    }
+
+    for (uint32_t i = 0; i < 4; ++i) {
+        CPPUNIT_ASSERT_EQUAL(0, (int)document::BucketId(16, 16).getBit(i));
+    }
+
+    CPPUNIT_ASSERT_EQUAL(1, (int)document::BucketId(16, 16).getBit(4));
+
+    for (uint32_t i = 5; i < 59; ++i) {
+        CPPUNIT_ASSERT_EQUAL(0, (int)document::BucketId(16, 16).getBit(i));
+    }
+
+    CPPUNIT_ASSERT_EQUAL(0, (int)document::BucketId(17, 0x0ffff).getBit(16));
+
+    for (uint32_t i = 0; i < 16; ++i) {
+        CPPUNIT_ASSERT_EQUAL(1, (int)document::BucketId(17, 0x0ffff).getBit(i));
+    }
+}
+
+void BucketTest::testBucketGeneration()
+{
+    BucketIdFactory factory;
+    DocumentId doc1("doc:ns:spec");
+    DocumentId doc2("doc:ns2:spec");
+    DocumentId doc3("doc:ns:spec2");
+    DocumentId userDoc1("userdoc:ns:18:spec");
+    DocumentId userDoc2("userdoc:ns2:18:spec2");
+    DocumentId userDoc3("userdoc:ns:19:spec");
+    DocumentId groupDoc1("groupdoc:ns:yahoo.com:spec");
+    DocumentId groupDoc2("groupdoc:ns2:yahoo.com:spec2");
+    DocumentId groupDoc3("groupdoc:ns:yahoo:spec");
+    DocumentId orderDoc1("orderdoc(31,19):ns:13:1268182861:foo");
+    DocumentId orderDoc2("orderdoc(31,19):ns:13:1205110861:foo");
+    DocumentId orderDoc3("orderdoc(31,19):ns:13:1205715661:foo");
+    DocumentId orderDoc4("orderdoc(4,0):ns:13:2:foo");
+    DocumentId orderDoc5("orderdoc(4,0):ns:13:4:foo");
+    DocumentId orderDoc6("orderdoc(4,0):ns:13:11:foo");
+
+    BucketId docBucket1(factory.getBucketId(doc1));
+    BucketId docBucket2(factory.getBucketId(doc2));
+    BucketId docBucket3(factory.getBucketId(doc3));
+    BucketId userDocBucket1(factory.getBucketId(userDoc1));
+    BucketId userDocBucket2(factory.getBucketId(userDoc2));
+    BucketId userDocBucket3(factory.getBucketId(userDoc3));
+    BucketId groupDocBucket1(factory.getBucketId(groupDoc1));
+    BucketId groupDocBucket2(factory.getBucketId(groupDoc2));
+    BucketId groupDocBucket3(factory.getBucketId(groupDoc3));
+    BucketId orderDocBucket1(factory.getBucketId(orderDoc1));
+    BucketId orderDocBucket2(factory.getBucketId(orderDoc2));
+    BucketId orderDocBucket3(factory.getBucketId(orderDoc3));
+    BucketId orderDocBucket4(factory.getBucketId(orderDoc4));
+    BucketId orderDocBucket5(factory.getBucketId(orderDoc5));
+    BucketId orderDocBucket6(factory.getBucketId(orderDoc6));
+
+    CPPUNIT_ASSERT_EQUAL(Hex(0xe99703f200000012ull), Hex(userDocBucket1.getRawId()));
+    CPPUNIT_ASSERT_EQUAL(Hex(0xebfa518a00000012ull), Hex(userDocBucket2.getRawId()));
+    CPPUNIT_ASSERT_EQUAL(Hex(0xeac1850800000013ull), Hex(userDocBucket3.getRawId()));
+
+    CPPUNIT_ASSERT_EQUAL(Hex(0xeae764e90000000dull), Hex(orderDocBucket1.getRawId()));
+    CPPUNIT_ASSERT_EQUAL(Hex(0xeacb85f10000000dull), Hex(orderDocBucket2.getRawId()));
+    CPPUNIT_ASSERT_EQUAL(Hex(0xea68ddf10000000dull), Hex(orderDocBucket3.getRawId()));
+
+    CPPUNIT_ASSERT_EQUAL(Hex(0xe87526540000000dull), Hex(orderDocBucket4.getRawId()));
+    CPPUNIT_ASSERT_EQUAL(Hex(0xea59f8f20000000dull), Hex(orderDocBucket5.getRawId()));
+    CPPUNIT_ASSERT_EQUAL(Hex(0xe9eb703d0000000dull), Hex(orderDocBucket6.getRawId()));
+
+    userDocBucket1.setUsedBits(16);
+    CPPUNIT_ASSERT_EQUAL(Hex(0x4000000000000012ull), Hex(userDocBucket1.getId()));
+    userDocBucket2.setUsedBits(16);
+    CPPUNIT_ASSERT_EQUAL(Hex(0x4000000000000012ull), Hex(userDocBucket2.getId()));
+    userDocBucket3.setUsedBits(16);
+    CPPUNIT_ASSERT_EQUAL(Hex(0x4000000000000013ull), Hex(userDocBucket3.getId()));
+
+    CPPUNIT_ASSERT_EQUAL(Hex(0xe90ce4b09a1acd50ull), Hex(groupDocBucket1.getRawId()));
+    CPPUNIT_ASSERT_EQUAL(Hex(0xe9cedaa49a1acd50ull), Hex(groupDocBucket2.getRawId()));
+    CPPUNIT_ASSERT_EQUAL(Hex(0xe8cdb18bafe81f24ull), Hex(groupDocBucket3.getRawId()));
+
+    groupDocBucket1.setUsedBits(16);
+    CPPUNIT_ASSERT_EQUAL(Hex(0x400000000000cd50ull), Hex(groupDocBucket1.getId()));
+    groupDocBucket2.setUsedBits(16);
+    CPPUNIT_ASSERT_EQUAL(Hex(0x400000000000cd50ull), Hex(groupDocBucket2.getId()));
+    groupDocBucket3.setUsedBits(16);
+    CPPUNIT_ASSERT_EQUAL(Hex(0x4000000000001f24ull), Hex(groupDocBucket3.getId()));
+
+    CPPUNIT_ASSERT_EQUAL(Hex(0xe980c9abd5fd8d11ull), Hex(docBucket1.getRawId()));
+    CPPUNIT_ASSERT_EQUAL(Hex(0xeafe870c5f9c37b9ull), Hex(docBucket2.getRawId()));
+    CPPUNIT_ASSERT_EQUAL(Hex(0xeaebe9473ecbcd69ull), Hex(docBucket3.getRawId()));
+
+    docBucket1.setUsedBits(16);
+    CPPUNIT_ASSERT_EQUAL(Hex(0x4000000000008d11ull), Hex(docBucket1.getId()));
+    docBucket2.setUsedBits(16);
+    CPPUNIT_ASSERT_EQUAL(Hex(0x40000000000037b9ull), Hex(docBucket2.getId()));
+    docBucket3.setUsedBits(16);
+    CPPUNIT_ASSERT_EQUAL(Hex(0x400000000000cd69ull), Hex(docBucket3.getId()));
+}
+
+void BucketTest::testBucketSerialization()
+{
+    BucketIdFactory factory;
+    DocumentId doc(DocIdString("ns", "spec"));
+    BucketId bucket(factory.getBucketId(doc));
+
+    std::ostringstream ost;
+    ost << bucket.getRawId();
+    CPPUNIT_ASSERT_EQUAL(std::string("16825669947722927377"),
+                         ost.str());
+
+    BucketId::Type id;
+    std::istringstream ist(ost.str());
+    ist >> id;
+    BucketId bucket2(id);
+
+    CPPUNIT_ASSERT_EQUAL(bucket, bucket2);
+}
+
+void BucketTest::testReverseBucket()
+{
+    {
+        BucketId id(0x3000000000000012ull);
+        CPPUNIT_ASSERT_EQUAL(Hex(0x480000000000000cull), Hex(id.toKey()));
+        CPPUNIT_ASSERT_EQUAL(Hex(id.getId()), Hex(document::BucketId::keyToBucketId(id.stripUnused().toKey())));
+    }
+
+    {
+        BucketId id(0x4000000000000012ull);
+        CPPUNIT_ASSERT_EQUAL(Hex(0x4800000000000010ull), Hex(id.toKey()));
+        CPPUNIT_ASSERT_EQUAL(Hex(id.getId()), Hex(document::BucketId::keyToBucketId(id.stripUnused().toKey())));
+    }
+
+    {
+        BucketId id(0x600000000000ffffull);
+        CPPUNIT_ASSERT_EQUAL(Hex(0xffff000000000018ull), Hex(id.toKey()));
+        CPPUNIT_ASSERT_EQUAL(Hex(id.getId()), Hex(document::BucketId::keyToBucketId(id.stripUnused().toKey())));
+    }
+
+    {
+        BucketId id(0x540000000001ffffull);
+        CPPUNIT_ASSERT_EQUAL(Hex(0xffff800000000015ull), Hex(id.toKey()));
+        CPPUNIT_ASSERT_EQUAL(Hex(id.getId()), Hex(document::BucketId::keyToBucketId(id.stripUnused().toKey())));
+    }
+
+    {
+        BucketId id(0xa80000000003ffffull);
+        CPPUNIT_ASSERT_EQUAL(Hex(0xffffc0000000002aull), Hex(id.toKey()));
+        CPPUNIT_ASSERT_EQUAL(Hex(id.getId()), Hex(document::BucketId::keyToBucketId(id.stripUnused().toKey())));
+    }
+
+    {
+        BucketId id(0xbc0000000007ffffull);
+        CPPUNIT_ASSERT_EQUAL(Hex(0xffffe0000000002full), Hex(id.toKey()));
+        CPPUNIT_ASSERT_EQUAL(Hex(id.getId()), Hex(document::BucketId::keyToBucketId(id.stripUnused().toKey())));
+    }
+    {
+        BucketId id(0xcc0000000002ffffull);
+        CPPUNIT_ASSERT_EQUAL(Hex(0xffff400000000033ull), Hex(id.toKey()));
+        CPPUNIT_ASSERT_EQUAL(Hex(id.getId()), Hex(document::BucketId::keyToBucketId(id.stripUnused().toKey())));
+    }
+    {
+        BucketId id(0xebffffffffffffffull);
+        CPPUNIT_ASSERT_EQUAL(Hex(0xfffffffffffffffaull), Hex(id.toKey()));
+        CPPUNIT_ASSERT_EQUAL(Hex(id.getId()), Hex(document::BucketId::keyToBucketId(id.stripUnused().toKey())));
+    }
+    {
+        BucketId id(0xeaaaaaaaaaaaaaaaull);
+        CPPUNIT_ASSERT_EQUAL(Hex(0x555555555555557aull), Hex(id.toKey()));
+        CPPUNIT_ASSERT_EQUAL(Hex(id.getId()), Hex(document::BucketId::keyToBucketId(id.stripUnused().toKey())));
+    }
+}
+
+void BucketTest::testContains() {
+    BucketId id(18, 0x123456789ULL);
+    CPPUNIT_ASSERT(id.contains(BucketId(20, 0x123456789ULL)));
+    CPPUNIT_ASSERT(id.contains(BucketId(18, 0x888f56789ULL)));
+    CPPUNIT_ASSERT(id.contains(BucketId(24, 0x888456789ULL)));
+    CPPUNIT_ASSERT(!id.contains(BucketId(24, 0x888886789ULL)));
+    CPPUNIT_ASSERT(!id.contains(BucketId(16, 0x123456789ULL)));
+}
+
+} // document
diff --git a/document/src/tests/cpp-globalidbucketids.txt b/document/src/tests/cpp-globalidbucketids.txt
new file mode 100644
index 00000000000..33dca454d3b
--- /dev/null
+++ b/document/src/tests/cpp-globalidbucketids.txt
@@ -0,0 +1,38 @@
+doc:ns:specific - gid(0x2c01a21163cb7d0ce85fddd6) - BucketId(0xeadd5fe811a2012c)
+doc:another:specific - gid(0xcd2ba528d1135e40605ce372) - BucketId(0xeae35c6028a52bcd)
+doc:ns:another - gid(0x1d5324270601e76a7a1f58b7) - BucketId(0xeb581f7a2724531d)
+userdoc:ns:100:specific - gid(0x64000000e7bd2a0d13c71935) - BucketId(0xe919c71300000064)
+userdoc:np:100:another - gid(0x64000000102745c9ceec3238) - BucketId(0xe832ecce00000064)
+userdoc:ns:101:specific - gid(0x6500000026238735e9cd9bb8) - BucketId(0xe89bcde900000065)
+groupdoc:ns:agroup:specific - gid(0x3272feb9fb729a50be795117) - BucketId(0xeb5179beb9fe7232)
+groupdoc:np:agroup:another - gid(0x3272feb93ff5e65f3cf4aba9) - BucketId(0xe9abf43cb9fe7232)
+groupdoc:ns:another:specific - gid(0xb32d73e51ae730d4fa104396) - BucketId(0xea4310fae5732db3)
+doc:ns:0 - gid(0x87817cf2f6d05976505e74be) - BucketId(0xea745e50f27c8187)
+doc:ns:1 - gid(0x911a03b253cb5b1c283b2024) - BucketId(0xe8203b28b2031a91)
+doc:ns:2 - gid(0x1d82e56be428cda364ed6875) - BucketId(0xe968ed646be5821d)
+doc:ns:3 - gid(0xf8d223e4e68e0d571b95a6d8) - BucketId(0xe8a6951be423d2f8)
+doc:ns:4 - gid(0xdab024c2e41747dc92a1b8e3) - BucketId(0xebb8a192c224b0da)
+doc:ns:5 - gid(0xdb9023e3080c94901734f948) - BucketId(0xe8f93417e32390db)
+doc:ns:6 - gid(0xbd84ae30c63f7fdef9edbf69) - BucketId(0xe9bfedf930ae84bd)
+doc:ns:7 - gid(0x463977cf070d06e204b8166f) - BucketId(0xeb16b804cf773946)
+doc:ns:8 - gid(0x46cf1241cec694a0c07af5e2) - BucketId(0xeaf57ac04112cf46)
+doc:ns:9 - gid(0x909b77593ef7b309a06d22ef) - BucketId(0xeb226da059779b90)
+doc:ns:10 - gid(0x4888f0b3796031003a8840fb) - BucketId(0xeb40883ab3f08848)
+doc:ns:11 - gid(0x18fae4cbc359765470c10fcd) - BucketId(0xe90fc170cbe4fa18)
+doc:ns:12 - gid(0xc902059d4ac551616aea5431) - BucketId(0xe954ea6a9d0502c9)
+doc:ns:13 - gid(0x639b6aa505018e29ca4e342d) - BucketId(0xe9344ecaa56a9b63)
+doc:ns:14 - gid(0x5fa1f02be952a9d0811e8ddd) - BucketId(0xe98d1e812bf0a15f)
+doc:ns:15 - gid(0xfbc851f81830365c84229c49) - BucketId(0xe99c2284f851c8fb)
+doc:ns:16 - gid(0x06313edc8072f4495329fb5b) - BucketId(0xebfb2953dc3e3106)
+doc:ns:17 - gid(0x3d9df3e147de3a5fbd5664e4) - BucketId(0xe86456bde1f39d3d)
+doc:ns:18 - gid(0x75512f41a8dbde1c8f86a97d) - BucketId(0xe9a9868f412f5175)
+doc:ns:19 - gid(0x15ae3aa9919b2e1e46d84ada) - BucketId(0xea4ad846a93aae15)
+id:ns:type::specific - gid(0x2067d966823ebdfb79537da1) - BucketId(0xe97d537966d96720)
+id:another:type::specific - gid(0x2067d9664809eb39a3b72218) - BucketId(0xe822b7a366d96720)
+id:ns:type::another - gid(0xb32d73e53e034c30c58a01ff) - BucketId(0xeb018ac5e5732db3)
+id:ns:type:n=100:specific - gid(0x64000000dd4c4b431749f95f) - BucketId(0xebf9491700000064)
+id:np:type:n=100:another - gid(0x6400000031359e4a70e670a8) - BucketId(0xe870e67000000064)
+id:ns:type:n=101:specific - gid(0x6500000083f2836ce5883df1) - BucketId(0xe93d88e500000065)
+id:ns:type:g=agroup:specific - gid(0x3272feb90fe348ef910b064a) - BucketId(0xea060b91b9fe7232)
+id:np:type:g=agroup:another - gid(0x3272feb96d53d40047cb4f23) - BucketId(0xeb4fcb47b9fe7232)
+id:ns:type:g=another:specific - gid(0xb32d73e58bd215d1451922cc) - BucketId(0xe8221945e5732db3)
diff --git a/document/src/tests/data/compressed.cfg b/document/src/tests/data/compressed.cfg
new file mode 100644
index 00000000000..740e56c47ac
--- /dev/null
+++ b/document/src/tests/data/compressed.cfg
@@ -0,0 +1,139 @@
+enablecompression true
+datatype[11]
+datatype[0].id -260050933
+datatype[0].arraytype[0]
+datatype[0].weightedsettype[0]
+datatype[0].structtype[1]
+datatype[0].structtype[0].name serializetest.header
+datatype[0].structtype[0].version 0
+datatype[0].structtype[0].compresstype LZ4
+datatype[0].structtype[0].compresslevel 9
+datatype[0].structtype[0].compressthreshold 99
+datatype[0].structtype[0].compressminsize 0
+datatype[0].structtype[0].field[4]
+datatype[0].structtype[0].field[0].name floatfield
+datatype[0].structtype[0].field[0].id[0]
+datatype[0].structtype[0].field[0].datatype 1
+datatype[0].structtype[0].field[1].name stringfield
+datatype[0].structtype[0].field[1].id[0]
+datatype[0].structtype[0].field[1].datatype 2
+datatype[0].structtype[0].field[2].name longfield
+datatype[0].structtype[0].field[2].id[0]
+datatype[0].structtype[0].field[2].datatype 4
+datatype[0].structtype[0].field[3].name urifield
+datatype[0].structtype[0].field[3].id[0]
+datatype[0].structtype[0].field[3].datatype 10
+datatype[0].documenttype[0]
+datatype[1].id 1001
+datatype[1].arraytype[1]
+datatype[1].arraytype[0].datatype 1
+datatype[1].weightedsettype[0]
+datatype[1].structtype[0]
+datatype[1].documenttype[0]
+datatype[2].id 2001
+datatype[2].arraytype[1]
+datatype[2].arraytype[0].datatype 1001
+datatype[2].weightedsettype[0]
+datatype[2].structtype[0]
+datatype[2].documenttype[0]
+datatype[3].id 1026122976
+datatype[3].arraytype[0]
+datatype[3].weightedsettype[0]
+datatype[3].structtype[1]
+datatype[3].structtype[0].name serializetest.body
+datatype[3].structtype[0].version 0
+datatype[3].structtype[0].compresstype LZ4
+datatype[3].structtype[0].compresslevel 9
+datatype[3].structtype[0].compressthreshold 99
+datatype[3].structtype[0].compressminsize 0
+datatype[3].structtype[0].field[8]
+datatype[3].structtype[0].field[0].name intfield
+datatype[3].structtype[0].field[0].id[0]
+datatype[3].structtype[0].field[0].datatype 0
+datatype[3].structtype[0].field[1].name rawfield
+datatype[3].structtype[0].field[1].id[0]
+datatype[3].structtype[0].field[1].datatype 3
+datatype[3].structtype[0].field[2].name doublefield
+datatype[3].structtype[0].field[2].id[0]
+datatype[3].structtype[0].field[2].datatype 5
+datatype[3].structtype[0].field[3].name contentfield
+datatype[3].structtype[0].field[3].id[0]
+datatype[3].structtype[0].field[3].datatype 2
+datatype[3].structtype[0].field[4].name bytefield
+datatype[3].structtype[0].field[4].id[0]
+datatype[3].structtype[0].field[4].datatype 16
+datatype[3].structtype[0].field[5].name arrayoffloatfield
+datatype[3].structtype[0].field[5].id[0]
+datatype[3].structtype[0].field[5].datatype 1001
+datatype[3].structtype[0].field[6].name arrayofarrayoffloatfield
+datatype[3].structtype[0].field[6].id[0]
+datatype[3].structtype[0].field[6].datatype 2001
+datatype[3].structtype[0].field[7].name docfield
+datatype[3].structtype[0].field[7].id[0]
+datatype[3].structtype[0].field[7].datatype 8
+datatype[3].documenttype[0]
+datatype[4].id 1306012852
+datatype[4].arraytype[0]
+datatype[4].weightedsettype[0]
+datatype[4].structtype[0]
+datatype[4].documenttype[1]
+datatype[4].documenttype[0].name serializetest
+datatype[4].documenttype[0].version 0
+datatype[4].documenttype[0].inherits[0]
+datatype[4].documenttype[0].headerstruct -260050933
+datatype[4].documenttype[0].bodystruct 1026122976
+datatype[5].id -1686125086
+datatype[5].arraytype[0]
+datatype[5].weightedsettype[0]
+datatype[5].structtype[1]
+datatype[5].structtype[0].name docindoc.header
+datatype[5].structtype[0].version 0
+datatype[5].structtype[0].field[0]
+datatype[5].documenttype[0]
+datatype[6].id 2030224503
+datatype[6].arraytype[0]
+datatype[6].weightedsettype[0]
+datatype[6].structtype[1]
+datatype[6].structtype[0].name docindoc.body
+datatype[6].structtype[0].version 0
+datatype[6].structtype[0].field[0]
+datatype[6].documenttype[0]
+datatype[7].id 1447635645
+datatype[7].arraytype[0]
+datatype[7].weightedsettype[0]
+datatype[7].structtype[0]
+datatype[7].documenttype[1]
+datatype[7].documenttype[0].name docindoc
+datatype[7].documenttype[0].version 0
+datatype[7].documenttype[0].inherits[0]
+datatype[7].documenttype[0].headerstruct -1686125086
+datatype[7].documenttype[0].bodystruct 2030224503
+datatype[8].id -792441727
+datatype[8].arraytype[0]
+datatype[8].weightedsettype[0]
+datatype[8].structtype[1]
+datatype[8].structtype[0].name mapindocindoc.header
+datatype[8].structtype[0].version 0
+datatype[8].structtype[0].field[0]
+datatype[8].documenttype[0]
+datatype[9].id 145123030
+datatype[9].arraytype[0]
+datatype[9].weightedsettype[0]
+datatype[9].structtype[1]
+datatype[9].structtype[0].name mapindocindoc.body
+datatype[9].structtype[0].version 0
+datatype[9].structtype[0].field[1]
+datatype[9].structtype[0].field[0].name stringinmap
+datatype[9].structtype[0].field[0].id[0]
+datatype[9].structtype[0].field[0].datatype 2
+datatype[9].documenttype[0]
+datatype[10].id -1456256770
+datatype[10].arraytype[0]
+datatype[10].weightedsettype[0]
+datatype[10].structtype[0]
+datatype[10].documenttype[1]
+datatype[10].documenttype[0].name mapindocindoc
+datatype[10].documenttype[0].version 0
+datatype[10].documenttype[0].inherits[0]
+datatype[10].documenttype[0].headerstruct -792441727
+datatype[10].documenttype[0].bodystruct 145123030
diff --git a/document/src/tests/data/cppdocument.cfg b/document/src/tests/data/cppdocument.cfg
new file mode 100644
index 00000000000..e4fcb5cb5bf
--- /dev/null
+++ b/document/src/tests/data/cppdocument.cfg
@@ -0,0 +1,131 @@
+enablecompression false
+datatype[11]
+datatype[0].id -260050933
+datatype[0].arraytype[0]
+datatype[0].weightedsettype[0]
+datatype[0].structtype[1]
+datatype[0].structtype[0].name serializetest.header
+datatype[0].structtype[0].version 0
+datatype[0].structtype[0].field[4]
+datatype[0].structtype[0].field[0].name floatfield
+datatype[0].structtype[0].field[0].id[0]
+datatype[0].structtype[0].field[0].datatype 1
+datatype[0].structtype[0].field[1].name stringfield
+datatype[0].structtype[0].field[1].id[0]
+datatype[0].structtype[0].field[1].datatype 2
+datatype[0].structtype[0].field[2].name longfield
+datatype[0].structtype[0].field[2].id[0]
+datatype[0].structtype[0].field[2].datatype 4
+datatype[0].structtype[0].field[3].name urifield
+datatype[0].structtype[0].field[3].id[0]
+datatype[0].structtype[0].field[3].datatype 10
+datatype[0].documenttype[0]
+datatype[1].id 1001
+datatype[1].arraytype[1]
+datatype[1].arraytype[0].datatype 1
+datatype[1].weightedsettype[0]
+datatype[1].structtype[0]
+datatype[1].documenttype[0]
+datatype[2].id 2001
+datatype[2].arraytype[1]
+datatype[2].arraytype[0].datatype 1001
+datatype[2].weightedsettype[0]
+datatype[2].structtype[0]
+datatype[2].documenttype[0]
+datatype[3].id 1026122976
+datatype[3].arraytype[0]
+datatype[3].weightedsettype[0]
+datatype[3].structtype[1]
+datatype[3].structtype[0].name serializetest.body
+datatype[3].structtype[0].version 0
+datatype[3].structtype[0].field[8]
+datatype[3].structtype[0].field[0].name intfield
+datatype[3].structtype[0].field[0].id[0]
+datatype[3].structtype[0].field[0].datatype 0
+datatype[3].structtype[0].field[1].name rawfield
+datatype[3].structtype[0].field[1].id[0]
+datatype[3].structtype[0].field[1].datatype 3
+datatype[3].structtype[0].field[2].name doublefield
+datatype[3].structtype[0].field[2].id[0]
+datatype[3].structtype[0].field[2].datatype 5
+datatype[3].structtype[0].field[3].name contentfield
+datatype[3].structtype[0].field[3].id[0]
+datatype[3].structtype[0].field[3].datatype 2
+datatype[3].structtype[0].field[4].name bytefield
+datatype[3].structtype[0].field[4].id[0]
+datatype[3].structtype[0].field[4].datatype 16
+datatype[3].structtype[0].field[5].name arrayoffloatfield
+datatype[3].structtype[0].field[5].id[0]
+datatype[3].structtype[0].field[5].datatype 1001
+datatype[3].structtype[0].field[6].name arrayofarrayoffloatfield
+datatype[3].structtype[0].field[6].id[0]
+datatype[3].structtype[0].field[6].datatype 2001
+datatype[3].structtype[0].field[7].name docfield
+datatype[3].structtype[0].field[7].id[0]
+datatype[3].structtype[0].field[7].datatype 8
+datatype[3].documenttype[0]
+datatype[4].id 1306012852
+datatype[4].arraytype[0]
+datatype[4].weightedsettype[0]
+datatype[4].structtype[0]
+datatype[4].documenttype[1]
+datatype[4].documenttype[0].name serializetest
+datatype[4].documenttype[0].version 0
+datatype[4].documenttype[0].inherits[0]
+datatype[4].documenttype[0].headerstruct -260050933
+datatype[4].documenttype[0].bodystruct 1026122976
+datatype[5].id -1686125086
+datatype[5].arraytype[0]
+datatype[5].weightedsettype[0]
+datatype[5].structtype[1]
+datatype[5].structtype[0].name docindoc.header
+datatype[5].structtype[0].version 0
+datatype[5].structtype[0].field[0]
+datatype[5].documenttype[0]
+datatype[6].id 2030224503
+datatype[6].arraytype[0]
+datatype[6].weightedsettype[0]
+datatype[6].structtype[1]
+datatype[6].structtype[0].name docindoc.body
+datatype[6].structtype[0].version 0
+datatype[6].structtype[0].field[0]
+datatype[6].documenttype[0]
+datatype[7].id 1447635645
+datatype[7].arraytype[0]
+datatype[7].weightedsettype[0]
+datatype[7].structtype[0]
+datatype[7].documenttype[1]
+datatype[7].documenttype[0].name docindoc
+datatype[7].documenttype[0].version 0
+datatype[7].documenttype[0].inherits[0]
+datatype[7].documenttype[0].headerstruct -1686125086
+datatype[7].documenttype[0].bodystruct 2030224503
+datatype[8].id -792441727
+datatype[8].arraytype[0]
+datatype[8].weightedsettype[0]
+datatype[8].structtype[1]
+datatype[8].structtype[0].name mapindocindoc.header
+datatype[8].structtype[0].version 0
+datatype[8].structtype[0].field[0]
+datatype[8].documenttype[0]
+datatype[9].id 145123030
+datatype[9].arraytype[0]
+datatype[9].weightedsettype[0]
+datatype[9].structtype[1]
+datatype[9].structtype[0].name mapindocindoc.body
+datatype[9].structtype[0].version 0
+datatype[9].structtype[0].field[1]
+datatype[9].structtype[0].field[0].name stringinmap
+datatype[9].structtype[0].field[0].id[0]
+datatype[9].structtype[0].field[0].datatype 2
+datatype[9].documenttype[0]
+datatype[10].id -1456256770
+datatype[10].arraytype[0]
+datatype[10].weightedsettype[0]
+datatype[10].structtype[0]
+datatype[10].documenttype[1]
+datatype[10].documenttype[0].name mapindocindoc
+datatype[10].documenttype[0].version 0
+datatype[10].documenttype[0].inherits[0]
+datatype[10].documenttype[0].headerstruct -792441727
+datatype[10].documenttype[0].bodystruct 145123030
diff --git a/document/src/tests/data/crossplatform-java-cpp-doctypes.cfg b/document/src/tests/data/crossplatform-java-cpp-doctypes.cfg
new file mode 100644
index 00000000000..6d33df468dd
--- /dev/null
+++ b/document/src/tests/data/crossplatform-java-cpp-doctypes.cfg
@@ -0,0 +1,202 @@
+enablecompression false
+documenttype[2]
+documenttype[0].id 1306012852
+documenttype[0].name "serializetest"
+documenttype[0].version 0
+documenttype[0].headerstruct -260050933
+documenttype[0].bodystruct 1026122976
+documenttype[0].inherits[0]
+documenttype[0].datatype[6]
+documenttype[0].datatype[0].id 9999
+documenttype[0].datatype[0].type MAP
+documenttype[0].datatype[0].array.element.id 0
+documenttype[0].datatype[0].map.key.id 2
+documenttype[0].datatype[0].map.value.id 2
+documenttype[0].datatype[0].wset.key.id 0
+documenttype[0].datatype[0].wset.createifnonexistent false
+documenttype[0].datatype[0].wset.removeifzero false
+documenttype[0].datatype[0].annotationref.annotation.id 0
+documenttype[0].datatype[0].sstruct.name ""
+documenttype[0].datatype[0].sstruct.version 0
+documenttype[0].datatype[0].sstruct.compression.type NONE
+documenttype[0].datatype[0].sstruct.compression.level 0
+documenttype[0].datatype[0].sstruct.compression.threshold 95
+documenttype[0].datatype[0].sstruct.compression.minsize 200
+documenttype[0].datatype[0].sstruct.field[0]
+documenttype[0].datatype[1].id 1001
+documenttype[0].datatype[1].type ARRAY
+documenttype[0].datatype[1].array.element.id 1
+documenttype[0].datatype[1].map.key.id 0
+documenttype[0].datatype[1].map.value.id 0
+documenttype[0].datatype[1].wset.key.id 0
+documenttype[0].datatype[1].wset.createifnonexistent false
+documenttype[0].datatype[1].wset.removeifzero false
+documenttype[0].datatype[1].annotationref.annotation.id 0
+documenttype[0].datatype[1].sstruct.name ""
+documenttype[0].datatype[1].sstruct.version 0
+documenttype[0].datatype[1].sstruct.compression.type NONE
+documenttype[0].datatype[1].sstruct.compression.level 0
+documenttype[0].datatype[1].sstruct.compression.threshold 95
+documenttype[0].datatype[1].sstruct.compression.minsize 200
+documenttype[0].datatype[1].sstruct.field[0]
+documenttype[0].datatype[2].id 2001
+documenttype[0].datatype[2].type ARRAY
+documenttype[0].datatype[2].array.element.id 1001
+documenttype[0].datatype[2].map.key.id 0
+documenttype[0].datatype[2].map.value.id 0
+documenttype[0].datatype[2].wset.key.id 0
+documenttype[0].datatype[2].wset.createifnonexistent false
+documenttype[0].datatype[2].wset.removeifzero false
+documenttype[0].datatype[2].annotationref.annotation.id 0
+documenttype[0].datatype[2].sstruct.name ""
+documenttype[0].datatype[2].sstruct.version 0
+documenttype[0].datatype[2].sstruct.compression.type NONE
+documenttype[0].datatype[2].sstruct.compression.level 0
+documenttype[0].datatype[2].sstruct.compression.threshold 95
+documenttype[0].datatype[2].sstruct.compression.minsize 200
+documenttype[0].datatype[2].sstruct.field[0]
+documenttype[0].datatype[3].id 437829
+documenttype[0].datatype[3].type WSET
+documenttype[0].datatype[3].array.element.id 0
+documenttype[0].datatype[3].map.key.id 0
+documenttype[0].datatype[3].map.value.id 0
+documenttype[0].datatype[3].wset.key.id 2
+documenttype[0].datatype[3].wset.createifnonexistent false
+documenttype[0].datatype[3].wset.removeifzero false
+documenttype[0].datatype[3].annotationref.annotation.id 0
+documenttype[0].datatype[3].sstruct.name ""
+documenttype[0].datatype[3].sstruct.version 0
+documenttype[0].datatype[3].sstruct.compression.type NONE
+documenttype[0].datatype[3].sstruct.compression.level 0
+documenttype[0].datatype[3].sstruct.compression.threshold 95
+documenttype[0].datatype[3].sstruct.compression.minsize 200
+documenttype[0].datatype[3].sstruct.field[0]
+documenttype[0].datatype[4].id -260050933
+documenttype[0].datatype[4].type STRUCT
+documenttype[0].datatype[4].array.element.id 0
+documenttype[0].datatype[4].map.key.id 0
+documenttype[0].datatype[4].map.value.id 0
+documenttype[0].datatype[4].wset.key.id 0
+documenttype[0].datatype[4].wset.createifnonexistent false
+documenttype[0].datatype[4].wset.removeifzero false
+documenttype[0].datatype[4].annotationref.annotation.id 0
+documenttype[0].datatype[4].sstruct.name "serializetest.header"
+documenttype[0].datatype[4].sstruct.version 0
+documenttype[0].datatype[4].sstruct.compression.type NONE
+documenttype[0].datatype[4].sstruct.compression.level 0
+documenttype[0].datatype[4].sstruct.compression.threshold 90
+documenttype[0].datatype[4].sstruct.compression.minsize 0
+documenttype[0].datatype[4].sstruct.field[4]
+documenttype[0].datatype[4].sstruct.field[0].name "floatfield"
+documenttype[0].datatype[4].sstruct.field[0].id 1055999199
+documenttype[0].datatype[4].sstruct.field[0].id_v6 657399019
+documenttype[0].datatype[4].sstruct.field[0].datatype 1
+documenttype[0].datatype[4].sstruct.field[1].name "longfield"
+documenttype[0].datatype[4].sstruct.field[1].id 1589309697
+documenttype[0].datatype[4].sstruct.field[1].id_v6 477609536
+documenttype[0].datatype[4].sstruct.field[1].datatype 4
+documenttype[0].datatype[4].sstruct.field[2].name "stringfield"
+documenttype[0].datatype[4].sstruct.field[2].id 1182460484
+documenttype[0].datatype[4].sstruct.field[2].id_v6 779638844
+documenttype[0].datatype[4].sstruct.field[2].datatype 2
+documenttype[0].datatype[4].sstruct.field[3].name "urifield"
+documenttype[0].datatype[4].sstruct.field[3].id 628407450
+documenttype[0].datatype[4].sstruct.field[3].id_v6 756285986
+documenttype[0].datatype[4].sstruct.field[3].datatype 10
+documenttype[0].datatype[5].id 1026122976
+documenttype[0].datatype[5].type STRUCT
+documenttype[0].datatype[5].array.element.id 0
+documenttype[0].datatype[5].map.key.id 0
+documenttype[0].datatype[5].map.value.id 0
+documenttype[0].datatype[5].wset.key.id 0
+documenttype[0].datatype[5].wset.createifnonexistent false
+documenttype[0].datatype[5].wset.removeifzero false
+documenttype[0].datatype[5].annotationref.annotation.id 0
+documenttype[0].datatype[5].sstruct.name "serializetest.body"
+documenttype[0].datatype[5].sstruct.version 0
+documenttype[0].datatype[5].sstruct.compression.type NONE
+documenttype[0].datatype[5].sstruct.compression.level 0
+documenttype[0].datatype[5].sstruct.compression.threshold 90
+documenttype[0].datatype[5].sstruct.compression.minsize 0
+documenttype[0].datatype[5].sstruct.field[9]
+documenttype[0].datatype[5].sstruct.field[0].name "arrayofarrayoffloatfield"
+documenttype[0].datatype[5].sstruct.field[0].id 2061447601
+documenttype[0].datatype[5].sstruct.field[0].id_v6 903107040
+documenttype[0].datatype[5].sstruct.field[0].datatype 2001
+documenttype[0].datatype[5].sstruct.field[1].name "arrayoffloatfield"
+documenttype[0].datatype[5].sstruct.field[1].id 1870932758
+documenttype[0].datatype[5].sstruct.field[1].id_v6 86189529
+documenttype[0].datatype[5].sstruct.field[1].datatype 1001
+documenttype[0].datatype[5].sstruct.field[2].name "bytefield"
+documenttype[0].datatype[5].sstruct.field[2].id 1924064342
+documenttype[0].datatype[5].sstruct.field[2].id_v6 2050488857
+documenttype[0].datatype[5].sstruct.field[2].datatype 16
+documenttype[0].datatype[5].sstruct.field[3].name "docfield"
+documenttype[0].datatype[5].sstruct.field[3].id 728739268
+documenttype[0].datatype[5].sstruct.field[3].id_v6 1725514984
+documenttype[0].datatype[5].sstruct.field[3].datatype 8
+documenttype[0].datatype[5].sstruct.field[4].name "doublefield"
+documenttype[0].datatype[5].sstruct.field[4].id 421343958
+documenttype[0].datatype[5].sstruct.field[4].id_v6 944141865
+documenttype[0].datatype[5].sstruct.field[4].datatype 5
+documenttype[0].datatype[5].sstruct.field[5].name "intfield"
+documenttype[0].datatype[5].sstruct.field[5].id 435380425
+documenttype[0].datatype[5].sstruct.field[5].id_v6 545156740
+documenttype[0].datatype[5].sstruct.field[5].datatype 0
+documenttype[0].datatype[5].sstruct.field[6].name "mapfield"
+documenttype[0].datatype[5].sstruct.field[6].id 162466023
+documenttype[0].datatype[5].sstruct.field[6].id_v6 1028589263
+documenttype[0].datatype[5].sstruct.field[6].datatype 9999
+documenttype[0].datatype[5].sstruct.field[7].name "rawfield"
+documenttype[0].datatype[5].sstruct.field[7].id 172982133
+documenttype[0].datatype[5].sstruct.field[7].id_v6 84029972
+documenttype[0].datatype[5].sstruct.field[7].datatype 3
+documenttype[0].datatype[5].sstruct.field[8].name "wsfield"
+documenttype[0].datatype[5].sstruct.field[8].id 1683714185
+documenttype[0].datatype[5].sstruct.field[8].id_v6 2988081
+documenttype[0].datatype[5].sstruct.field[8].datatype 437829
+documenttype[0].annotationtype[0]
+documenttype[1].id 1447635645
+documenttype[1].name "docindoc"
+documenttype[1].version 0
+documenttype[1].headerstruct -1686125086
+documenttype[1].bodystruct 2030224503
+documenttype[1].inherits[0]
+documenttype[1].datatype[2]
+documenttype[1].datatype[0].id -1686125086
+documenttype[1].datatype[0].type STRUCT
+documenttype[1].datatype[0].array.element.id 0
+documenttype[1].datatype[0].map.key.id 0
+documenttype[1].datatype[0].map.value.id 0
+documenttype[1].datatype[0].wset.key.id 0
+documenttype[1].datatype[0].wset.createifnonexistent false
+documenttype[1].datatype[0].wset.removeifzero false
+documenttype[1].datatype[0].annotationref.annotation.id 0
+documenttype[1].datatype[0].sstruct.name "docindoc.header"
+documenttype[1].datatype[0].sstruct.version 0
+documenttype[1].datatype[0].sstruct.compression.type NONE
+documenttype[1].datatype[0].sstruct.compression.level 0
+documenttype[1].datatype[0].sstruct.compression.threshold 90
+documenttype[1].datatype[0].sstruct.compression.minsize 0
+documenttype[1].datatype[0].sstruct.field[0]
+documenttype[1].datatype[1].id 2030224503
+documenttype[1].datatype[1].type STRUCT
+documenttype[1].datatype[1].array.element.id 0
+documenttype[1].datatype[1].map.key.id 0
+documenttype[1].datatype[1].map.value.id 0
+documenttype[1].datatype[1].wset.key.id 0
+documenttype[1].datatype[1].wset.createifnonexistent false
+documenttype[1].datatype[1].wset.removeifzero false
+documenttype[1].datatype[1].annotationref.annotation.id 0
+documenttype[1].datatype[1].sstruct.name "docindoc.body"
+documenttype[1].datatype[1].sstruct.version 0
+documenttype[1].datatype[1].sstruct.compression.type NONE
+documenttype[1].datatype[1].sstruct.compression.level 0
+documenttype[1].datatype[1].sstruct.compression.threshold 90
+documenttype[1].datatype[1].sstruct.compression.minsize 0
+documenttype[1].datatype[1].sstruct.field[1]
+documenttype[1].datatype[1].sstruct.field[0].name "stringindocfield"
+documenttype[1].datatype[1].sstruct.field[0].id 102054923
+documenttype[1].datatype[1].sstruct.field[0].id_v6 917294979
+documenttype[1].datatype[1].sstruct.field[0].datatype 2
+documenttype[1].annotationtype[0]
diff --git a/document/src/tests/data/crossplatform-java-cpp-document.cfg b/document/src/tests/data/crossplatform-java-cpp-document.cfg
new file mode 100644
index 00000000000..f12dae77fc0
--- /dev/null
+++ b/document/src/tests/data/crossplatform-java-cpp-document.cfg
@@ -0,0 +1,128 @@
+enablecompression false
+datatype[10]
+datatype[0].id 9999
+datatype[0].arraytype[0]
+datatype[0].maptype[1]
+datatype[0].maptype[0].keytype 2
+datatype[0].maptype[0].valtype 2
+datatype[0].weightedsettype[0]
+datatype[0].structtype[0]
+datatype[0].documenttype[0]
+datatype[1].id 1001
+datatype[1].arraytype[1]
+datatype[1].arraytype[0].datatype 1
+datatype[1].weightedsettype[0]
+datatype[1].structtype[0]
+datatype[1].documenttype[0]
+datatype[2].id 2001
+datatype[2].arraytype[1]
+datatype[2].arraytype[0].datatype 1001
+datatype[2].weightedsettype[0]
+datatype[2].structtype[0]
+datatype[2].documenttype[0]
+datatype[3].id 437829
+datatype[3].arraytype[0]
+datatype[3].weightedsettype[1]
+datatype[3].weightedsettype[0].datatype 2
+datatype[3].weightedsettype[0].createifnonexistant false
+datatype[3].weightedsettype[0].removeifzero false
+datatype[3].structtype[0]
+datatype[3].documenttype[0]
+datatype[4].id -1686125086
+datatype[4].arraytype[0]
+datatype[4].weightedsettype[0]
+datatype[4].structtype[1]
+datatype[4].structtype[0].name docindoc.header
+datatype[4].structtype[0].version 0
+datatype[4].structtype[0].field[0]
+datatype[4].documenttype[0]
+datatype[5].id 2030224503
+datatype[5].arraytype[0]
+datatype[5].weightedsettype[0]
+datatype[5].structtype[1]
+datatype[5].structtype[0].name docindoc.body
+datatype[5].structtype[0].version 0
+datatype[5].structtype[0].field[1]
+datatype[5].structtype[0].field[0].name stringindocfield
+datatype[5].structtype[0].field[0].id[0]
+datatype[5].structtype[0].field[0].datatype 2
+datatype[5].documenttype[0]
+datatype[6].id 1447635645
+datatype[6].arraytype[0]
+datatype[6].weightedsettype[0]
+datatype[6].structtype[0]
+datatype[6].documenttype[1]
+datatype[6].documenttype[0].name docindoc
+datatype[6].documenttype[0].version 0
+datatype[6].documenttype[0].inherits[0]
+datatype[6].documenttype[0].headerstruct -1686125086
+datatype[6].documenttype[0].bodystruct 2030224503
+datatype[7].id -260050933
+datatype[7].arraytype[0]
+datatype[7].weightedsettype[0]
+datatype[7].structtype[1]
+datatype[7].structtype[0].name serializetest.header
+datatype[7].structtype[0].version 0
+datatype[7].structtype[0].field[4]
+datatype[7].structtype[0].field[0].name floatfield
+datatype[7].structtype[0].field[0].id[0]
+datatype[7].structtype[0].field[0].datatype 1
+datatype[7].structtype[0].field[1].name stringfield
+datatype[7].structtype[0].field[1].id[0]
+datatype[7].structtype[0].field[1].datatype 2
+datatype[7].structtype[0].field[2].name longfield
+datatype[7].structtype[0].field[2].id[0]
+datatype[7].structtype[0].field[2].datatype 4
+datatype[7].structtype[0].field[3].name urifield
+datatype[7].structtype[0].field[3].id[0]
+datatype[7].structtype[0].field[3].datatype 10
+datatype[7].documenttype[0]
+datatype[8].id 1026122976
+datatype[8].arraytype[0]
+datatype[8].weightedsettype[0]
+datatype[8].structtype[1]
+datatype[8].structtype[0].name serializetest.body
+datatype[8].structtype[0].version 0
+datatype[8].structtype[0].field[10]
+datatype[8].structtype[0].field[0].name intfield
+datatype[8].structtype[0].field[0].id[0]
+datatype[8].structtype[0].field[0].datatype 0
+datatype[8].structtype[0].field[1].name rawfield
+datatype[8].structtype[0].field[1].id[0]
+datatype[8].structtype[0].field[1].datatype 3
+datatype[8].structtype[0].field[2].name doublefield
+datatype[8].structtype[0].field[2].id[0]
+datatype[8].structtype[0].field[2].datatype 5
+datatype[8].structtype[0].field[3].name contentfield
+datatype[8].structtype[0].field[3].id[0]
+datatype[8].structtype[0].field[3].datatype 2
+datatype[8].structtype[0].field[4].name bytefield
+datatype[8].structtype[0].field[4].id[0]
+datatype[8].structtype[0].field[4].datatype 16
+datatype[8].structtype[0].field[5].name arrayoffloatfield
+datatype[8].structtype[0].field[5].id[0]
+datatype[8].structtype[0].field[5].datatype 1001
+datatype[8].structtype[0].field[6].name arrayofarrayoffloatfield
+datatype[8].structtype[0].field[6].id[0]
+datatype[8].structtype[0].field[6].datatype 2001
+datatype[8].structtype[0].field[7].name docfield
+datatype[8].structtype[0].field[7].id[0]
+datatype[8].structtype[0].field[7].datatype 8
+datatype[8].structtype[0].field[8].name wsfield
+datatype[8].structtype[0].field[8].id[0]
+datatype[8].structtype[0].field[8].datatype 437829
+datatype[8].structtype[0].field[9].name mapfield
+datatype[8].structtype[0].field[9].id[0]
+datatype[8].structtype[0].field[9].datatype 9999
+datatype[8].documenttype[0]
+datatype[9].id 1306012852
+datatype[9].arraytype[0]
+datatype[9].weightedsettype[0]
+datatype[9].structtype[0]
+datatype[9].documenttype[1]
+datatype[9].documenttype[0].name serializetest
+datatype[9].documenttype[0].version 0
+datatype[9].documenttype[0].inherits[0]
+datatype[9].documenttype[0].headerstruct -260050933
+datatype[9].documenttype[0].bodystruct 1026122976
+
diff --git a/document/src/tests/data/defaultdoctypes.cfg b/document/src/tests/data/defaultdoctypes.cfg
new file mode 100644
index 00000000000..8d9f71cfeb9
--- /dev/null
+++ b/document/src/tests/data/defaultdoctypes.cfg
@@ -0,0 +1,150 @@
+enablecompression false
+documenttype[1]
+documenttype[0].id -1175657560
+documenttype[0].name "testdoc"
+documenttype[0].version 0
+documenttype[0].headerstruct 5000
+documenttype[0].bodystruct 5001
+documenttype[0].inherits[0]
+documenttype[0].datatype[5]
+documenttype[0].datatype[0].id 1000
+documenttype[0].datatype[0].type ARRAY
+documenttype[0].datatype[0].array.element.id 0
+documenttype[0].datatype[0].map.key.id 0
+documenttype[0].datatype[0].map.value.id 0
+documenttype[0].datatype[0].wset.key.id 0
+documenttype[0].datatype[0].wset.createifnonexistent false
+documenttype[0].datatype[0].wset.removeifzero false
+documenttype[0].datatype[0].annotationref.annotation.id 0
+documenttype[0].datatype[0].sstruct.name ""
+documenttype[0].datatype[0].sstruct.version 0
+documenttype[0].datatype[0].sstruct.compression.type NONE
+documenttype[0].datatype[0].sstruct.compression.level 0
+documenttype[0].datatype[0].sstruct.compression.threshold 95
+documenttype[0].datatype[0].sstruct.compression.minsize 200
+documenttype[0].datatype[0].sstruct.field[0]
+documenttype[0].datatype[1].id 1003
+documenttype[0].datatype[1].type ARRAY
+documenttype[0].datatype[1].array.element.id 3
+documenttype[0].datatype[1].map.key.id 0
+documenttype[0].datatype[1].map.value.id 0
+documenttype[0].datatype[1].wset.key.id 0
+documenttype[0].datatype[1].wset.createifnonexistent false
+documenttype[0].datatype[1].wset.removeifzero false
+documenttype[0].datatype[1].annotationref.annotation.id 0
+documenttype[0].datatype[1].sstruct.name ""
+documenttype[0].datatype[1].sstruct.version 0
+documenttype[0].datatype[1].sstruct.compression.type NONE
+documenttype[0].datatype[1].sstruct.compression.level 0
+documenttype[0].datatype[1].sstruct.compression.threshold 95
+documenttype[0].datatype[1].sstruct.compression.minsize 200
+documenttype[0].datatype[1].sstruct.field[0]
+documenttype[0].datatype[2].id 2002
+documenttype[0].datatype[2].type WSET
+documenttype[0].datatype[2].array.element.id 0
+documenttype[0].datatype[2].map.key.id 0
+documenttype[0].datatype[2].map.value.id 0
+documenttype[0].datatype[2].wset.key.id 2
+documenttype[0].datatype[2].wset.createifnonexistent false
+documenttype[0].datatype[2].wset.removeifzero false
+documenttype[0].datatype[2].annotationref.annotation.id 0
+documenttype[0].datatype[2].sstruct.name ""
+documenttype[0].datatype[2].sstruct.version 0
+documenttype[0].datatype[2].sstruct.compression.type NONE
+documenttype[0].datatype[2].sstruct.compression.level 0
+documenttype[0].datatype[2].sstruct.compression.threshold 95
+documenttype[0].datatype[2].sstruct.compression.minsize 200
+documenttype[0].datatype[2].sstruct.field[0]
+documenttype[0].datatype[3].id 5000
+documenttype[0].datatype[3].type STRUCT
+documenttype[0].datatype[3].array.element.id 0
+documenttype[0].datatype[3].map.key.id 0
+documenttype[0].datatype[3].map.value.id 0
+documenttype[0].datatype[3].wset.key.id 0
+documenttype[0].datatype[3].wset.createifnonexistent false
+documenttype[0].datatype[3].wset.removeifzero false
+documenttype[0].datatype[3].annotationref.annotation.id 0
+documenttype[0].datatype[3].sstruct.name "testdoc.header"
+documenttype[0].datatype[3].sstruct.version 0
+documenttype[0].datatype[3].sstruct.compression.type NONE
+documenttype[0].datatype[3].sstruct.compression.level 0
+documenttype[0].datatype[3].sstruct.compression.threshold 90
+documenttype[0].datatype[3].sstruct.compression.minsize 0
+documenttype[0].datatype[3].sstruct.field[3]
+documenttype[0].datatype[3].sstruct.field[0].name "doubleattr"
+documenttype[0].datatype[3].sstruct.field[0].id 149562827
+documenttype[0].datatype[3].sstruct.field[0].id_v6 630428365
+documenttype[0].datatype[3].sstruct.field[0].datatype 5
+documenttype[0].datatype[3].sstruct.field[1].name "floatattr"
+documenttype[0].datatype[3].sstruct.field[1].id 1633158509
+documenttype[0].datatype[3].sstruct.field[1].id_v6 1853512789
+documenttype[0].datatype[3].sstruct.field[1].datatype 1
+documenttype[0].datatype[3].sstruct.field[2].name "intattr"
+documenttype[0].datatype[3].sstruct.field[2].id 2079212392
+documenttype[0].datatype[3].sstruct.field[2].id_v6 1840069917
+documenttype[0].datatype[3].sstruct.field[2].datatype 0
+documenttype[0].datatype[4].id 5001
+documenttype[0].datatype[4].type STRUCT
+documenttype[0].datatype[4].array.element.id 0
+documenttype[0].datatype[4].map.key.id 0
+documenttype[0].datatype[4].map.value.id 0
+documenttype[0].datatype[4].wset.key.id 0
+documenttype[0].datatype[4].wset.createifnonexistent false
+documenttype[0].datatype[4].wset.removeifzero false
+documenttype[0].datatype[4].annotationref.annotation.id 0
+documenttype[0].datatype[4].sstruct.name "testdoc.body"
+documenttype[0].datatype[4].sstruct.version 0
+documenttype[0].datatype[4].sstruct.compression.type NONE
+documenttype[0].datatype[4].sstruct.compression.level 0
+documenttype[0].datatype[4].sstruct.compression.threshold 90
+documenttype[0].datatype[4].sstruct.compression.minsize 0
+documenttype[0].datatype[4].sstruct.field[12]
+documenttype[0].datatype[4].sstruct.field[0].name "arrayattr"
+documenttype[0].datatype[4].sstruct.field[0].id 505184256
+documenttype[0].datatype[4].sstruct.field[0].id_v6 1771774378
+documenttype[0].datatype[4].sstruct.field[0].datatype 1000
+documenttype[0].datatype[4].sstruct.field[1].name "byteattr"
+documenttype[0].datatype[4].sstruct.field[1].id 2134674304
+documenttype[0].datatype[4].sstruct.field[1].id_v6 973038129
+documenttype[0].datatype[4].sstruct.field[1].datatype 16
+documenttype[0].datatype[4].sstruct.field[2].name "docfield"
+documenttype[0].datatype[4].sstruct.field[2].id 728739268
+documenttype[0].datatype[4].sstruct.field[2].id_v6 1725514984
+documenttype[0].datatype[4].sstruct.field[2].datatype 8
+documenttype[0].datatype[4].sstruct.field[3].name "longattr"
+documenttype[0].datatype[4].sstruct.field[3].id 790774460
+documenttype[0].datatype[4].sstruct.field[3].id_v6 192291556
+documenttype[0].datatype[4].sstruct.field[3].datatype 4
+documenttype[0].datatype[4].sstruct.field[4].name "minattr"
+documenttype[0].datatype[4].sstruct.field[4].id 1829587385
+documenttype[0].datatype[4].sstruct.field[4].id_v6 741727195
+documenttype[0].datatype[4].sstruct.field[4].datatype 0
+documenttype[0].datatype[4].sstruct.field[5].name "minattr2"
+documenttype[0].datatype[4].sstruct.field[5].id 128399609
+documenttype[0].datatype[4].sstruct.field[5].id_v6 920037076
+documenttype[0].datatype[4].sstruct.field[5].datatype 0
+documenttype[0].datatype[4].sstruct.field[6].name "rawarrayattr"
+documenttype[0].datatype[4].sstruct.field[6].id 730665199
+documenttype[0].datatype[4].sstruct.field[6].id_v6 1023469062
+documenttype[0].datatype[4].sstruct.field[6].datatype 1003
+documenttype[0].datatype[4].sstruct.field[7].name "rawattr"
+documenttype[0].datatype[4].sstruct.field[7].id 141092158
+documenttype[0].datatype[4].sstruct.field[7].id_v6 1247497470
+documenttype[0].datatype[4].sstruct.field[7].datatype 3
+documenttype[0].datatype[4].sstruct.field[8].name "stringattr"
+documenttype[0].datatype[4].sstruct.field[8].id 1869816099
+documenttype[0].datatype[4].sstruct.field[8].id_v6 1592475234
+documenttype[0].datatype[4].sstruct.field[8].datatype 2
+documenttype[0].datatype[4].sstruct.field[9].name "stringattr2"
+documenttype[0].datatype[4].sstruct.field[9].id 1054720742
+documenttype[0].datatype[4].sstruct.field[9].id_v6 1869816099
+documenttype[0].datatype[4].sstruct.field[9].datatype 2
+documenttype[0].datatype[4].sstruct.field[10].name "stringweightedsetattr"
+documenttype[0].datatype[4].sstruct.field[10].id 1791723468
+documenttype[0].datatype[4].sstruct.field[10].id_v6 676317895
+documenttype[0].datatype[4].sstruct.field[10].datatype 2002
+documenttype[0].datatype[4].sstruct.field[11].name "uri"
+documenttype[0].datatype[4].sstruct.field[11].id 31928604
+documenttype[0].datatype[4].sstruct.field[11].id_v6 1003424810
+documenttype[0].datatype[4].sstruct.field[11].datatype 2
+documenttype[0].annotationtype[0]
diff --git a/document/src/tests/data/defaultdocument.cfg b/document/src/tests/data/defaultdocument.cfg
new file mode 100644
index 00000000000..9780f43def6
--- /dev/null
+++ b/document/src/tests/data/defaultdocument.cfg
@@ -0,0 +1,94 @@
+enablecompression false
+datatype[6]
+datatype[0].id 1000
+datatype[0].arraytype[1]
+datatype[0].arraytype[0].datatype 0
+datatype[0].weightedsettype[0]
+datatype[0].structtype[0]
+datatype[0].documenttype[0]
+datatype[1].id 1003
+datatype[1].arraytype[1]
+datatype[1].arraytype[0].datatype 3
+datatype[1].weightedsettype[0]
+datatype[1].structtype[0]
+datatype[1].documenttype[0]
+datatype[2].id 2002
+datatype[2].arraytype[0]
+datatype[2].weightedsettype[1]
+datatype[2].weightedsettype[0].datatype 2
+datatype[2].weightedsettype[0].createifnonexistant false
+datatype[2].weightedsettype[0].removeifzero false
+datatype[2].structtype[0]
+datatype[2].documenttype[0]
+datatype[3].id 5000
+datatype[3].arraytype[0]
+datatype[3].weightedsettype[0]
+datatype[3].structtype[1]
+datatype[3].structtype[0].name testdoc.header
+datatype[3].structtype[0].version 0
+datatype[3].structtype[0].field[3]
+datatype[3].structtype[0].field[0].name intattr
+datatype[3].structtype[0].field[0].id[0]
+datatype[3].structtype[0].field[0].datatype 0
+datatype[3].structtype[0].field[1].name doubleattr
+datatype[3].structtype[0].field[1].id[0]
+datatype[3].structtype[0].field[1].datatype 5
+datatype[3].structtype[0].field[2].name floatattr
+datatype[3].structtype[0].field[2].id[0]
+datatype[3].structtype[0].field[2].datatype 1
+datatype[3].documenttype[0]
+datatype[4].id 5001
+datatype[4].arraytype[0]
+datatype[4].weightedsettype[0]
+datatype[4].structtype[1]
+datatype[4].structtype[0].name testdoc.body
+datatype[4].structtype[0].version 0
+datatype[4].structtype[0].field[11]
+datatype[4].structtype[0].field[0].name stringattr
+datatype[4].structtype[0].field[0].id[0]
+datatype[4].structtype[0].field[0].datatype 2
+datatype[4].structtype[0].field[1].name stringattr2
+datatype[4].structtype[0].field[1].id[0]
+datatype[4].structtype[0].field[1].datatype 2
+datatype[4].structtype[0].field[2].name longattr
+datatype[4].structtype[0].field[2].id[0]
+datatype[4].structtype[0].field[2].datatype 4
+datatype[4].structtype[0].field[3].name byteattr
+datatype[4].structtype[0].field[3].id[0]
+datatype[4].structtype[0].field[3].datatype 16
+datatype[4].structtype[0].field[4].name rawattr
+datatype[4].structtype[0].field[4].id[0]
+datatype[4].structtype[0].field[4].datatype 3
+datatype[4].structtype[0].field[5].name minattr
+datatype[4].structtype[0].field[5].id[0]
+datatype[4].structtype[0].field[5].datatype 0
+datatype[4].structtype[0].field[6].name minattr2
+datatype[4].structtype[0].field[6].id[0]
+datatype[4].structtype[0].field[6].datatype 0
+datatype[4].structtype[0].field[7].name arrayattr
+datatype[4].structtype[0].field[7].id[0]
+datatype[4].structtype[0].field[7].datatype 1000
+datatype[4].structtype[0].field[8].name rawarrayattr
+datatype[4].structtype[0].field[8].id[0]
+datatype[4].structtype[0].field[8].datatype 1003
+datatype[4].structtype[0].field[9].name stringweightedsetattr
+datatype[4].structtype[0].field[9].id[0]
+datatype[4].structtype[0].field[9].datatype 2002
+datatype[4].structtype[0].field[10].name uri
+datatype[4].structtype[0].field[10].id[0]
+datatype[4].structtype[0].field[10].datatype 2
+datatype[4].structtype[0].field[11].name docfield
+datatype[4].structtype[0].field[11].id[0]
+datatype[4].structtype[0].field[11].datatype 8
+datatype[4].documenttype[0]
+datatype[5].id 5002
+datatype[5].arraytype[0]
+datatype[5].weightedsettype[0]
+datatype[5].structtype[0]
+datatype[5].documenttype[1]
+datatype[5].documenttype[0].name testdoc
+datatype[5].documenttype[0].version 0
+datatype[5].documenttype[0].inherits[0]
+datatype[5].documenttype[0].headerstruct 5000
+datatype[5].documenttype[0].bodystruct 5001
+
diff --git a/document/src/tests/data/docmancfg.txt b/document/src/tests/data/docmancfg.txt
new file mode 100644
index 00000000000..9507211edf3
--- /dev/null
+++ b/document/src/tests/data/docmancfg.txt
@@ -0,0 +1,170 @@
+enablecompression false
+datatype[16]
+datatype[0].id 101
+datatype[0].arraytype[1]
+datatype[0].arraytype[0].datatype 1
+datatype[0].weightedsettype[0]
+datatype[0].structtype[0]
+datatype[0].documenttype[0]
+datatype[1].id 102
+datatype[1].arraytype[1]
+datatype[1].arraytype[0].datatype 2
+datatype[1].weightedsettype[0]
+datatype[1].structtype[0]
+datatype[1].documenttype[0]
+datatype[2].id 103
+datatype[2].arraytype[0]
+datatype[2].weightedsettype[0]
+datatype[2].structtype[1]
+datatype[2].structtype[0].name teststruct
+datatype[2].structtype[0].version 0
+datatype[2].structtype[0].field[2]
+datatype[2].structtype[0].field[0].name Foo
+datatype[2].structtype[0].field[0].id[0]
+datatype[2].structtype[0].field[0].datatype 2
+datatype[2].structtype[0].field[1].name Bar
+datatype[2].structtype[0].field[1].id[0]
+datatype[2].structtype[0].field[1].datatype 2
+datatype[2].documenttype[0]
+datatype[3].id 104
+datatype[3].arraytype[0]
+datatype[3].weightedsettype[0]
+datatype[3].structtype[1]
+datatype[3].structtype[0].name teststruct
+datatype[3].structtype[0].version 1
+datatype[3].structtype[0].field[2]
+datatype[3].structtype[0].field[0].name Foooo
+datatype[3].structtype[0].field[0].id[0]
+datatype[3].structtype[0].field[0].datatype 2
+datatype[3].structtype[0].field[1].name Bar
+datatype[3].structtype[0].field[1].id[0]
+datatype[3].structtype[0].field[1].datatype 2
+datatype[3].documenttype[0]
+datatype[4].id 1000
+datatype[4].arraytype[0]
+datatype[4].weightedsettype[0]
+datatype[4].structtype[1]
+datatype[4].structtype[0].name crawler_type.header
+datatype[4].structtype[0].version 0
+datatype[4].structtype[0].field[5]
+datatype[4].structtype[0].field[0].name URI
+datatype[4].structtype[0].field[0].id[1]
+datatype[4].structtype[0].field[0].id[0].id 0
+datatype[4].structtype[0].field[0].datatype 2
+datatype[4].structtype[0].field[1].name CONTEXT
+datatype[4].structtype[0].field[1].id[1]
+datatype[4].structtype[0].field[1].id[0].id 1
+datatype[4].structtype[0].field[1].datatype 2
+datatype[4].structtype[0].field[2].name CONTENT
+datatype[4].structtype[0].field[2].id[1]
+datatype[4].structtype[0].field[2].id[0].id 2
+datatype[4].structtype[0].field[2].datatype 3
+datatype[4].structtype[0].field[3].name CONTENT_TYPE
+datatype[4].structtype[0].field[3].id[1]
+datatype[4].structtype[0].field[3].id[0].id 3
+datatype[4].structtype[0].field[3].datatype 2
+datatype[4].structtype[0].field[4].name LAST_MODIFIED
+datatype[4].structtype[0].field[4].id[1]
+datatype[4].structtype[0].field[4].id[0].id 4
+datatype[4].structtype[0].field[4].datatype 0
+datatype[4].documenttype[0]
+datatype[5].id 1001
+datatype[5].arraytype[0]
+datatype[5].weightedsettype[0]
+datatype[5].structtype[1]
+datatype[5].structtype[0].name crawler_type.body
+datatype[5].structtype[0].version 0
+datatype[5].structtype[0].field[0]
+datatype[5].documenttype[0]
+datatype[6].id 1002
+datatype[6].arraytype[0]
+datatype[6].weightedsettype[0]
+datatype[6].structtype[0]
+datatype[6].documenttype[1]
+datatype[6].documenttype[0].name crawler_type
+datatype[6].documenttype[0].version 0
+datatype[6].documenttype[0].inherits[0]
+datatype[6].documenttype[0].headerstruct 1000
+datatype[6].documenttype[0].bodystruct 1001
+datatype[7].id 2000
+datatype[7].arraytype[0]
+datatype[7].weightedsettype[0]
+datatype[7].structtype[1]
+datatype[7].structtype[0].name multimedia_type.header
+datatype[7].structtype[0].version 0
+datatype[7].structtype[0].field[7]
+datatype[7].structtype[0].field[0].name URL_KEYWORDS
+datatype[7].structtype[0].field[0].id[1]
+datatype[7].structtype[0].field[0].id[0].id 147
+datatype[7].structtype[0].field[0].datatype 102
+datatype[7].structtype[0].field[1].name MULTIMEDIA_LINKTYPES
+datatype[7].structtype[0].field[1].id[1]
+datatype[7].structtype[0].field[1].id[0].id 148
+datatype[7].structtype[0].field[1].datatype 102
+datatype[7].structtype[0].field[2].name THUMBNAIL_URLS
+datatype[7].structtype[0].field[2].id[1]
+datatype[7].structtype[0].field[2].id[0].id 149
+datatype[7].structtype[0].field[2].datatype 102
+datatype[7].structtype[0].field[3].name FROM_LINK_LIST
+datatype[7].structtype[0].field[3].id[1]
+datatype[7].structtype[0].field[3].id[0].id 150
+datatype[7].structtype[0].field[3].datatype 101
+datatype[7].structtype[0].field[4].name FROM_LINK_LANGUAGE_LIST
+datatype[7].structtype[0].field[4].id[1]
+datatype[7].structtype[0].field[4].id[0].id 151
+datatype[7].structtype[0].field[4].datatype 102
+datatype[7].structtype[0].field[5].name TESTSTRUCT1
+datatype[7].structtype[0].field[5].id[1]
+datatype[7].structtype[0].field[5].id[0].id 153
+datatype[7].structtype[0].field[5].datatype 103
+datatype[7].structtype[0].field[6].name TESTSTRUCT2
+datatype[7].structtype[0].field[6].id[1]
+datatype[7].structtype[0].field[6].id[0].id 154
+datatype[7].structtype[0].field[6].datatype 104
+datatype[7].documenttype[0]
+datatype[8].id 2001
+datatype[8].structtype[1]
+datatype[8].structtype[0].name multimedia_type.body
+datatype[8].structtype[0].version 0
+datatype[8].structtype[0].field[0]
+datatype[9].id 2002
+datatype[9].documenttype[1]
+datatype[9].documenttype[0].name multimedia_type
+datatype[9].documenttype[0].version 0
+datatype[9].documenttype[0].inherits[1]
+datatype[9].documenttype[0].inherits[0].name crawler_type
+datatype[9].documenttype[0].inherits[0].version 0
+datatype[9].documenttype[0].headerstruct 2000
+datatype[9].documenttype[0].bodystruct 2001
+datatype[10].id 2003
+datatype[10].maptype[1]
+datatype[10].maptype[0].keytype 2 
+datatype[10].maptype[0].valtype 2
+datatype[11].id 2004
+datatype[11].maptype[1]
+datatype[11].maptype[0].keytype 0
+datatype[11].maptype[0].valtype 0
+datatype[12].id 2005
+datatype[12].maptype[1]
+datatype[12].maptype[0].keytype 2003
+datatype[12].maptype[0].valtype 2004
+datatype[13].id 2006
+datatype[13].structtype[1]
+datatype[13].structtype[0].name map.header
+datatype[13].structtype[0].version 0
+datatype[13].structtype[0].field[1]
+datatype[13].structtype[0].field[0].name mymap
+datatype[13].structtype[0].field[0].id[1]
+datatype[13].structtype[0].field[0].id[0].id 155
+datatype[13].structtype[0].field[0].datatype 2005
+datatype[14].id 2007
+datatype[14].structtype[1]
+datatype[14].structtype[0].name map.body
+datatype[14].structtype[0].version 0
+datatype[14].structtype[0].field[0]
+datatype[15].id 2008
+datatype[15].documenttype[1]
+datatype[15].documenttype[0].name mapdoc
+datatype[15].documenttype[0].version 0
+datatype[15].documenttype[0].headerstruct 2006
+datatype[15].documenttype[0].bodystruct 2007
diff --git a/document/src/tests/data/docselectcfg1.txt b/document/src/tests/data/docselectcfg1.txt
new file mode 100644
index 00000000000..aeb20ea42ea
--- /dev/null
+++ b/document/src/tests/data/docselectcfg1.txt
@@ -0,0 +1,16 @@
+documenttype testdoc
+documenttypeversion -1
+documentselect[3]
+documentselect[0].type and
+documentselect[0].subcount 2
+documentselect[0].field ""
+documentselect[0].value ""
+documentselect[1].type equals
+documentselect[1].subcount 0
+documentselect[1].field "stringattr"
+documentselect[1].value "hei"
+documentselect[2].type equals
+documentselect[2].subcount 0
+documentselect[2].field "intattr"
+documentselect[2].value "100"
+
diff --git a/document/src/tests/data/docselectcfg2.txt b/document/src/tests/data/docselectcfg2.txt
new file mode 100644
index 00000000000..b96e12331f1
--- /dev/null
+++ b/document/src/tests/data/docselectcfg2.txt
@@ -0,0 +1,35 @@
+documenttype testdoc
+documenttypeversion 0
+documentselect[8]
+documentselect[0].type or
+documentselect[0].subcount 2
+documentselect[0].field ""
+documentselect[0].value ""
+documentselect[1].type and
+documentselect[1].subcount 2
+documentselect[1].field ""
+documentselect[1].value ""
+documentselect[2].type not
+documentselect[2].subcount 1
+documentselect[2].field ""
+documentselect[2].value ""
+documentselect[3].type equals
+documentselect[3].subcount 0
+documentselect[3].field "stringattr"
+documentselect[3].value "hei"
+documentselect[4].type equals
+documentselect[4].subcount 0
+documentselect[4].field "intattr"
+documentselect[4].value "100"
+documentselect[5].type and
+documentselect[5].subcount 2
+documentselect[5].field ""
+documentselect[5].value ""
+documentselect[6].type equals
+documentselect[6].subcount 0
+documentselect[6].field "URI"
+documentselect[6].value "http://www.ntnu.no/"
+documentselect[7].type lt
+documentselect[7].subcount 0
+documentselect[7].field "floatattr"
+documentselect[7].value "100.5"
diff --git a/document/src/tests/data/doctypesconfigtest.cfg b/document/src/tests/data/doctypesconfigtest.cfg
new file mode 100644
index 00000000000..36666bc7f3e
--- /dev/null
+++ b/document/src/tests/data/doctypesconfigtest.cfg
@@ -0,0 +1,184 @@
+enablecompression false
+documenttype[3]
+documenttype[0].id -332662462
+documenttype[0].name "base1"
+documenttype[0].version 0
+documenttype[0].headerstruct 762677181
+documenttype[0].bodystruct -1448787694
+documenttype[0].inherits[0]
+documenttype[0].datatype[3]
+documenttype[0].datatype[0].id 1000
+documenttype[0].datatype[0].type ARRAY
+documenttype[0].datatype[0].array.element.id 0
+documenttype[0].datatype[0].map.key.id 0
+documenttype[0].datatype[0].map.value.id 0
+documenttype[0].datatype[0].wset.key.id 0
+documenttype[0].datatype[0].wset.createifnonexistent false
+documenttype[0].datatype[0].wset.removeifzero false
+documenttype[0].datatype[0].annotationref.annotation.id 0
+documenttype[0].datatype[0].sstruct.name ""
+documenttype[0].datatype[0].sstruct.version 0
+documenttype[0].datatype[0].sstruct.compression.type NONE
+documenttype[0].datatype[0].sstruct.compression.level 0
+documenttype[0].datatype[0].sstruct.compression.threshold 95
+documenttype[0].datatype[0].sstruct.compression.minsize 200
+documenttype[0].datatype[0].sstruct.field[0]
+documenttype[0].datatype[1].id 762677181
+documenttype[0].datatype[1].type STRUCT
+documenttype[0].datatype[1].array.element.id 0
+documenttype[0].datatype[1].map.key.id 0
+documenttype[0].datatype[1].map.value.id 0
+documenttype[0].datatype[1].wset.key.id 0
+documenttype[0].datatype[1].wset.createifnonexistent false
+documenttype[0].datatype[1].wset.removeifzero false
+documenttype[0].datatype[1].annotationref.annotation.id 0
+documenttype[0].datatype[1].sstruct.name "base1.header"
+documenttype[0].datatype[1].sstruct.version 0
+documenttype[0].datatype[1].sstruct.compression.type NONE
+documenttype[0].datatype[1].sstruct.compression.level 0
+documenttype[0].datatype[1].sstruct.compression.threshold 90
+documenttype[0].datatype[1].sstruct.compression.minsize 0
+documenttype[0].datatype[1].sstruct.field[3]
+documenttype[0].datatype[1].sstruct.field[0].name "field1"
+documenttype[0].datatype[1].sstruct.field[0].id 36262354
+documenttype[0].datatype[1].sstruct.field[0].id_v6 1141913055
+documenttype[0].datatype[1].sstruct.field[0].datatype 0
+documenttype[0].datatype[1].sstruct.field[1].name "field2"
+documenttype[0].datatype[1].sstruct.field[1].id 1388988771
+documenttype[0].datatype[1].sstruct.field[1].id_v6 1215205431
+documenttype[0].datatype[1].sstruct.field[1].datatype 0
+documenttype[0].datatype[1].sstruct.field[2].name "fieldarray1"
+documenttype[0].datatype[1].sstruct.field[2].id 1030920758
+documenttype[0].datatype[1].sstruct.field[2].id_v6 1073985444
+documenttype[0].datatype[1].sstruct.field[2].datatype 1000
+documenttype[0].datatype[2].id -1448787694
+documenttype[0].datatype[2].type STRUCT
+documenttype[0].datatype[2].array.element.id 0
+documenttype[0].datatype[2].map.key.id 0
+documenttype[0].datatype[2].map.value.id 0
+documenttype[0].datatype[2].wset.key.id 0
+documenttype[0].datatype[2].wset.createifnonexistent false
+documenttype[0].datatype[2].wset.removeifzero false
+documenttype[0].datatype[2].annotationref.annotation.id 0
+documenttype[0].datatype[2].sstruct.name "base1.body"
+documenttype[0].datatype[2].sstruct.version 0
+documenttype[0].datatype[2].sstruct.compression.type NONE
+documenttype[0].datatype[2].sstruct.compression.level 0
+documenttype[0].datatype[2].sstruct.compression.threshold 90
+documenttype[0].datatype[2].sstruct.compression.minsize 0
+documenttype[0].datatype[2].sstruct.field[0]
+documenttype[0].annotationtype[0]
+documenttype[1].id -332661500
+documenttype[1].name "base2"
+documenttype[1].version 1
+documenttype[1].headerstruct 566163677
+documenttype[1].bodystruct 294022642
+documenttype[1].inherits[0]
+documenttype[1].datatype[2]
+documenttype[1].datatype[0].id 566163677
+documenttype[1].datatype[0].type STRUCT
+documenttype[1].datatype[0].array.element.id 0
+documenttype[1].datatype[0].map.key.id 0
+documenttype[1].datatype[0].map.value.id 0
+documenttype[1].datatype[0].wset.key.id 0
+documenttype[1].datatype[0].wset.createifnonexistent false
+documenttype[1].datatype[0].wset.removeifzero false
+documenttype[1].datatype[0].annotationref.annotation.id 0
+documenttype[1].datatype[0].sstruct.name "base2.header"
+documenttype[1].datatype[0].sstruct.version 1
+documenttype[1].datatype[0].sstruct.compression.type NONE
+documenttype[1].datatype[0].sstruct.compression.level 0
+documenttype[1].datatype[0].sstruct.compression.threshold 90
+documenttype[1].datatype[0].sstruct.compression.minsize 0
+documenttype[1].datatype[0].sstruct.field[1]
+documenttype[1].datatype[0].sstruct.field[0].name "field4"
+documenttype[1].datatype[0].sstruct.field[0].id 1556675886
+documenttype[1].datatype[0].sstruct.field[0].id_v6 1677332587
+documenttype[1].datatype[0].sstruct.field[0].datatype 2
+documenttype[1].datatype[1].id 294022642
+documenttype[1].datatype[1].type STRUCT
+documenttype[1].datatype[1].array.element.id 0
+documenttype[1].datatype[1].map.key.id 0
+documenttype[1].datatype[1].map.value.id 0
+documenttype[1].datatype[1].wset.key.id 0
+documenttype[1].datatype[1].wset.createifnonexistent false
+documenttype[1].datatype[1].wset.removeifzero false
+documenttype[1].datatype[1].annotationref.annotation.id 0
+documenttype[1].datatype[1].sstruct.name "base2.body"
+documenttype[1].datatype[1].sstruct.version 1
+documenttype[1].datatype[1].sstruct.compression.type NONE
+documenttype[1].datatype[1].sstruct.compression.level 0
+documenttype[1].datatype[1].sstruct.compression.threshold 90
+documenttype[1].datatype[1].sstruct.compression.minsize 0
+documenttype[1].datatype[1].sstruct.field[1]
+documenttype[1].datatype[1].sstruct.field[0].name "field3"
+documenttype[1].datatype[1].sstruct.field[0].id 722536103
+documenttype[1].datatype[1].sstruct.field[0].id_v6 1036475953
+documenttype[1].datatype[1].sstruct.field[0].datatype 2
+documenttype[1].annotationtype[0]
+documenttype[2].id 787712161
+documenttype[2].name "derived"
+documenttype[2].version 2
+documenttype[2].headerstruct 296510914
+documenttype[2].bodystruct -1359887401
+documenttype[2].inherits[2]
+documenttype[2].inherits[0].id -332662462
+documenttype[2].inherits[1].id -332661500
+documenttype[2].datatype[2]
+documenttype[2].datatype[0].id 296510914
+documenttype[2].datatype[0].type STRUCT
+documenttype[2].datatype[0].array.element.id 0
+documenttype[2].datatype[0].map.key.id 0
+documenttype[2].datatype[0].map.value.id 0
+documenttype[2].datatype[0].wset.key.id 0
+documenttype[2].datatype[0].wset.createifnonexistent false
+documenttype[2].datatype[0].wset.removeifzero false
+documenttype[2].datatype[0].annotationref.annotation.id 0
+documenttype[2].datatype[0].sstruct.name "derived.header"
+documenttype[2].datatype[0].sstruct.version 2
+documenttype[2].datatype[0].sstruct.compression.type NONE
+documenttype[2].datatype[0].sstruct.compression.level 0
+documenttype[2].datatype[0].sstruct.compression.threshold 90
+documenttype[2].datatype[0].sstruct.compression.minsize 0
+documenttype[2].datatype[0].sstruct.field[4]
+documenttype[2].datatype[0].sstruct.field[0].name "field1"
+documenttype[2].datatype[0].sstruct.field[0].id 36262354
+documenttype[2].datatype[0].sstruct.field[0].id_v6 1141913055
+documenttype[2].datatype[0].sstruct.field[0].datatype 0
+documenttype[2].datatype[0].sstruct.field[1].name "field2"
+documenttype[2].datatype[0].sstruct.field[1].id 1388988771
+documenttype[2].datatype[0].sstruct.field[1].id_v6 1215205431
+documenttype[2].datatype[0].sstruct.field[1].datatype 0
+documenttype[2].datatype[0].sstruct.field[2].name "field4"
+documenttype[2].datatype[0].sstruct.field[2].id 1556675886
+documenttype[2].datatype[0].sstruct.field[2].id_v6 1677332587
+documenttype[2].datatype[0].sstruct.field[2].datatype 2
+documenttype[2].datatype[0].sstruct.field[3].name "fieldarray1"
+documenttype[2].datatype[0].sstruct.field[3].id 1030920758
+documenttype[2].datatype[0].sstruct.field[3].id_v6 1073985444
+documenttype[2].datatype[0].sstruct.field[3].datatype 1000
+documenttype[2].datatype[1].id -1359887401
+documenttype[2].datatype[1].type STRUCT
+documenttype[2].datatype[1].array.element.id 0
+documenttype[2].datatype[1].map.key.id 0
+documenttype[2].datatype[1].map.value.id 0
+documenttype[2].datatype[1].wset.key.id 0
+documenttype[2].datatype[1].wset.createifnonexistent false
+documenttype[2].datatype[1].wset.removeifzero false
+documenttype[2].datatype[1].annotationref.annotation.id 0
+documenttype[2].datatype[1].sstruct.name "derived.body"
+documenttype[2].datatype[1].sstruct.version 2
+documenttype[2].datatype[1].sstruct.compression.type NONE
+documenttype[2].datatype[1].sstruct.compression.level 0
+documenttype[2].datatype[1].sstruct.compression.threshold 90
+documenttype[2].datatype[1].sstruct.compression.minsize 0
+documenttype[2].datatype[1].sstruct.field[2]
+documenttype[2].datatype[1].sstruct.field[0].name "field3"
+documenttype[2].datatype[1].sstruct.field[0].id 722536103
+documenttype[2].datatype[1].sstruct.field[0].id_v6 1036475953
+documenttype[2].datatype[1].sstruct.field[0].datatype 2
+documenttype[2].datatype[1].sstruct.field[1].name "field5"
+documenttype[2].datatype[1].sstruct.field[1].id 1352306621
+documenttype[2].datatype[1].sstruct.field[1].id_v6 1511574032
+documenttype[2].datatype[1].sstruct.field[1].datatype 1
+documenttype[2].annotationtype[0]
diff --git a/document/src/tests/data/document-cpp-currentversion-lz4-9.dat b/document/src/tests/data/document-cpp-currentversion-lz4-9.dat
new file mode 100644
index 00000000000..5f31ac0ab78
--- /dev/null
+++ b/document/src/tests/data/document-cpp-currentversion-lz4-9.dat
diff --git a/document/src/tests/data/document-cpp-currentversion-uncompressed.dat b/document/src/tests/data/document-cpp-currentversion-uncompressed.dat
new file mode 100644
index 00000000000..6a5c5e90b84
--- /dev/null
+++ b/document/src/tests/data/document-cpp-currentversion-uncompressed.dat
diff --git a/document/src/tests/data/document-cpp-v7-uncompressed.dat b/document/src/tests/data/document-cpp-v7-uncompressed.dat
new file mode 100644
index 00000000000..5f073e33438
--- /dev/null
+++ b/document/src/tests/data/document-cpp-v7-uncompressed.dat
diff --git a/document/src/tests/data/document-cpp-v8-uncompressed.dat b/document/src/tests/data/document-cpp-v8-uncompressed.dat
new file mode 100644
index 00000000000..429334fc102
--- /dev/null
+++ b/document/src/tests/data/document-cpp-v8-uncompressed.dat
diff --git a/document/src/tests/data/embeddeddocument.cfg b/document/src/tests/data/embeddeddocument.cfg
new file mode 100644
index 00000000000..baefc0d7338
--- /dev/null
+++ b/document/src/tests/data/embeddeddocument.cfg
@@ -0,0 +1,102 @@
+enablecompression false
+datatype[8]
+datatype[0].id 1001
+datatype[0].arraytype[1]
+datatype[0].arraytype[0].datatype 1
+datatype[0].weightedsettype[0]
+datatype[0].structtype[0]
+datatype[0].documenttype[0]
+datatype[1].id 2001
+datatype[1].arraytype[1]
+datatype[1].arraytype[0].datatype 1001
+datatype[1].weightedsettype[0]
+datatype[1].structtype[0]
+datatype[1].documenttype[0]
+datatype[2].id -1686125086
+datatype[2].arraytype[0]
+datatype[2].weightedsettype[0]
+datatype[2].structtype[1]
+datatype[2].structtype[0].name docindoc.header
+datatype[2].structtype[0].version 0
+datatype[2].structtype[0].field[0]
+datatype[2].documenttype[0]
+datatype[3].id 2030224503
+datatype[3].arraytype[0]
+datatype[3].weightedsettype[0]
+datatype[3].structtype[1]
+datatype[3].structtype[0].name docindoc.body
+datatype[3].structtype[0].version 0
+datatype[3].structtype[0].field[1]
+datatype[3].structtype[0].field[0].name stringindoc
+datatype[3].structtype[0].field[0].id[0]
+datatype[3].structtype[0].field[0].datatype 2
+datatype[3].documenttype[0]
+datatype[4].id 1447635645
+datatype[4].arraytype[0]
+datatype[4].weightedsettype[0]
+datatype[4].structtype[0]
+datatype[4].documenttype[1]
+datatype[4].documenttype[0].name docindoc
+datatype[4].documenttype[0].version 0
+datatype[4].documenttype[0].inherits[0]
+datatype[4].documenttype[0].headerstruct -1686125086
+datatype[4].documenttype[0].bodystruct 2030224503
+datatype[5].id -260050933
+datatype[5].arraytype[0]
+datatype[5].weightedsettype[0]
+datatype[5].structtype[1]
+datatype[5].structtype[0].name serializetest.header
+datatype[5].structtype[0].version 0
+datatype[5].structtype[0].field[4]
+datatype[5].structtype[0].field[0].name floatfield
+datatype[5].structtype[0].field[0].id[0]
+datatype[5].structtype[0].field[0].datatype 1
+datatype[5].structtype[0].field[1].name stringfield
+datatype[5].structtype[0].field[1].id[0]
+datatype[5].structtype[0].field[1].datatype 2
+datatype[5].structtype[0].field[2].name longfield
+datatype[5].structtype[0].field[2].id[0]
+datatype[5].structtype[0].field[2].datatype 4
+datatype[5].structtype[0].field[3].name urifield
+datatype[5].structtype[0].field[3].id[0]
+datatype[5].structtype[0].field[3].datatype 10
+datatype[5].documenttype[0]
+datatype[6].id 1026122976
+datatype[6].arraytype[0]
+datatype[6].weightedsettype[0]
+datatype[6].structtype[1]
+datatype[6].structtype[0].name serializetest.body
+datatype[6].structtype[0].version 0
+datatype[6].structtype[0].field[7]
+datatype[6].structtype[0].field[0].name intfield
+datatype[6].structtype[0].field[0].id[0]
+datatype[6].structtype[0].field[0].datatype 0
+datatype[6].structtype[0].field[1].name rawfield
+datatype[6].structtype[0].field[1].id[0]
+datatype[6].structtype[0].field[1].datatype 3
+datatype[6].structtype[0].field[2].name doublefield
+datatype[6].structtype[0].field[2].id[0]
+datatype[6].structtype[0].field[2].datatype 5
+datatype[6].structtype[0].field[3].name bytefield
+datatype[6].structtype[0].field[3].id[0]
+datatype[6].structtype[0].field[3].datatype 16
+datatype[6].structtype[0].field[4].name arrayoffloatfield
+datatype[6].structtype[0].field[4].id[0]
+datatype[6].structtype[0].field[4].datatype 1001
+datatype[6].structtype[0].field[5].name arrayofarrayoffloatfield
+datatype[6].structtype[0].field[5].id[0]
+datatype[6].structtype[0].field[5].datatype 2001
+datatype[6].structtype[0].field[6].name docfield
+datatype[6].structtype[0].field[6].id[0]
+datatype[6].structtype[0].field[6].datatype 8
+datatype[6].documenttype[0]
+datatype[7].id 1306012852
+datatype[7].arraytype[0]
+datatype[7].weightedsettype[0]
+datatype[7].structtype[0]
+datatype[7].documenttype[1]
+datatype[7].documenttype[0].name serializetest
+datatype[7].documenttype[0].version 0
+datatype[7].documenttype[0].inherits[0]
+datatype[7].documenttype[0].headerstruct -260050933
+datatype[7].documenttype[0].bodystruct 1026122976
diff --git a/document/src/tests/data/inheritancetest.cfg b/document/src/tests/data/inheritancetest.cfg
new file mode 100644
index 00000000000..1b2a704de92
--- /dev/null
+++ b/document/src/tests/data/inheritancetest.cfg
@@ -0,0 +1,354 @@
+enablecompression false
+documenttype[5]
+documenttype[0].bodystruct 380234823
+documenttype[0].headerstruct 1811691954
+documenttype[0].id -602119955
+documenttype[0].name "common"
+documenttype[0].version 0
+documenttype[0].annotationtype[0]
+documenttype[0].datatype[3]
+documenttype[0].datatype[0].id 1811691954
+documenttype[0].datatype[0].type STRUCT
+documenttype[0].datatype[0].sstruct.name "common.header"
+documenttype[0].datatype[0].sstruct.version 0
+documenttype[0].datatype[0].sstruct.field[3]
+documenttype[0].datatype[0].sstruct.field[0].datatype 0
+documenttype[0].datatype[0].sstruct.field[0].id 1501857786
+documenttype[0].datatype[0].sstruct.field[0].id_v6 1023904001
+documenttype[0].datatype[0].sstruct.field[0].name "popularity"
+documenttype[0].datatype[0].sstruct.field[1].datatype 2
+documenttype[0].datatype[0].sstruct.field[1].id 567626448
+documenttype[0].datatype[0].sstruct.field[1].id_v6 29129762
+documenttype[0].datatype[0].sstruct.field[1].name "title"
+documenttype[0].datatype[0].sstruct.field[2].datatype 0
+documenttype[0].datatype[0].sstruct.field[2].id 624630872
+documenttype[0].datatype[0].sstruct.field[2].id_v6 1508520108
+documenttype[0].datatype[0].sstruct.field[2].name "year"
+documenttype[0].datatype[1].id 380234823
+documenttype[0].datatype[1].type STRUCT
+documenttype[0].datatype[1].sstruct.name "common.body"
+documenttype[0].datatype[1].sstruct.version 0
+documenttype[0].datatype[1].sstruct.field[1]
+documenttype[0].datatype[1].sstruct.field[0].datatype 1
+documenttype[0].datatype[1].sstruct.field[0].id 1225171018
+documenttype[0].datatype[1].sstruct.field[0].id_v6 205748629
+documenttype[0].datatype[1].sstruct.field[0].name "price"
+documenttype[0].datatype[2].id 874542262
+documenttype[0].datatype[2].type STRUCT
+documenttype[0].datatype[2].sstruct.name "address"
+documenttype[0].datatype[2].sstruct.version 0
+documenttype[0].datatype[2].sstruct.field[2]
+documenttype[0].datatype[2].sstruct.field[0].datatype 2
+documenttype[0].datatype[2].sstruct.field[0].id 725636280
+documenttype[0].datatype[2].sstruct.field[0].id_v6 2127904
+documenttype[0].datatype[2].sstruct.field[0].name "city"
+documenttype[0].datatype[2].sstruct.field[1].datatype 2
+documenttype[0].datatype[2].sstruct.field[1].id 1954764240
+documenttype[0].datatype[2].sstruct.field[1].id_v6 1527533866
+documenttype[0].datatype[2].sstruct.field[1].name "street"
+documenttype[0].inherits[1]
+documenttype[0].inherits[0].id 8
+documenttype[1].bodystruct 993120973
+documenttype[1].headerstruct -1910204744
+documenttype[1].id 1412693671
+documenttype[1].name "music"
+documenttype[1].version 0
+documenttype[1].annotationtype[0]
+documenttype[1].datatype[5]
+documenttype[1].datatype[0].id 1328286588
+documenttype[1].datatype[0].type WSET
+documenttype[1].datatype[0].wset.createifnonexistent false
+documenttype[1].datatype[0].wset.removeifzero false
+documenttype[1].datatype[0].wset.key.id 2
+documenttype[1].datatype[1].id 993120973
+documenttype[1].datatype[1].type STRUCT
+documenttype[1].datatype[1].sstruct.name "music.body"
+documenttype[1].datatype[1].sstruct.version 0
+documenttype[1].datatype[1].sstruct.field[1]
+documenttype[1].datatype[1].sstruct.field[0].datatype 5
+documenttype[1].datatype[1].sstruct.field[0].id 552396752
+documenttype[1].datatype[1].sstruct.field[0].id_v6 713868363
+documenttype[1].datatype[1].sstruct.field[0].name "randomdouble"
+documenttype[1].datatype[2].id 874542262
+documenttype[1].datatype[2].type STRUCT
+documenttype[1].datatype[2].sstruct.name "address"
+documenttype[1].datatype[2].sstruct.version 0
+documenttype[1].datatype[2].sstruct.field[2]
+documenttype[1].datatype[2].sstruct.field[0].datatype 2
+documenttype[1].datatype[2].sstruct.field[0].id 725636280
+documenttype[1].datatype[2].sstruct.field[0].id_v6 2127904
+documenttype[1].datatype[2].sstruct.field[0].name "city"
+documenttype[1].datatype[2].sstruct.field[1].datatype 2
+documenttype[1].datatype[2].sstruct.field[1].id 1954764240
+documenttype[1].datatype[2].sstruct.field[1].id_v6 1527533866
+documenttype[1].datatype[2].sstruct.field[1].name "street"
+documenttype[1].datatype[3].id -1857943347
+documenttype[1].datatype[3].type ARRAY
+documenttype[1].datatype[3].array.element.id 874542262
+documenttype[1].datatype[4].id -1910204744
+documenttype[1].datatype[4].type STRUCT
+documenttype[1].datatype[4].sstruct.name "music.header"
+documenttype[1].datatype[4].sstruct.version 0
+documenttype[1].datatype[4].sstruct.field[6]
+documenttype[1].datatype[4].sstruct.field[0].datatype -1857943347
+documenttype[1].datatype[4].sstruct.field[0].id 785651450
+documenttype[1].datatype[4].sstruct.field[0].id_v6 1210851114
+documenttype[1].datatype[4].sstruct.field[0].name "addresses"
+documenttype[1].datatype[4].sstruct.field[1].datatype 1328286588
+documenttype[1].datatype[4].sstruct.field[1].id 1692189119
+documenttype[1].datatype[4].sstruct.field[1].id_v6 866139998
+documenttype[1].datatype[4].sstruct.field[1].name "albums"
+documenttype[1].datatype[4].sstruct.field[2].datatype 2
+documenttype[1].datatype[4].sstruct.field[2].id 51903611
+documenttype[1].datatype[4].sstruct.field[2].id_v6 1854279480
+documenttype[1].datatype[4].sstruct.field[2].name "artist"
+documenttype[1].datatype[4].sstruct.field[3].datatype 2
+documenttype[1].datatype[4].sstruct.field[3].id 306708139
+documenttype[1].datatype[4].sstruct.field[3].id_v6 1047427207
+documenttype[1].datatype[4].sstruct.field[3].name "phrase"
+documenttype[1].datatype[4].sstruct.field[4].datatype 874542262
+documenttype[1].datatype[4].sstruct.field[4].id 776401886
+documenttype[1].datatype[4].sstruct.field[4].id_v6 1741595836
+documenttype[1].datatype[4].sstruct.field[4].name "randomAddr"
+documenttype[1].datatype[4].sstruct.field[5].datatype 10
+documenttype[1].datatype[4].sstruct.field[5].id 2095970198
+documenttype[1].datatype[4].sstruct.field[5].id_v6 1826424031
+documenttype[1].datatype[4].sstruct.field[5].name "url"
+documenttype[1].inherits[2]
+documenttype[1].inherits[0].id -602119955
+documenttype[1].inherits[1].id 8
+documenttype[2].bodystruct -727249584
+documenttype[2].headerstruct -1623901061
+documenttype[2].id 1722744388
+documenttype[2].name "music_search"
+documenttype[2].version 0
+documenttype[2].annotationtype[0]
+documenttype[2].datatype[5]
+documenttype[2].datatype[0].id 1328286588
+documenttype[2].datatype[0].type WSET
+documenttype[2].datatype[0].wset.createifnonexistent false
+documenttype[2].datatype[0].wset.removeifzero false
+documenttype[2].datatype[0].wset.key.id 2
+documenttype[2].datatype[1].id 874542262
+documenttype[2].datatype[1].type STRUCT
+documenttype[2].datatype[1].sstruct.name "address"
+documenttype[2].datatype[1].sstruct.version 0
+documenttype[2].datatype[1].sstruct.field[2]
+documenttype[2].datatype[1].sstruct.field[0].datatype 2
+documenttype[2].datatype[1].sstruct.field[0].id 725636280
+documenttype[2].datatype[1].sstruct.field[0].id_v6 2127904
+documenttype[2].datatype[1].sstruct.field[0].name "city"
+documenttype[2].datatype[1].sstruct.field[1].datatype 2
+documenttype[2].datatype[1].sstruct.field[1].id 1954764240
+documenttype[2].datatype[1].sstruct.field[1].id_v6 1527533866
+documenttype[2].datatype[1].sstruct.field[1].name "street"
+documenttype[2].datatype[2].id -727249584
+documenttype[2].datatype[2].type STRUCT
+documenttype[2].datatype[2].sstruct.name "music_search.body"
+documenttype[2].datatype[2].sstruct.version 0
+documenttype[2].datatype[2].sstruct.field[2]
+documenttype[2].datatype[2].sstruct.field[0].datatype 1
+documenttype[2].datatype[2].sstruct.field[0].id 1225171018
+documenttype[2].datatype[2].sstruct.field[0].id_v6 205748629
+documenttype[2].datatype[2].sstruct.field[0].name "price"
+documenttype[2].datatype[2].sstruct.field[1].datatype 5
+documenttype[2].datatype[2].sstruct.field[1].id 552396752
+documenttype[2].datatype[2].sstruct.field[1].id_v6 713868363
+documenttype[2].datatype[2].sstruct.field[1].name "randomdouble"
+documenttype[2].datatype[3].id -1857943347
+documenttype[2].datatype[3].type ARRAY
+documenttype[2].datatype[3].array.element.id 874542262
+documenttype[2].datatype[4].id -1623901061
+documenttype[2].datatype[4].type STRUCT
+documenttype[2].datatype[4].sstruct.name "music_search.header"
+documenttype[2].datatype[4].sstruct.version 0
+documenttype[2].datatype[4].sstruct.field[11]
+documenttype[2].datatype[4].sstruct.field[00].datatype -1857943347
+documenttype[2].datatype[4].sstruct.field[00].id 785651450
+documenttype[2].datatype[4].sstruct.field[00].id_v6 1210851114
+documenttype[2].datatype[4].sstruct.field[00].name "addresses"
+documenttype[2].datatype[4].sstruct.field[01].datatype 1328286588
+documenttype[2].datatype[4].sstruct.field[01].id 1692189119
+documenttype[2].datatype[4].sstruct.field[01].id_v6 866139998
+documenttype[2].datatype[4].sstruct.field[01].name "albums"
+documenttype[2].datatype[4].sstruct.field[02].datatype 2
+documenttype[2].datatype[4].sstruct.field[02].id 51903611
+documenttype[2].datatype[4].sstruct.field[02].id_v6 1854279480
+documenttype[2].datatype[4].sstruct.field[02].name "artist"
+documenttype[2].datatype[4].sstruct.field[03].datatype 2
+documenttype[2].datatype[4].sstruct.field[03].id 306708139
+documenttype[2].datatype[4].sstruct.field[03].id_v6 1047427207
+documenttype[2].datatype[4].sstruct.field[03].name "phrase"
+documenttype[2].datatype[4].sstruct.field[04].datatype 0
+documenttype[2].datatype[4].sstruct.field[04].id 1501857786
+documenttype[2].datatype[4].sstruct.field[04].id_v6 1023904001
+documenttype[2].datatype[4].sstruct.field[04].name "popularity"
+documenttype[2].datatype[4].sstruct.field[05].datatype 874542262
+documenttype[2].datatype[4].sstruct.field[05].id 776401886
+documenttype[2].datatype[4].sstruct.field[05].id_v6 1741595836
+documenttype[2].datatype[4].sstruct.field[05].name "randomAddr"
+documenttype[2].datatype[4].sstruct.field[06].datatype 2
+documenttype[2].datatype[4].sstruct.field[06].id 1883197392
+documenttype[2].datatype[4].sstruct.field[06].id_v6 699950698
+documenttype[2].datatype[4].sstruct.field[06].name "rankfeatures"
+documenttype[2].datatype[4].sstruct.field[07].datatype 2
+documenttype[2].datatype[4].sstruct.field[07].id 1840337115
+documenttype[2].datatype[4].sstruct.field[07].id_v6 1981648971
+documenttype[2].datatype[4].sstruct.field[07].name "summaryfeatures"
+documenttype[2].datatype[4].sstruct.field[08].datatype 2
+documenttype[2].datatype[4].sstruct.field[08].id 567626448
+documenttype[2].datatype[4].sstruct.field[08].id_v6 29129762
+documenttype[2].datatype[4].sstruct.field[08].name "title"
+documenttype[2].datatype[4].sstruct.field[09].datatype 111553393
+documenttype[2].datatype[4].sstruct.field[09].id 2119414873
+documenttype[2].datatype[4].sstruct.field[09].id_v6 1826424031
+documenttype[2].datatype[4].sstruct.field[09].name "url"
+documenttype[2].datatype[4].sstruct.field[10].datatype 0
+documenttype[2].datatype[4].sstruct.field[10].id 624630872
+documenttype[2].datatype[4].sstruct.field[10].id_v6 1508520108
+documenttype[2].datatype[4].sstruct.field[10].name "year"
+documenttype[2].inherits[1]
+documenttype[2].inherits[0].id 8
+documenttype[3].bodystruct 1051291304
+documenttype[3].headerstruct -1843091501
+documenttype[3].id 64693740
+documenttype[3].name "books"
+documenttype[3].version 0
+documenttype[3].annotationtype[0]
+documenttype[3].datatype[5]
+documenttype[3].datatype[0].id 2125154557
+documenttype[3].datatype[0].type MAP
+documenttype[3].datatype[0].map.key.id 2
+documenttype[3].datatype[0].map.value.id 1
+documenttype[3].datatype[1].id 1051291304
+documenttype[3].datatype[1].type STRUCT
+documenttype[3].datatype[1].sstruct.name "books.body"
+documenttype[3].datatype[1].sstruct.version 0
+documenttype[3].datatype[1].sstruct.field[3]
+documenttype[3].datatype[1].sstruct.field[0].datatype 2125154557
+documenttype[3].datatype[1].sstruct.field[0].id 492235657
+documenttype[3].datatype[1].sstruct.field[0].id_v6 1647256015
+documenttype[3].datatype[1].sstruct.field[0].name "bindPrice"
+documenttype[3].datatype[1].sstruct.field[1].datatype 16
+documenttype[3].datatype[1].sstruct.field[1].id 1891624676
+documenttype[3].datatype[1].sstruct.field[1].id_v6 531118233
+documenttype[3].datatype[1].sstruct.field[1].name "grade"
+documenttype[3].datatype[1].sstruct.field[2].datatype 4
+documenttype[3].datatype[1].sstruct.field[2].id 782940634
+documenttype[3].datatype[1].sstruct.field[2].id_v6 698610188
+documenttype[3].datatype[1].sstruct.field[2].name "pages"
+documenttype[3].datatype[2].id 874542262
+documenttype[3].datatype[2].type STRUCT
+documenttype[3].datatype[2].sstruct.name "address"
+documenttype[3].datatype[2].sstruct.version 0
+documenttype[3].datatype[2].sstruct.field[2]
+documenttype[3].datatype[2].sstruct.field[0].datatype 2
+documenttype[3].datatype[2].sstruct.field[0].id 725636280
+documenttype[3].datatype[2].sstruct.field[0].id_v6 2127904
+documenttype[3].datatype[2].sstruct.field[0].name "city"
+documenttype[3].datatype[2].sstruct.field[1].datatype 2
+documenttype[3].datatype[2].sstruct.field[1].id 1954764240
+documenttype[3].datatype[2].sstruct.field[1].id_v6 1527533866
+documenttype[3].datatype[2].sstruct.field[1].name "street"
+documenttype[3].datatype[3].id -1857943347
+documenttype[3].datatype[3].type ARRAY
+documenttype[3].datatype[3].array.element.id 874542262
+documenttype[3].datatype[4].id -1843091501
+documenttype[3].datatype[4].type STRUCT
+documenttype[3].datatype[4].sstruct.name "books.header"
+documenttype[3].datatype[4].sstruct.version 0
+documenttype[3].datatype[4].sstruct.field[2]
+documenttype[3].datatype[4].sstruct.field[0].datatype -1857943347
+documenttype[3].datatype[4].sstruct.field[0].id 785651450
+documenttype[3].datatype[4].sstruct.field[0].id_v6 1210851114
+documenttype[3].datatype[4].sstruct.field[0].name "addresses"
+documenttype[3].datatype[4].sstruct.field[1].datatype 2
+documenttype[3].datatype[4].sstruct.field[1].id 644499292
+documenttype[3].datatype[4].sstruct.field[1].id_v6 177126295
+documenttype[3].datatype[4].sstruct.field[1].name "author"
+documenttype[3].inherits[2]
+documenttype[3].inherits[0].id -602119955
+documenttype[3].inherits[1].id 8
+documenttype[4].bodystruct -2128841451
+documenttype[4].headerstruct 66045696
+documenttype[4].id 1789857631
+documenttype[4].name "books_search"
+documenttype[4].version 0
+documenttype[4].annotationtype[0]
+documenttype[4].datatype[5]
+documenttype[4].datatype[0].id 2125154557
+documenttype[4].datatype[0].type MAP
+documenttype[4].datatype[0].map.key.id 2
+documenttype[4].datatype[0].map.value.id 1
+documenttype[4].datatype[1].id -2128841451
+documenttype[4].datatype[1].type STRUCT
+documenttype[4].datatype[1].sstruct.name "books_search.body"
+documenttype[4].datatype[1].sstruct.version 0
+documenttype[4].datatype[1].sstruct.field[3]
+documenttype[4].datatype[1].sstruct.field[0].datatype 2125154557
+documenttype[4].datatype[1].sstruct.field[0].id 492235657
+documenttype[4].datatype[1].sstruct.field[0].id_v6 1647256015
+documenttype[4].datatype[1].sstruct.field[0].name "bindPrice"
+documenttype[4].datatype[1].sstruct.field[1].datatype 16
+documenttype[4].datatype[1].sstruct.field[1].id 1891624676
+documenttype[4].datatype[1].sstruct.field[1].id_v6 531118233
+documenttype[4].datatype[1].sstruct.field[1].name "grade"
+documenttype[4].datatype[1].sstruct.field[2].datatype 1
+documenttype[4].datatype[1].sstruct.field[2].id 1225171018
+documenttype[4].datatype[1].sstruct.field[2].id_v6 205748629
+documenttype[4].datatype[1].sstruct.field[2].name "price"
+documenttype[4].datatype[2].id 874542262
+documenttype[4].datatype[2].type STRUCT
+documenttype[4].datatype[2].sstruct.name "address"
+documenttype[4].datatype[2].sstruct.version 0
+documenttype[4].datatype[2].sstruct.field[2]
+documenttype[4].datatype[2].sstruct.field[0].datatype 2
+documenttype[4].datatype[2].sstruct.field[0].id 725636280
+documenttype[4].datatype[2].sstruct.field[0].id_v6 2127904
+documenttype[4].datatype[2].sstruct.field[0].name "city"
+documenttype[4].datatype[2].sstruct.field[1].datatype 2
+documenttype[4].datatype[2].sstruct.field[1].id 1954764240
+documenttype[4].datatype[2].sstruct.field[1].id_v6 1527533866
+documenttype[4].datatype[2].sstruct.field[1].name "street"
+documenttype[4].datatype[3].id -1857943347
+documenttype[4].datatype[3].type ARRAY
+documenttype[4].datatype[3].array.element.id 874542262
+documenttype[4].datatype[4].id 66045696
+documenttype[4].datatype[4].type STRUCT
+documenttype[4].datatype[4].sstruct.name "books_search.header"
+documenttype[4].datatype[4].sstruct.version 0
+documenttype[4].datatype[4].sstruct.field[8]
+documenttype[4].datatype[4].sstruct.field[0].datatype -1857943347
+documenttype[4].datatype[4].sstruct.field[0].id 785651450
+documenttype[4].datatype[4].sstruct.field[0].id_v6 1210851114
+documenttype[4].datatype[4].sstruct.field[0].name "addresses"
+documenttype[4].datatype[4].sstruct.field[1].datatype 2
+documenttype[4].datatype[4].sstruct.field[1].id 644499292
+documenttype[4].datatype[4].sstruct.field[1].id_v6 177126295
+documenttype[4].datatype[4].sstruct.field[1].name "author"
+documenttype[4].datatype[4].sstruct.field[2].datatype 4
+documenttype[4].datatype[4].sstruct.field[2].id 782940634
+documenttype[4].datatype[4].sstruct.field[2].id_v6 698610188
+documenttype[4].datatype[4].sstruct.field[2].name "pages"
+documenttype[4].datatype[4].sstruct.field[3].datatype 0
+documenttype[4].datatype[4].sstruct.field[3].id 1501857786
+documenttype[4].datatype[4].sstruct.field[3].id_v6 1023904001
+documenttype[4].datatype[4].sstruct.field[3].name "popularity"
+documenttype[4].datatype[4].sstruct.field[4].datatype 2
+documenttype[4].datatype[4].sstruct.field[4].id 1883197392
+documenttype[4].datatype[4].sstruct.field[4].id_v6 699950698
+documenttype[4].datatype[4].sstruct.field[4].name "rankfeatures"
+documenttype[4].datatype[4].sstruct.field[5].datatype 2
+documenttype[4].datatype[4].sstruct.field[5].id 1840337115
+documenttype[4].datatype[4].sstruct.field[5].id_v6 1981648971
+documenttype[4].datatype[4].sstruct.field[5].name "summaryfeatures"
+documenttype[4].datatype[4].sstruct.field[6].datatype 2
+documenttype[4].datatype[4].sstruct.field[6].id 567626448
+documenttype[4].datatype[4].sstruct.field[6].id_v6 29129762
+documenttype[4].datatype[4].sstruct.field[6].name "title"
+documenttype[4].datatype[4].sstruct.field[7].datatype 0
+documenttype[4].datatype[4].sstruct.field[7].id 624630872
+documenttype[4].datatype[4].sstruct.field[7].id_v6 1508520108
+documenttype[4].datatype[4].sstruct.field[7].name "year"
+documenttype[4].inherits[1]
+documenttype[4].inherits[0].id 8
diff --git a/document/src/tests/data/serialize-fieldpathupdate-cpp.dat b/document/src/tests/data/serialize-fieldpathupdate-cpp.dat
new file mode 100644
index 00000000000..cf5f20f7a0c
--- /dev/null
+++ b/document/src/tests/data/serialize-fieldpathupdate-cpp.dat
diff --git a/document/src/tests/data/serialize-fieldpathupdate-java.dat b/document/src/tests/data/serialize-fieldpathupdate-java.dat
new file mode 100644
index 00000000000..cf5f20f7a0c
--- /dev/null
+++ b/document/src/tests/data/serialize-fieldpathupdate-java.dat
diff --git a/document/src/tests/data/serializejava-compressed.dat b/document/src/tests/data/serializejava-compressed.dat
new file mode 100644
index 00000000000..5c8721c097c
--- /dev/null
+++ b/document/src/tests/data/serializejava-compressed.dat
diff --git a/document/src/tests/data/serializejava.dat b/document/src/tests/data/serializejava.dat
new file mode 100644
index 00000000000..3dd9f8fcd52
--- /dev/null
+++ b/document/src/tests/data/serializejava.dat
diff --git a/document/src/tests/data/serializejavawithannotations.dat b/document/src/tests/data/serializejavawithannotations.dat
new file mode 100644
index 00000000000..a648b72404f
--- /dev/null
+++ b/document/src/tests/data/serializejavawithannotations.dat
diff --git a/document/src/tests/data/serializeupdatecpp.dat b/document/src/tests/data/serializeupdatecpp.dat
new file mode 100644
index 00000000000..73d83b2ab45
--- /dev/null
+++ b/document/src/tests/data/serializeupdatecpp.dat
diff --git a/document/src/tests/data/serializeupdatejava.dat b/document/src/tests/data/serializeupdatejava.dat
new file mode 100644
index 00000000000..e2a98d42fb1
--- /dev/null
+++ b/document/src/tests/data/serializeupdatejava.dat
diff --git a/document/src/tests/data/serializev6.dat b/document/src/tests/data/serializev6.dat
new file mode 100644
index 00000000000..555acb9ecc7
--- /dev/null
+++ b/document/src/tests/data/serializev6.dat
diff --git a/document/src/tests/data/variablesizedocument.cfg b/document/src/tests/data/variablesizedocument.cfg
new file mode 100644
index 00000000000..959f6b7b12d
--- /dev/null
+++ b/document/src/tests/data/variablesizedocument.cfg
@@ -0,0 +1,34 @@
+enablecompression false
+datatype[3]
+datatype[0].id -1633334123
+datatype[0].arraytype[0]
+datatype[0].weightedsettype[0]
+datatype[0].structtype[1]
+datatype[0].structtype[0].name varsize.header
+datatype[0].structtype[0].version 0
+datatype[0].structtype[0].field[1]
+datatype[0].structtype[0].field[0].name headerstring
+datatype[0].structtype[0].field[0].id[0]
+datatype[0].structtype[0].field[0].datatype 2
+datatype[0].documenttype[0]
+datatype[1].id -785359894
+datatype[1].arraytype[0]
+datatype[1].weightedsettype[0]
+datatype[1].structtype[1]
+datatype[1].structtype[0].name varsize.body
+datatype[1].structtype[0].version 0
+datatype[1].structtype[0].field[1]
+datatype[1].structtype[0].field[0].name bodystring
+datatype[1].structtype[0].field[0].id[0]
+datatype[1].structtype[0].field[0].datatype 2
+datatype[1].documenttype[0]
+datatype[2].id 211908458
+datatype[2].arraytype[0]
+datatype[2].weightedsettype[0]
+datatype[2].structtype[0]
+datatype[2].documenttype[1]
+datatype[2].documenttype[0].name varsize
+datatype[2].documenttype[0].version 0
+datatype[2].documenttype[0].inherits[0]
+datatype[2].documenttype[0].headerstruct -1633334123
+datatype[2].documenttype[0].bodystruct -785359894
diff --git a/document/src/tests/data/versionscfg.txt b/document/src/tests/data/versionscfg.txt
new file mode 100644
index 00000000000..a79e6040bc4
--- /dev/null
+++ b/document/src/tests/data/versionscfg.txt
@@ -0,0 +1,176 @@
+enablecompression false
+datatype[14]
+datatype[0].id 101
+datatype[0].arraytype[1]
+datatype[0].arraytype[0].datatype 1
+datatype[0].weightedsettype[0]
+datatype[0].structtype[0]
+datatype[0].documenttype[0]
+datatype[1].id 102
+datatype[1].arraytype[1]
+datatype[1].arraytype[0].datatype 2
+datatype[1].weightedsettype[0]
+datatype[1].structtype[0]
+datatype[1].documenttype[0]
+datatype[2].id 1000
+datatype[2].arraytype[0]
+datatype[2].weightedsettype[0]
+datatype[2].structtype[1]
+datatype[2].structtype[0].name crawler_type.header
+datatype[2].structtype[0].version 0
+datatype[2].structtype[0].field[5]
+datatype[2].structtype[0].field[0].name URI
+datatype[2].structtype[0].field[0].id[0]
+datatype[2].structtype[0].field[0].datatype 2
+datatype[2].structtype[0].field[1].name CONTEXT
+datatype[2].structtype[0].field[1].id[0]
+datatype[2].structtype[0].field[1].datatype 2
+datatype[2].structtype[0].field[2].name CONTENT
+datatype[2].structtype[0].field[2].id[0]
+datatype[2].structtype[0].field[2].datatype 3
+datatype[2].structtype[0].field[3].name CONTENT_TYPE
+datatype[2].structtype[0].field[3].id[0]
+datatype[2].structtype[0].field[3].datatype 2
+datatype[2].structtype[0].field[4].name LAST_MODIFIED
+datatype[2].structtype[0].field[4].id[0]
+datatype[2].structtype[0].field[4].datatype 0
+datatype[2].documenttype[0]
+datatype[3].id 1001
+datatype[3].arraytype[0]
+datatype[3].weightedsettype[0]
+datatype[3].structtype[1]
+datatype[3].structtype[0].name crawler_type.body
+datatype[3].structtype[0].version 0
+datatype[3].structtype[0].field[0]
+datatype[3].documenttype[0]
+datatype[4].id 1002
+datatype[4].arraytype[0]
+datatype[4].weightedsettype[0]
+datatype[4].structtype[0]
+datatype[4].documenttype[1]
+datatype[4].documenttype[0].name crawler_type
+datatype[4].documenttype[0].version 0
+datatype[4].documenttype[0].inherits[0]
+datatype[4].documenttype[0].headerstruct 1000
+datatype[4].documenttype[0].bodystruct 1001
+datatype[5].id 2000
+datatype[5].arraytype[0]
+datatype[5].weightedsettype[0]
+datatype[5].structtype[1]
+datatype[5].structtype[0].name crawler_type.header
+datatype[5].structtype[0].version 2
+datatype[5].structtype[0].field[3]
+datatype[5].structtype[0].field[0].name URI
+datatype[5].structtype[0].field[0].id[0]
+datatype[5].structtype[0].field[0].datatype 2
+datatype[5].structtype[0].field[1].name CONTEXT
+datatype[5].structtype[0].field[1].id[0]
+datatype[5].structtype[0].field[1].datatype 2
+datatype[5].structtype[0].field[2].name CONTENT
+datatype[5].structtype[0].field[2].id[0]
+datatype[5].structtype[0].field[2].datatype 3
+datatype[5].documenttype[0]
+datatype[6].id 2001
+datatype[6].arraytype[0]
+datatype[6].weightedsettype[0]
+datatype[6].structtype[1]
+datatype[6].structtype[0].name crawler_type.body
+datatype[6].structtype[0].version 2
+datatype[6].structtype[0].field[0]
+datatype[6].documenttype[0]
+datatype[7].id 2002
+datatype[7].arraytype[0]
+datatype[7].weightedsettype[0]
+datatype[7].structtype[0]
+datatype[7].documenttype[1]
+datatype[7].documenttype[0].name crawler_type
+datatype[7].documenttype[0].version 2
+datatype[7].documenttype[0].inherits[0]
+datatype[7].documenttype[0].headerstruct 2000
+datatype[7].documenttype[0].bodystruct 2001
+datatype[8].id 3000
+datatype[8].arraytype[0]
+datatype[8].weightedsettype[0]
+datatype[8].structtype[1]
+datatype[8].structtype[0].name multimedia_type.header
+datatype[8].structtype[0].version 0
+datatype[8].structtype[0].field[5]
+datatype[8].structtype[0].field[0].name URL_KEYWORDS
+datatype[8].structtype[0].field[0].id[0]
+datatype[8].structtype[0].field[0].datatype 102
+datatype[8].structtype[0].field[1].name MULTIMEDIA_LINKTYPES
+datatype[8].structtype[0].field[1].id[0]
+datatype[8].structtype[0].field[1].datatype 102
+datatype[8].structtype[0].field[2].name THUMBNAIL_URLS
+datatype[8].structtype[0].field[2].id[0]
+datatype[8].structtype[0].field[2].datatype 102
+datatype[8].structtype[0].field[3].name FROM_LINK_LIST
+datatype[8].structtype[0].field[3].id[0]
+datatype[8].structtype[0].field[3].datatype 101
+datatype[8].structtype[0].field[4].name FROM_LINK_LANGUAGE_LIST
+datatype[8].structtype[0].field[4].id[0]
+datatype[8].structtype[0].field[4].datatype 102
+datatype[8].documenttype[0]
+datatype[9].id 3001
+datatype[9].arraytype[0]
+datatype[9].weightedsettype[0]
+datatype[9].structtype[1]
+datatype[9].structtype[0].name multimedia_type.body
+datatype[9].structtype[0].version 0
+datatype[9].structtype[0].field[0]
+datatype[9].documenttype[0]
+datatype[10].id 3002
+datatype[10].arraytype[0]
+datatype[10].weightedsettype[0]
+datatype[10].structtype[0]
+datatype[10].documenttype[1]
+datatype[10].documenttype[0].name multimedia_type
+datatype[10].documenttype[0].version 0
+datatype[10].documenttype[0].inherits[1]
+datatype[10].documenttype[0].inherits[0].name crawler_type
+datatype[10].documenttype[0].inherits[0].version 0
+datatype[10].documenttype[0].headerstruct 3000
+datatype[10].documenttype[0].bodystruct 3001
+datatype[11].id 4000
+datatype[11].arraytype[0]
+datatype[11].weightedsettype[0]
+datatype[11].structtype[1]
+datatype[11].structtype[0].name multimedia_type.header
+datatype[11].structtype[0].version 2
+datatype[11].structtype[0].field[5]
+datatype[11].structtype[0].field[0].name URL_KEYWORDS
+datatype[11].structtype[0].field[0].id[0]
+datatype[11].structtype[0].field[0].datatype 102
+datatype[11].structtype[0].field[1].name MULTIMEDIA_LINKTYPES
+datatype[11].structtype[0].field[1].id[0]
+datatype[11].structtype[0].field[1].datatype 102
+datatype[11].structtype[0].field[2].name THUMBNAIL_URIS
+datatype[11].structtype[0].field[2].id[0]
+datatype[11].structtype[0].field[2].datatype 102
+datatype[11].structtype[0].field[3].name FROM_LINK_LIST
+datatype[11].structtype[0].field[3].id[0]
+datatype[11].structtype[0].field[3].datatype 101
+datatype[11].structtype[0].field[4].name FROM_LINK_LANGUAGE_LIST
+datatype[11].structtype[0].field[4].id[0]
+datatype[11].structtype[0].field[4].datatype 102
+datatype[11].documenttype[0]
+datatype[12].id 4001
+datatype[12].arraytype[0]
+datatype[12].weightedsettype[0]
+datatype[12].structtype[1]
+datatype[12].structtype[0].name multimedia_type.body
+datatype[12].structtype[0].version 2
+datatype[12].structtype[0].field[0]
+datatype[12].documenttype[0]
+datatype[13].id 4002
+datatype[13].arraytype[0]
+datatype[13].weightedsettype[0]
+datatype[13].structtype[0]
+datatype[13].documenttype[1]
+datatype[13].documenttype[0].name multimedia_type
+datatype[13].documenttype[0].version 2
+datatype[13].documenttype[0].inherits[1]
+datatype[13].documenttype[0].inherits[0].name crawler_type
+datatype[13].documenttype[0].inherits[0].version 2
+datatype[13].documenttype[0].headerstruct 4000
+datatype[13].documenttype[0].bodystruct 4001
diff --git a/document/src/tests/datatype/.gitignore b/document/src/tests/datatype/.gitignore
new file mode 100644
index 00000000000..bbfc05cb12e
--- /dev/null
+++ b/document/src/tests/datatype/.gitignore
@@ -0,0 +1,5 @@
+*.So
+*_test
+.depend
+Makefile
+document_datatype_test_app
diff --git a/document/src/tests/datatype/CMakeLists.txt b/document/src/tests/datatype/CMakeLists.txt
new file mode 100644
index 00000000000..399bccdf691
--- /dev/null
+++ b/document/src/tests/datatype/CMakeLists.txt
@@ -0,0 +1,10 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(document_datatype_test_app
+    SOURCES
+    datatype_test.cpp
+    DEPENDS
+    document
+    AFTER
+    document_documentconfig
+)
+vespa_add_test(NAME document_datatype_test_app COMMAND document_datatype_test_app)
diff --git a/document/src/tests/datatype/datatype_test.cpp b/document/src/tests/datatype/datatype_test.cpp
new file mode 100644
index 00000000000..d084e4f90ef
--- /dev/null
+++ b/document/src/tests/datatype/datatype_test.cpp
@@ -0,0 +1,64 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Unit tests for datatype.
+
+#include <vespa/log/log.h>
+LOG_SETUP("datatype_test");
+#include <vespa/fastos/fastos.h>
+
+#include <vespa/document/base/field.h>
+#include <vespa/document/datatype/arraydatatype.h>
+#include <vespa/document/datatype/structdatatype.h>
+#include <vespa/document/fieldvalue/longfieldvalue.h>
+#include <vespa/vespalib/testkit/testapp.h>
+
+using namespace document;
+
+namespace {
+
+TEST("require that ArrayDataType can be assigned to.") {
+    ArrayDataType type1(*DataType::STRING);
+    ArrayDataType type2(*DataType::INT);
+    type1 = type1;
+    EXPECT_EQUAL(*DataType::STRING, type1.getNestedType());
+    type1 = type2;
+    EXPECT_EQUAL(*DataType::INT, type1.getNestedType());
+}
+
+TEST("require that ArrayDataType can be cloned.") {
+    ArrayDataType type1(*DataType::STRING);
+    std::unique_ptr<ArrayDataType> type2(type1.clone());
+    ASSERT_TRUE(type2.get());
+    EXPECT_EQUAL(*DataType::STRING, type2->getNestedType());
+}
+
+TEST("require that assignment operator works for LongFieldValue") {
+    LongFieldValue val;
+    val = "1";
+    EXPECT_EQUAL(1, val.getValue());
+    val = 2;
+    EXPECT_EQUAL(2, val.getValue());
+    val = static_cast<int64_t>(3);
+    EXPECT_EQUAL(3, val.getValue());
+    val = 4.0f;
+    EXPECT_EQUAL(4, val.getValue());
+    val = 5.0;
+    EXPECT_EQUAL(5, val.getValue());
+}
+
+TEST("require that StructDataType can redeclare identical fields.") {
+    StructDataType s("foo");
+    Field field1("field1", 42, *DataType::STRING, true);
+    Field field2("field2", 42, *DataType::STRING, true);
+
+    s.addField(field1);
+    s.addField(field1);  // ok
+    s.addInheritedField(field1);  // ok
+    EXPECT_EXCEPTION(s.addField(field2), vespalib::IllegalArgumentException,
+                     "Field id in use by field Field(field1");
+    s.addInheritedField(field2);
+    EXPECT_FALSE(s.hasField(field2.getName()));
+}
+
+}  // namespace
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/document/src/tests/documentcalculatortestcase.cpp b/document/src/tests/documentcalculatortestcase.cpp
new file mode 100644
index 00000000000..00de5427961
--- /dev/null
+++ b/document/src/tests/documentcalculatortestcase.cpp
@@ -0,0 +1,202 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/document/base/testdocrepo.h>
+#include <vespa/document/fieldvalue/fieldvalues.h>
+#include <vespa/vespalib/io/fileutil.h>
+#include <vespa/vdstestlib/cppunit/macros.h>
+#include <vespa/document/base/documentcalculator.h>
+#include <vespa/document/fieldvalue/intfieldvalue.h>
+#include <vespa/document/fieldvalue/longfieldvalue.h>
+#include <vespa/document/fieldvalue/floatfieldvalue.h>
+
+#include <fstream>
+
+namespace document {
+
+class DocumentCalculatorTest : public CppUnit::TestFixture {
+    TestDocRepo _testRepo;
+
+public:
+    const DocumentTypeRepo &getRepo() { return _testRepo.getTypeRepo(); }
+
+    void setUp() {}
+    void tearDown() {}
+
+    void testConstant();
+    void testSimple();
+    void testVariables();
+    void testFields();
+    void testDivideByZero();
+    void testModByZero();
+    void testFieldsDivZero();
+    void testFieldNotSet();
+    void testFieldNotFound();
+    void testByteSubtractionZeroResult();
+
+    CPPUNIT_TEST_SUITE(DocumentCalculatorTest);
+    CPPUNIT_TEST(testConstant);
+    CPPUNIT_TEST(testSimple);
+    CPPUNIT_TEST(testVariables);
+    CPPUNIT_TEST(testFields);
+    CPPUNIT_TEST(testDivideByZero);
+    CPPUNIT_TEST(testModByZero);
+    CPPUNIT_TEST(testFieldsDivZero);
+    CPPUNIT_TEST(testFieldNotSet);
+    CPPUNIT_TEST(testFieldNotFound);
+    CPPUNIT_TEST(testByteSubtractionZeroResult);
+    CPPUNIT_TEST_SUITE_END();
+};
+
+CPPUNIT_TEST_SUITE_REGISTRATION(DocumentCalculatorTest);
+
+
+void
+DocumentCalculatorTest::testConstant() {
+    DocumentCalculator::VariableMap variables;
+    DocumentCalculator calc(getRepo(), "4.0");
+
+    Document doc(*_testRepo.getDocumentType("testdoctype1"),
+                 DocumentId("doc:test:foo"));
+    CPPUNIT_ASSERT_EQUAL(4.0, calc.evaluate(doc, variables));
+}
+
+void
+DocumentCalculatorTest::testSimple() {
+    DocumentCalculator::VariableMap variables;
+    DocumentCalculator calc(getRepo(), "(3 + 5) / 2");
+
+    Document doc(*_testRepo.getDocumentType("testdoctype1"),
+                 DocumentId("doc:test:foo"));
+    CPPUNIT_ASSERT_EQUAL(4.0, calc.evaluate(doc, variables));
+}
+
+void
+DocumentCalculatorTest::testVariables() {
+    DocumentCalculator::VariableMap variables;
+    variables["x"] = 3.0;
+    variables["y"] = 5.0;
+    DocumentCalculator calc(getRepo(), "($x + $y) / 2");
+
+    Document doc(*_testRepo.getDocumentType("testdoctype1"),
+                 DocumentId("doc:test:foo"));
+    CPPUNIT_ASSERT_EQUAL(4.0, calc.evaluate(doc, variables));
+}
+
+void
+DocumentCalculatorTest::testFields() {
+    DocumentCalculator::VariableMap variables;
+    variables["x"] = 3.0;
+    variables["y"] = 5.0;
+    DocumentCalculator calc(getRepo(), "(testdoctype1.headerval + testdoctype1"
+                            ".hfloatval) / testdoctype1.headerlongval");
+
+    Document doc(*_testRepo.getDocumentType("testdoctype1"),
+                 DocumentId("doc:test:foo"));
+    doc.setValue(doc.getField("headerval"), IntFieldValue(5));
+    doc.setValue(doc.getField("hfloatval"), FloatFieldValue(3.0));
+    doc.setValue(doc.getField("headerlongval"), LongFieldValue(2));
+    CPPUNIT_ASSERT_EQUAL(4.0, calc.evaluate(doc, variables));
+}
+
+void
+DocumentCalculatorTest::testFieldsDivZero() {
+    DocumentCalculator::VariableMap variables;
+    variables["x"] = 3.0;
+    variables["y"] = 5.0;
+    DocumentCalculator calc(getRepo(), "(testdoctype1.headerval + testdoctype1"
+                            ".hfloatval) / testdoctype1.headerlongval");
+
+    Document doc(*_testRepo.getDocumentType("testdoctype1"),
+                 DocumentId("doc:test:foo"));
+    doc.setValue(doc.getField("headerval"), IntFieldValue(5));
+    doc.setValue(doc.getField("hfloatval"), FloatFieldValue(3.0));
+    doc.setValue(doc.getField("headerlongval"), LongFieldValue(0));
+    try {
+        calc.evaluate(doc, variables);
+        CPPUNIT_ASSERT(false);
+    } catch (const vespalib::IllegalArgumentException& e) {
+        // OK
+    }
+}
+
+void
+DocumentCalculatorTest::testDivideByZero() {
+    DocumentCalculator::VariableMap variables;
+    DocumentCalculator calc(getRepo(), "(3 + 5) / 0");
+
+    Document doc(*_testRepo.getDocumentType("testdoctype1"),
+                 DocumentId("doc:test:foo"));
+    try {
+        calc.evaluate(doc, variables);
+        CPPUNIT_ASSERT(false);
+    } catch (const vespalib::IllegalArgumentException& e) {
+        // OK
+    }
+}
+
+void
+DocumentCalculatorTest::testModByZero() {
+    DocumentCalculator::VariableMap variables;
+    DocumentCalculator calc(getRepo(), "(3 + 5) % 0");
+
+    Document doc(*_testRepo.getDocumentType("testdoctype1"),
+                 DocumentId("doc:test:foo"));
+    try {
+        calc.evaluate(doc, variables);
+        CPPUNIT_ASSERT(false);
+    } catch (const vespalib::IllegalArgumentException& e) {
+        // OK
+    }
+}
+
+void
+DocumentCalculatorTest::testFieldNotSet() {
+    DocumentCalculator::VariableMap variables;
+    DocumentCalculator calc(getRepo(), "(testdoctype1.headerval + testdoctype1"
+                            ".hfloatval) / testdoctype1.headerlongval");
+
+    Document doc(*_testRepo.getDocumentType("testdoctype1"),
+                 DocumentId("doc:test:foo"));
+    doc.setValue(doc.getField("hfloatval"), FloatFieldValue(3.0));
+    doc.setValue(doc.getField("headerlongval"), LongFieldValue(2));
+    try {
+        calc.evaluate(doc, variables);
+        CPPUNIT_ASSERT(false);
+    } catch (const vespalib::IllegalArgumentException&) {
+        // OK
+    }
+}
+
+void
+DocumentCalculatorTest::testFieldNotFound() {
+    DocumentCalculator::VariableMap variables;
+    DocumentCalculator calc(getRepo(),
+                            "(testdoctype1.mynotfoundfield + testdoctype1"
+                            ".hfloatval) / testdoctype1.headerlongval");
+
+    Document doc(*_testRepo.getDocumentType("testdoctype1"),
+                 DocumentId("doc:test:foo"));
+    doc.setValue(doc.getField("hfloatval"), FloatFieldValue(3.0));
+    doc.setValue(doc.getField("headerlongval"), LongFieldValue(2));
+    try {
+        calc.evaluate(doc, variables);
+        CPPUNIT_ASSERT(false);
+    } catch (const vespalib::IllegalArgumentException&) {
+        // OK
+    }
+}
+
+void
+DocumentCalculatorTest::testByteSubtractionZeroResult() {
+    DocumentCalculator::VariableMap variables;
+    DocumentCalculator calc(getRepo(), "testdoctype1.byteval - 3");
+
+    Document doc(*_testRepo.getDocumentType("testdoctype1"),
+                 DocumentId("doc:test:foo"));
+    doc.setValue(doc.getField("byteval"), ByteFieldValue(3));
+    CPPUNIT_ASSERT_EQUAL(0.0, calc.evaluate(doc, variables));
+}
+
+}
diff --git a/document/src/tests/documentidtest.cpp b/document/src/tests/documentidtest.cpp
new file mode 100644
index 00000000000..fd60c6c436b
--- /dev/null
+++ b/document/src/tests/documentidtest.cpp
@@ -0,0 +1,183 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <cppunit/extensions/HelperMacros.h>
+#include <vespa/document/serialization/vespadocumentdeserializer.h>
+#include <vespa/document/serialization/vespadocumentserializer.h>
+#include <vespa/vespalib/objects/nbostream.h>
+#include <sstream>
+#include <string>
+#include <vespa/document/bucket/bucketidfactory.h>
+#include <vespa/document/base/documentid.h>
+#include <vespa/document/datatype/documenttype.h>
+#include <vespa/vespalib/util/md5.h>
+
+using document::VespaDocumentDeserializer;
+using document::VespaDocumentSerializer;
+using vespalib::nbostream;
+
+namespace document {
+
+struct DocumentIdTest : public CppUnit::TestFixture {
+    void generateJavaComplianceFile();
+    void testOutput();
+    void testEqualityOperator();
+    void testCopying();
+    void testParseId();
+    void checkNtnuGlobalId();
+    void testDocGlobalId();
+    void freestandingLocationFromGroupNameFuncMatchesIdLocation();
+
+    CPPUNIT_TEST_SUITE(DocumentIdTest);
+    CPPUNIT_TEST(testEqualityOperator);
+    CPPUNIT_TEST(testOutput);
+    CPPUNIT_TEST(testCopying);
+    CPPUNIT_TEST(generateJavaComplianceFile);
+    CPPUNIT_TEST(testParseId);
+    CPPUNIT_TEST(checkNtnuGlobalId);
+    CPPUNIT_TEST(testDocGlobalId);
+    CPPUNIT_TEST(freestandingLocationFromGroupNameFuncMatchesIdLocation);
+    CPPUNIT_TEST_SUITE_END();
+
+};
+
+CPPUNIT_TEST_SUITE_REGISTRATION(DocumentIdTest);
+
+namespace {
+    void writeGlobalIdBucketId(std::ostream& out, const std::string& id) {
+        BucketIdFactory factory;
+        out << id << " - " << document::DocumentId(id).getGlobalId()
+            << " - " << factory.getBucketId(document::DocumentId(id)).toString()
+            << "\n";
+    }
+}
+
+void DocumentIdTest::generateJavaComplianceFile()
+{
+    {  // Generate file with globalids and bucket ID of various document ids,
+       // which java will use to ensure equal implementations.
+        std::ostringstream ost;
+        writeGlobalIdBucketId(ost, "doc:ns:specific");
+        writeGlobalIdBucketId(ost, "doc:another:specific");
+        writeGlobalIdBucketId(ost, "doc:ns:another");
+        writeGlobalIdBucketId(ost, "userdoc:ns:100:specific");
+        writeGlobalIdBucketId(ost, "userdoc:np:100:another");
+        writeGlobalIdBucketId(ost, "userdoc:ns:101:specific");
+        writeGlobalIdBucketId(ost, "groupdoc:ns:agroup:specific");
+        writeGlobalIdBucketId(ost, "groupdoc:np:agroup:another");
+        writeGlobalIdBucketId(ost, "groupdoc:ns:another:specific");
+        for (uint32_t i=0; i<20; ++i) {
+            std::ostringstream ost2;
+            ost2 << i;
+            writeGlobalIdBucketId(ost, "doc:ns:"+ost2.str());
+        }
+        writeGlobalIdBucketId(ost, "id:ns:type::specific");
+        writeGlobalIdBucketId(ost, "id:another:type::specific");
+        writeGlobalIdBucketId(ost, "id:ns:type::another");
+        writeGlobalIdBucketId(ost, "id:ns:type:n=100:specific");
+        writeGlobalIdBucketId(ost, "id:np:type:n=100:another");
+        writeGlobalIdBucketId(ost, "id:ns:type:n=101:specific");
+        writeGlobalIdBucketId(ost, "id:ns:type:g=agroup:specific");
+        writeGlobalIdBucketId(ost, "id:np:type:g=agroup:another");
+        writeGlobalIdBucketId(ost, "id:ns:type:g=another:specific");
+        FastOS_File file;
+        CPPUNIT_ASSERT(file.OpenWriteOnlyTruncate("cpp-globalidbucketids.txt"));
+        std::string content(ost.str());
+        CPPUNIT_ASSERT(file.CheckedWrite(content.c_str(), content.size()));
+        CPPUNIT_ASSERT(file.Close());
+    }
+}
+
+
+void DocumentIdTest::testOutput()
+{
+    DocumentId id(DocIdString("crawler", "http://www.yahoo.com"));
+
+    std::ostringstream ost;
+    ost << id;
+    std::string expected("doc:crawler:http://www.yahoo.com");
+    CPPUNIT_ASSERT_EQUAL(expected, ost.str());
+
+    CPPUNIT_ASSERT_EQUAL(vespalib::string(expected), id.toString());
+
+    expected = "DocumentId(id = doc:crawler:http://www.yahoo.com, "
+                          "gid(0x928baffb39cf32004542fb60))";
+    CPPUNIT_ASSERT_EQUAL(expected, static_cast<Printable&>(id).toString(true));
+}
+
+namespace {
+    template<class T>
+    std::string getNotEqualMessage(const T& t1, const T& t2) {
+        std::ostringstream ost;
+        ost << "Expected instances to be different. This was not the case:\n"
+            << t1 << "\n" << t2 << "\n";
+        return ost.str();
+    }
+}
+
+void DocumentIdTest::testEqualityOperator()
+{
+    std::string uri(DocIdString("crawler", "http://www.yahoo.com").toString());
+
+    DocumentId id1(uri);
+    DocumentId id2(uri);
+    DocumentId id3("doc:crawler:http://www.yahoo.no/");
+
+    CPPUNIT_ASSERT_EQUAL(id1, id2);
+    CPPUNIT_ASSERT_MESSAGE(getNotEqualMessage(id1, id3), !(id1 == id3));
+}
+
+void DocumentIdTest::testCopying()
+{
+    std::string uri(DocIdString("crawler", "http://www.yahoo.com/").toString());
+
+    DocumentId id1(uri);
+    DocumentId id2(id1);
+    DocumentId id3("doc:ns:foo");
+    id3 = id2;
+
+    CPPUNIT_ASSERT_EQUAL(id1, id2);
+    CPPUNIT_ASSERT_EQUAL(id1, id3);
+}
+
+void
+DocumentIdTest::testParseId()
+{
+    // Moved to base/documentid_test.cpp
+}
+
+void
+DocumentIdTest::checkNtnuGlobalId()
+{
+    DocumentId id("doc:crawler:http://www.ntnu.no/");
+    CPPUNIT_ASSERT_EQUAL(vespalib::string("gid(0xb8863740be14221c0ac77896)"),
+                         id.getGlobalId().toString());
+}
+
+void
+DocumentIdTest::testDocGlobalId()
+{
+        // Test that location of doc scheme documents are set correctly, such
+        // that the location is the first bytes of the original GID.
+    std::string id("doc:crawler:http://www.ntnu.no/");
+    DocumentId did(id);
+
+    unsigned char key[16];
+    fastc_md5sum(reinterpret_cast<const unsigned char*>(id.c_str()),
+                 id.size(), key);
+
+    CPPUNIT_ASSERT_EQUAL(GlobalId(key), did.getGlobalId());
+}
+
+void
+DocumentIdTest::freestandingLocationFromGroupNameFuncMatchesIdLocation()
+{
+    CPPUNIT_ASSERT_EQUAL(
+            DocumentId("id::foo:g=zoid:bar").getScheme().getLocation(),
+            GroupDocIdString::locationFromGroupName("zoid"));
+    CPPUNIT_ASSERT_EQUAL(
+            DocumentId("id::bar:g=doink:baz").getScheme().getLocation(),
+            GroupDocIdString::locationFromGroupName("doink"));
+}
+
+} // document
diff --git a/document/src/tests/documentselectparsertest.cpp b/document/src/tests/documentselectparsertest.cpp
new file mode 100644
index 00000000000..357e6eb2a68
--- /dev/null
+++ b/document/src/tests/documentselectparsertest.cpp
@@ -0,0 +1,1278 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+
+//#include <boost/regex/icu.hpp>
+#include <vespa/fastos/fastos.h>
+#include <cppunit/TestFixture.h>
+#include <cppunit/extensions/HelperMacros.h>
+#include <vespa/document/repo/configbuilder.h>
+#include <vespa/document/repo/documenttyperepo.h>
+#include <iostream>
+#include <memory>
+#include <vespa/document/update/assignvalueupdate.h>
+#include <vespa/document/base/testdocman.h>
+#include <vespa/document/base/testdocrepo.h>
+#include <vespa/document/select/parser.h>
+#include <vespa/document/select/visitor.h>
+#include <vespa/document/select/bodyfielddetector.h>
+#include <vespa/document/select/valuenode.h>
+#include <vespa/document/select/branch.h>
+#include <vespa/document/select/simpleparser.h>
+#include <vespa/document/select/constant.h>
+#include <vespa/document/select/invalidconstant.h>
+#include <vespa/document/select/doctype.h>
+#include <vespa/document/select/compare.h>
+
+using namespace document::config_builder;
+
+namespace document {
+
+class DocumentSelectParserTest : public CppUnit::TestFixture {
+    CPPUNIT_TEST_SUITE(DocumentSelectParserTest);
+    CPPUNIT_TEST(testParseTerminals);
+    CPPUNIT_TEST(testParseBranches);
+    CPPUNIT_TEST(testOperators);
+    CPPUNIT_TEST(testVisitor);
+    CPPUNIT_TEST(testUtf8);
+    CPPUNIT_TEST(testBodyFieldDetection);
+    CPPUNIT_TEST(testDocumentUpdates);
+    CPPUNIT_TEST_SUITE_END();
+
+    BucketIdFactory _bucketIdFactory;
+    std::unique_ptr<select::Parser> _parser;
+    std::vector<Document::SP > _doc;
+    std::vector<DocumentUpdate::SP > _update;
+
+    Document::SP createDoc(
+            const std::string& doctype, const std::string& id, uint32_t hint,
+            double hfloat, const std::string& hstr, const std::string& cstr,
+            uint64_t hlong = 0);
+
+    DocumentUpdate::SP createUpdate(
+            const std::string& doctype, const std::string& id, uint32_t hint,
+            const std::string& hstr);
+
+    template <typename ContainsType>
+    select::ResultList doParse(const vespalib::stringref& expr,
+                               const ContainsType& t);
+public:
+
+    DocumentSelectParserTest()
+        : _bucketIdFactory() {}
+
+    void setUp();
+    void tearDown() {}
+    void createDocs();
+
+    void testParseTerminals();
+    void testParseBranches();
+    void testOperators();
+    void testOperators0();
+    void testOperators1();
+    void testOperators2();
+    void testOperators3();
+    void testOperators4();
+    void testOperators5();
+    void testOperators6();
+    void testOperators7();
+    void testOperators8();
+    void testOperators9();
+    void testVisitor();
+    void testUtf8();
+    void testBodyFieldDetection();
+    void testDocumentUpdates();
+    void testDocumentUpdates0();
+    void testDocumentUpdates1();
+    void testDocumentUpdates2();
+    void testDocumentUpdates3();
+    void testDocumentUpdates4();
+    void testDocumentUpdates5();
+};
+
+CPPUNIT_TEST_SUITE_REGISTRATION(DocumentSelectParserTest);
+
+namespace {
+    DocumentTypeRepo::SP _repo;
+}
+
+void DocumentSelectParserTest::setUp()
+{
+    DocumenttypesConfigBuilderHelper builder(TestDocRepo::getDefaultConfig());
+    builder.document(535424777, "notandor",
+                     Struct("notandor.header"), Struct("notandor.body"));
+    builder.document(1348665801, "ornotand",
+                     Struct("ornotand.header"), Struct("ornotand.body"));
+    builder.document(-1848670693, "andornot",
+                     Struct("andornot.header"), Struct("andornot.body"));
+    builder.document(-1193328712, "idid",
+                     Struct("idid.header"), Struct("idid.body"));
+    builder.document(-1673092522, "usergroup",
+                     Struct("usergroup.header"),
+                     Struct("usergroup.body"));
+    _repo.reset(new DocumentTypeRepo(builder.config()));
+
+    _parser.reset(new select::Parser(*_repo, _bucketIdFactory));
+}
+
+Document::SP DocumentSelectParserTest::createDoc(
+        const std::string& doctype, const std::string& id, uint32_t hint,
+        double hfloat, const std::string& hstr, const std::string& cstr,
+        uint64_t hlong)
+{
+    const DocumentType* type = _repo->getDocumentType(doctype);
+    Document::SP doc(new Document(*type, DocumentId(id)));
+    doc->setValue(doc->getField("headerval"), IntFieldValue(hint));
+
+    if (hlong != 0) {
+        doc->setValue(doc->getField("headerlongval"), LongFieldValue(hlong));
+    }
+    doc->setValue(doc->getField("hfloatval"), FloatFieldValue(hfloat));
+    doc->setValue(doc->getField("hstringval"), StringFieldValue(hstr.c_str()));
+    doc->setValue(doc->getField("content"), StringFieldValue(cstr.c_str()));
+    return doc;
+}
+
+DocumentUpdate::SP DocumentSelectParserTest::createUpdate(
+        const std::string& doctype, const std::string& id, uint32_t hint,
+        const std::string& hstr)
+{
+    const DocumentType* type = _repo->getDocumentType(doctype);
+    DocumentUpdate::SP doc(
+            new DocumentUpdate(*type, DocumentId(id)));
+    doc->addUpdate(FieldUpdate(doc->getType().getField("headerval"))
+                      .addUpdate(AssignValueUpdate(IntFieldValue(hint))));
+    doc->addUpdate(FieldUpdate(doc->getType().getField("hstringval"))
+                      .addUpdate(AssignValueUpdate(StringFieldValue(hstr))));
+    return doc;
+}
+
+void
+DocumentSelectParserTest::createDocs()
+{
+    _doc.clear();
+    _doc.push_back(createDoc(
+                           "testdoctype1", "doc:myspace:anything", 24, 2.0, "foo", "bar", 0));  // DOC 0
+    _doc.push_back(createDoc(
+                           "testdoctype1", "doc:anotherspace:foo", 13, 4.1, "bar", "foo", 0));  // DOC 1
+        // Add some arrays and structs to doc 1
+    {
+        StructFieldValue sval(_doc.back()->getField("mystruct").getDataType());
+        sval.set("key", 14);
+        sval.set("value", "structval");
+        _doc.back()->setValue("mystruct", sval);
+        ArrayFieldValue
+            aval(_doc.back()->getField("structarray").getDataType());
+        {
+            StructFieldValue sval1(aval.getNestedType());
+            sval1.set("key", 15);
+            sval1.set("value", "structval1");
+            StructFieldValue sval2(aval.getNestedType());
+            sval2.set("key", 16);
+            sval2.set("value", "structval2");
+            aval.add(sval1);
+            aval.add(sval2);
+        }
+        _doc.back()->setValue("structarray", aval);
+
+        MapFieldValue mval(_doc.back()->getField("mymap").getDataType());
+        mval.put(document::IntFieldValue(3), document::StringFieldValue("a"));
+        mval.put(document::IntFieldValue(5), document::StringFieldValue("b"));
+        mval.put(document::IntFieldValue(7), document::StringFieldValue("c"));
+        _doc.back()->setValue("mymap", mval);
+
+        MapFieldValue
+            amval(_doc.back()->getField("structarrmap").getDataType());
+        amval.put(StringFieldValue("foo"), aval);
+
+        ArrayFieldValue
+            abval(_doc.back()->getField("structarray").getDataType());
+        {
+            StructFieldValue sval1(aval.getNestedType());
+            sval1.set("key", 17);
+            sval1.set("value", "structval3");
+            StructFieldValue sval2(aval.getNestedType());
+            sval2.set("key", 18);
+            sval2.set("value", "structval4");
+            abval.add(sval1);
+            abval.add(sval2);
+        }
+
+        amval.put(StringFieldValue("bar"), abval);
+        _doc.back()->setValue("structarrmap", amval);
+
+        WeightedSetFieldValue wsval(
+                _doc.back()->getField("stringweightedset").getDataType());
+        wsval.add("foo");
+        wsval.add("val1");
+        wsval.add("val2");
+        wsval.add("val3");
+        wsval.add("val4");
+        _doc.back()->setValue("stringweightedset", wsval);
+
+        WeightedSetFieldValue wsbytes(
+                _doc.back()->getField("byteweightedset").getDataType());
+        wsbytes.add(ByteFieldValue(5));
+        wsbytes.add(ByteFieldValue(75));
+        wsbytes.add(ByteFieldValue(255));
+        wsbytes.add(ByteFieldValue(0));
+        _doc.back()->setValue("byteweightedset", wsbytes);
+    }
+
+    _doc.push_back(createDoc(
+                           "testdoctype1", "userdoc:myspace:1234:footype1", 15, 1.0, "some", "some", 0));  // DOC 2
+        // Add empty struct and array
+    {
+        StructFieldValue sval(_doc.back()->getField("mystruct").getDataType());
+        _doc.back()->setValue("mystruct", sval);
+        ArrayFieldValue aval(
+                _doc.back()->getField("structarray").getDataType());
+        _doc.back()->setValue("structarray", aval);
+    }
+    _doc.push_back(createDoc(
+                           "testdoctype1", "groupdoc:myspace:yahoo:bar", 14, 2.4, "Yet", "\xE4\xB8\xBA\xE4\xBB\x80", 0)); // DOC 3
+    _doc.push_back(createDoc(
+                           "testdoctype2", "doc:myspace:inheriteddoc", 10, 1.4, "inherited", "")); // DOC 4
+    _doc.push_back(createDoc(
+        "testdoctype1", "userdoc:footype:123456789:aardvark",
+        10, 1.4, "inherited", "", 0));  // DOC 5
+    _doc.push_back(createDoc(
+        "testdoctype1", "userdoc:footype:1234:highlong",
+        10, 1.4, "inherited", "", 2651257743)); // DOC 6
+    _doc.push_back(createDoc(
+        "testdoctype1", "userdoc:footype:1234:highlong",
+        10, 1.4, "inherited", "", -2651257743)); // DOC 7
+    _doc.push_back(createDoc(  // DOC 8
+        "testdoctype1", "orderdoc(4,4):footype:1234:12:highlong",
+        10, 1.4, "inherited", "", -2651257743));
+    _doc.push_back(createDoc(  // DOC 9
+        "testdoctype1", "orderdoc(4,4):footype:mygroup:12:highlong",
+        10, 1.4, "inherited", "", -2651257743));
+    _doc.push_back(createDoc( // DOC 10. As DOC 0 but with version 2.
+        "testdoctype1", "doc:myspace:anything", 24, 2.0, "foo", "bar", 0));
+    _doc.push_back(createDoc(
+        "testdoctype1", "id:footype:testdoctype1:n=12345:foo",
+        10, 1.4, "inherited", "", 42)); // DOC 11
+    _doc.push_back(createDoc(
+        "testdoctype1", "id:myspace:testdoctype1:g=xyzzy:foo",
+        10, 1.4, "inherited", "", 42)); // DOC 12
+
+    _update.clear();
+    _update.push_back(createUpdate(
+        "testdoctype1", "doc:myspace:anything", 20, "hmm"));
+    _update.push_back(createUpdate(
+        "testdoctype1", "doc:anotherspace:foo", 10, "foo"));
+    _update.push_back(createUpdate(
+        "testdoctype1", "userdoc:myspace:1234:footype1", 0, "foo"));
+    _update.push_back(createUpdate(
+        "testdoctype1", "groupdoc:myspace:yahoo:bar", 3, "\xE4\xBA\xB8\xE4\xBB\x80"));
+    _update.push_back(createUpdate(
+        "testdoctype2", "doc:myspace:inheriteddoc", 10, "bar"));
+}
+
+namespace {
+void doVerifyParse(select::Node *node, const std::string &query,
+                   const char *expected) {
+    std::string message("Query "+query+" failed to parse.");
+    CPPUNIT_ASSERT_MESSAGE(message, node != 0);
+    std::ostringstream actual;
+    actual << *node;
+
+    std::string exp(expected != 0 ? std::string(expected) : query);
+    CPPUNIT_ASSERT_EQUAL(exp, actual.str());
+    // Test that cloning gives the same result
+    std::unique_ptr<select::Node> clonedNode(node->clone());
+    std::ostringstream clonedStr;
+    clonedStr << *clonedNode;
+    CPPUNIT_ASSERT_EQUAL(exp, clonedStr.str());
+}
+
+void verifySimpleParse(const std::string& query, const char* expected = 0) {
+    BucketIdFactory factory;
+    select::simple::SelectionParser parser(factory);
+    std::string message("Query "+query+" failed to parse.");
+    CPPUNIT_ASSERT_MESSAGE(message, parser.parse(query));
+    std::unique_ptr<select::Node> node(parser.getNode());
+    doVerifyParse(node.get(), query, expected);
+}
+
+void verifyParse(const std::string& query, const char* expected = 0) {
+    BucketIdFactory factory;
+    select::Parser parser(*_repo, factory);
+    std::unique_ptr<select::Node> node(parser.parse(query));
+    doVerifyParse(node.get(), query, expected);
+}
+
+    void verifyFailedParse(const std::string& query, const std::string& error) {
+        try{
+            BucketIdFactory factory;
+            TestDocRepo test_repo;
+            select::Parser parser(test_repo.getTypeRepo(), factory);
+            std::unique_ptr<select::Node> node(parser.parse(query));
+            CPPUNIT_FAIL("Expected exception parsing query '"+query+"'");
+        } catch (select::ParsingFailedException& e) {
+            std::string message(e.what());
+            if (message.size() > error.size())
+                message = message.substr(0, error.size());
+            std::string failure("Expected: " + error + "\n- Actual  : "
+                                + std::string(e.what()));
+            CPPUNIT_ASSERT_MESSAGE(failure, error == message);
+        }
+    }
+}
+
+void DocumentSelectParserTest::testParseTerminals()
+{
+    createDocs();
+
+      // Test number value
+    verifyParse("", "true");
+    verifyParse("testdoctype1.headerval == 123");
+    verifyParse("testdoctype1.headerval == +123.53", "testdoctype1.headerval == 123.53");
+    verifyParse("testdoctype1.headerval == -123.5");
+    verifyParse("testdoctype1.headerval == 234123.523e3",
+                "testdoctype1.headerval == 2.34124e+08");
+    verifyParse("testdoctype1.headerval == -234123.523E-3",
+                "testdoctype1.headerval == -234.124");
+    verifyFailedParse("testdoctype1.headerval == aaa", "ParsingFailedException: "
+            "Unexpected token at position 23 ('== aaa') in query "
+            "'testdoctype1.headerval == aaa', at fullParse in ");
+      // Test string value
+    verifyParse("testdoctype1.headerval == \"test\"");
+    std::unique_ptr<select::Node> node(
+            _parser->parse("testdoctype1.headerval == \"test\""));
+    const select::Compare& compnode(
+            dynamic_cast<const select::Compare&>(*node));
+    const select::FieldValueNode& fnode(
+            dynamic_cast<const select::FieldValueNode&>(compnode.getLeft()));
+    const select::StringValueNode& vnode(
+            dynamic_cast<const select::StringValueNode&>(compnode.getRight()));
+    /*
+    CPPUNIT_ASSERT_EQUAL(vespalib::string("testdoctype1"),
+                         fnode.getDocType()->getName());
+                         */
+    CPPUNIT_ASSERT_EQUAL(vespalib::string("headerval"), fnode.getFieldName());
+    CPPUNIT_ASSERT_EQUAL(vespalib::string("test"), vnode.getValue());
+      // Test whitespace
+    verifyParse("testdoctype1.headerval == \"te st \"");
+    verifyParse(" \t testdoctype1.headerval\t==  \t \"test\"\t",
+                "testdoctype1.headerval == \"test\"");
+      // Test escaping
+    verifyParse("testdoctype1.headerval == \"tab\\ttest\"");
+    verifyParse("testdoctype1.headerval == \"tab\\x09test\"",
+                "testdoctype1.headerval == \"tab\\ttest\"");
+    verifyParse("testdoctype1.headerval == \"tab\\x055test\"");
+    verifyFailedParse("testdoctype1.headerval == \"tab\\x0notcomplete\"",
+            "ParsingFailedException: Unexpected token at position 23 "
+            "('== \"tab\\x0') in query 'testdoctype1.headerval == \"tab\\x0notcomplete\"', "
+            "at fullParse in ");
+    verifyFailedParse("testdoctype1.headerval == \"tab\\ysf\"",
+            "ParsingFailedException: Unexpected token at position 23 "
+            "('== \"tab\\ys') in query 'testdoctype1.headerval == \"tab\\ysf\"', "
+            "at fullParse in ");
+    node = _parser->parse("testdoctype1.headerval == \"\\tt\\x48 \\n\"");
+    select::Compare& escapednode(dynamic_cast<select::Compare&>(*node));
+    const select::StringValueNode& escval(
+        dynamic_cast<const select::StringValueNode&>(escapednode.getRight()));
+    CPPUNIT_ASSERT_EQUAL(vespalib::string("\ttH \n"), escval.getValue());
+      // Test illegal operator
+    verifyFailedParse("testdoctype1.headerval <> 12", "ParsingFailedException: Unexpected"
+            " token at position 23 ('<> 12') in query 'testdoctype1.headerval <> 12', at");
+      // Test <= <, > >=
+    verifyParse("testdoctype1.headerval >= 123");
+    verifyParse("testdoctype1.headerval > 123");
+    verifyParse("testdoctype1.headerval <= 123");
+    verifyParse("testdoctype1.headerval < 123");
+    verifyParse("testdoctype1.headerval != 123");
+
+      // Test defined
+    verifyParse("testdoctype1.headerval", "testdoctype1.headerval != null");
+
+      // Test bools
+    verifyParse("TRUE");
+    verifyParse("FALSE");
+    verifyParse("true");
+    verifyParse("false");
+    verifyParse("faLSe");
+    verifyFailedParse("fal se", "ParsingFailedException: Unexpected token at "
+                                "position 4 ('se') in query 'fal se', at");
+
+      // Test document types
+    verifyParse("testdoctype1");
+    verifyFailedParse("mytype", "ParsingFailedException: Document type mytype "
+                                "not found");
+    verifyParse("_test_doctype3_");
+    verifyParse("_test_doctype3_._only_in_child_ == 0");
+
+      // Test document id with simple parser.
+    verifySimpleParse("id == \"userdoc:ns:mytest\"");
+    verifySimpleParse("id.namespace == \"myspace\"");
+    verifySimpleParse("id.scheme == \"userdoc\"");
+    verifySimpleParse("id.type == \"testdoctype1\"");
+    verifySimpleParse("id.group == \"yahoo.com\"");
+    verifySimpleParse("id.user == 1234");
+    verifySimpleParse("id.user == 0x12456ab", "id.user == 19158699");
+
+      // Test document id
+    verifyParse("id == \"userdoc:ns:mytest\"");
+    verifyParse("id.namespace == \"myspace\"");
+    verifyParse("id.scheme == \"userdoc\"");
+    verifyParse("id.type == \"testdoctype1\"");
+    verifyParse("id.user == 1234");
+    verifyParse("id.user == 0x12456ab", "id.user == 19158699");
+    verifyParse("id.group == \"yahoo.com\"");
+    verifyParse("id.order(10,5) < 100");
+
+    verifyParse("id.specific == \"mypart\"");
+    verifyParse("id.bucket == 1234");
+    verifyParse("id.bucket == 0x800000", "id.bucket == 8388608");
+    verifyParse("id.bucket == 0x80a000", "id.bucket == 8429568");
+    verifyParse("id.bucket == 0x80000000000000f2",
+                "id.bucket == -9223372036854775566");
+    verifyParse("id.gid == \"gid(0xd755743aea262650274d70f0)\"");
+
+    // Test search column
+    verifyParse("searchcolumn.10 == 2");
+
+      // Test other operators
+    verifyParse("id.scheme = \"*doc\"");
+    verifyParse("testdoctype1.hstringval =~ \"(john|barry|shrek)\"");
+
+      // Verify functions
+    verifyParse("id.hash() == 124");
+    verifyParse("id.specific.hash() == 124");
+    verifyParse("testdoctype1.hstringval.lowercase() == \"chang\"");
+    verifyParse("testdoctype1.hstringval.lowercase().hash() == 124");
+    verifyFailedParse("testdoctype1 == 8", "ParsingFailedException: Unexpected token"
+            " at position 13 ('== 8') in query 'testdoctype1 == 8', at fullParse in ");
+    verifyParse("testdoctype1.hintval > now()");
+    verifyParse("testdoctype1.hintval > now().abs()");
+
+      // Value grouping
+    verifyParse("(123) < (200)");
+    verifyParse("(\"hmm\") < (id.scheme)");
+
+      // Arithmetics
+    verifyParse("1 + 2 > 1");
+    verifyParse("1 - 2 > 1");
+    verifyParse("1 * 2 > 1");
+    verifyParse("1 / 2 > 1");
+    verifyParse("1 % 2 > 1");
+    verifyParse("(1 + 2) * (4 - 2) == 1");
+    verifyParse("23 + 643 / 34 % 10 > 34");
+
+      // CJK stuff
+    verifyParse("testdoctype1.hstringval = \"\xE4\xB8\xBA\xE4\xBB\x80\"",
+                "testdoctype1.hstringval = \"\\xe4\\xb8\\xba\\xe4\\xbb\\x80\"");
+
+      // Strange doctype names
+    verifyParse("notandor");
+    verifyParse("ornotand");
+    verifyParse("andornot");
+    verifyParse("idid");
+    verifyParse("usergroup");
+}
+
+void DocumentSelectParserTest::testParseBranches()
+{
+    createDocs();
+
+    verifyParse("TRUE or FALSE aNd FALSE oR TRUE");
+    verifyParse("TRUE and FALSE or FALSE and TRUE");
+    verifyParse("TRUE or FALSE and FALSE or TRUE");
+    verifyParse("(TRUE or FALSE) and (FALSE or TRUE)");
+    verifyParse("true or (not false) and not true");
+
+        // Test number branching with node branches
+    verifyParse("((243) < 300 and (\"FOO\").lowercase() == (\"foo\"))");
+
+      // Strange doctype names
+    verifyParse("notandor and ornotand");
+    verifyParse("ornotand or andornot");
+    verifyParse("not andornot");
+    verifyParse("idid or not usergroup");
+    verifyParse("not(andornot or idid)", "not (andornot or idid)");
+}
+
+template <typename ContainsType>
+select::ResultList
+DocumentSelectParserTest::doParse(const vespalib::stringref& expr,
+                                  const ContainsType& t)
+{
+    std::unique_ptr<select::Node> root(_parser->parse(expr));
+    select::ResultList result(root->contains(t));
+
+    std::unique_ptr<select::Node> cloned(root->clone());
+    select::ResultList clonedResult(cloned->contains(t));
+
+    std::unique_ptr<select::Node> traced(_parser->parse(expr));
+    std::ostringstream oss;
+    oss << "for expr: " << expr << "\n";
+    select::ResultList tracedResult(root->trace(t, oss));
+
+    CPPUNIT_ASSERT_EQUAL_MESSAGE(expr, result, clonedResult);
+    CPPUNIT_ASSERT_EQUAL_MESSAGE(oss.str(), result, tracedResult);
+
+    return result;
+}
+
+#define PARSE(expr, doc, result) \
+    CPPUNIT_ASSERT_EQUAL_MESSAGE(expr, select::ResultList(select::Result::result), \
+                                 doParse(expr, (doc)));
+
+#define PARSEI(expr, doc, result) \
+    CPPUNIT_ASSERT_EQUAL_MESSAGE(std::string("Doc: ") + expr,          \
+                                 select::ResultList(select::Result::result), \
+                                 doParse(expr, (doc)));                 \
+    CPPUNIT_ASSERT_EQUAL_MESSAGE(std::string("Doc id: ") + expr,       \
+                                 select::ResultList(select::Result::result), \
+                                 doParse(expr, (doc).getId()));
+
+void DocumentSelectParserTest::testOperators()
+{
+    testOperators0();
+    testOperators1();
+    testOperators2();
+    testOperators3();
+    testOperators4();
+    testOperators5();
+    testOperators6();
+    testOperators7();
+    testOperators8();
+    testOperators9();
+}
+
+void DocumentSelectParserTest::testOperators0()
+{
+    createDocs();
+
+    /* Code for tracing result to see what went wrong
+    {
+        std::ostringstream ost;
+        _parser->parse("id.specific.hash() % 10 = 8")->trace(*_doc[0], ost);
+        ost << "\n\n";
+        _parser->parse("id.specific.hash() % 10 = 8")
+                ->trace(_doc[0]->getId(), ost);
+        std::cerr << ost.str() << "\n";
+    } // */
+
+        // Check that comparison operators work.
+    PARSE("", *_doc[0], True);
+    PARSE("30 < 10", *_doc[0], False);
+    PARSE("10 < 30", *_doc[0], True);
+    PARSE("30 < 10", *_doc[0], False);
+    PARSE("10 < 30", *_doc[0], True);
+    PARSE("30 <= 10", *_doc[0], False);
+    PARSE("10 <= 30", *_doc[0], True);
+    PARSE("30 <= 30", *_doc[0], True);
+    PARSE("10 >= 30", *_doc[0], False);
+    PARSE("30 >= 10", *_doc[0], True);
+    PARSE("30 >= 30", *_doc[0], True);
+
+    PARSE("10 > 30", *_doc[0], False);
+    PARSE("30 > 10", *_doc[0], True);
+    PARSE("30 == 10", *_doc[0], False);
+    PARSE("30 == 30", *_doc[0], True);
+    PARSE("30 != 10", *_doc[0], True);
+    PARSE("30 != 30", *_doc[0], False);
+    PARSE("\"foo\" != \"bar\"", *_doc[0], True);
+    PARSE("\"foo\" != \"foo\"", *_doc[0], False);
+    PARSE("\"foo\" == 'bar'", *_doc[0], False);
+    PARSE("\"foo\" == 'foo'", *_doc[0], True);
+    PARSE("\"bar\" = \"a\"", *_doc[0], False);
+    PARSE("\"bar\" = \"*a*\"", *_doc[0], True);
+    PARSE("\"bar\" = \"\"", *_doc[0], False);
+    PARSE("\"\" = \"\"", *_doc[0], True);
+    PARSE("\"bar\" =~ \"^a$\"", *_doc[0], False);
+    PARSE("\"bar\" =~ \"a\"", *_doc[0], True);
+    PARSE("\"bar\" =~ \"\"", *_doc[0], True);
+    PARSE("\"\" =~ \"\"", *_doc[0], True);
+    PARSE("30 = 10", *_doc[0], False);
+    PARSE("30 = 30", *_doc[0], True);
+}
+
+void DocumentSelectParserTest::testOperators1()
+{
+    createDocs();
+
+        // Mix of types should within numbers, but otherwise not match
+    PARSE("30 < 10.2", *_doc[0], False);
+    PARSE("10.2 < 30", *_doc[0], True);
+    PARSE("30 < \"foo\"", *_doc[0], Invalid);
+    PARSE("30 > \"foo\"", *_doc[0], Invalid);
+    PARSE("30 != \"foo\"", *_doc[0], Invalid);
+    PARSE("14.2 <= \"foo\"", *_doc[0], Invalid);
+    PARSE("null == null", *_doc[0], True);
+    PARSE("null = null", *_doc[0], True);
+    PARSE("\"bar\" == null", *_doc[0], False);
+    PARSE("14.3 == null", *_doc[0], False);
+    PARSE("null = 0", *_doc[0], False);
+
+        // Field values
+    PARSE("testdoctype1.headerval = 24", *_doc[0], True);
+    PARSE("testdoctype1.headerval = 24", *_doc[1], False);
+    PARSE("testdoctype1.headerval = 13", *_doc[0], False);
+    PARSE("testdoctype1.headerval = 13", *_doc[1], True);
+    PARSE("testdoctype1.hfloatval = 2.0", *_doc[0], True);
+    PARSE("testdoctype1.hfloatval = 1.0", *_doc[1], False);
+    PARSE("testdoctype1.hfloatval = 4.1", *_doc[0], False);
+    PARSE("testdoctype1.hfloatval > 4.09 and testdoctype1.hfloatval < 4.11",
+            *_doc[1], True);
+    PARSE("testdoctype1.content = \"bar\"", *_doc[0], True);
+    PARSE("testdoctype1.content = \"bar\"", *_doc[1], False);
+    PARSE("testdoctype1.content = \"foo\"", *_doc[0], False);
+    PARSE("testdoctype1.content = \"foo\"", *_doc[1], True);
+    PARSE("testdoctype1.hstringval == testdoctype1.content", *_doc[0], False);
+    PARSE("testdoctype1.hstringval == testdoctype1.content", *_doc[2], True);
+    PARSE("testdoctype1.byteweightedset == 7", *_doc[1], False);
+    PARSE("testdoctype1.byteweightedset == 5", *_doc[1], True);
+
+        // Document types
+    PARSE("testdoctype1", *_doc[0], True);
+    PARSE("testdoctype2", *_doc[0], False);
+
+        // Inherited doctypes
+    PARSE("testdoctype2", *_doc[4], True);
+    PARSE("testdoctype2", *_doc[3], False);
+    PARSE("testdoctype1", *_doc[4], True);
+    PARSE("testdoctype1.headerval = 10", *_doc[4], True);
+}
+
+void DocumentSelectParserTest::testOperators2()
+{
+    createDocs();
+
+        // Id values
+    PARSEI("id == \"doc:myspace:anything\"", *_doc[0], True);
+    PARSEI(" iD==  \"doc:myspace:anything\"  ", *_doc[0], True);
+    PARSEI("id == \"doc:myspa:nything\"", *_doc[0], False);
+    PARSEI("Id.scHeme == \"doc\"", *_doc[0], True);
+    PARSEI("id.scheme == \"userdoc\"", *_doc[0], False);
+    PARSEI("id.type == \"testdoctype1\"", *_doc[11], True);
+    PARSEI("id.type == \"wrong_type\"", *_doc[11], False);
+    PARSEI("id.type == \"unknown\"", *_doc[0], Invalid);
+    PARSEI("Id.namespaCe == \"myspace\"", *_doc[0], True);
+    PARSEI("id.NaMespace == \"pace\"", *_doc[0], False);
+    PARSEI("id.specific == \"anything\"", *_doc[0], True);
+    PARSEI("id.user=1234", *_doc[2], True);
+    PARSEI("id.user == 1234", *_doc[0], Invalid);
+    PARSEI("id.group == 1234", *_doc[3], Invalid);
+    PARSEI("id.group == \"yahoo\"", *_doc[3], True);
+    PARSEI("id.bucket == 1234", *_doc[0], False);
+    PARSEI("id.order(4,4) == 12", *_doc[8], True);
+    PARSEI("id.order(4,4) < 20", *_doc[8], True);
+    PARSEI("id.order(4,4) > 12", *_doc[8], False);
+    PARSEI("id.order(5,5) <= 12", *_doc[8], Invalid);
+    PARSEI("id.order(4,4) <= 12", *_doc[8], True);
+    PARSEI("id.order(4,4) == 12", *_doc[0], Invalid);
+    PARSEI("id.user=12345", *_doc[11], True);
+    PARSEI("id.group == \"xyzzy\"", *_doc[12], True);
+}
+
+void DocumentSelectParserTest::testOperators3()
+{
+    createDocs();
+    {
+        std::ostringstream ost;
+        ost << "id.bucket == " << BucketId(16, 4006).getId() ;
+        PARSEI(ost.str(), *_doc[0], True);
+    }
+    {
+        std::ostringstream ost;
+        ost << "id.bucket == " << BucketId(17, 4006).getId() ;
+        PARSEI(ost.str(), *_doc[0], False);
+    }
+    {
+        std::ostringstream ost;
+        ost << "id.bucket == " << BucketId(17, 69542).getId() ;
+        PARSEI(ost.str(), *_doc[0], True);
+    }
+    {
+        std::ostringstream ost;
+        ost << "id.bucket == " << BucketId(16, 1234).getId() ;
+        PARSEI(ost.str(), *_doc[0], False);
+    }
+
+    PARSEI("id.bucket == \"foo\"", *_doc[0], Invalid);
+
+    std::string gidmatcher = "id.gid == \"" + _doc[0]->getId().getGlobalId().toString() + "\"";
+    PARSEI(gidmatcher, *_doc[0], True);
+
+    PARSEI("id.user=123456789 and id = \"userdoc:footype:123456789:aardvark\"", *_doc[5], True);
+    PARSEI("id == \"userdoc:footype:123456789:badger\"", *_doc[5], False);
+
+    PARSEI("id.user = 1234", *_doc[8], True);
+    PARSEI("id.group == \"1234\"", *_doc[8], True);
+    PARSEI("id.group == \"mygroup\"", *_doc[9], True);
+
+    // Searchcolumn policy
+    PARSE("searchcolumn.10 == 8", *_doc[0], True);
+}
+
+void DocumentSelectParserTest::testOperators4()
+{
+    createDocs();
+
+        // Branch operators
+    PARSEI("true and false", *_doc[0], False);
+    PARSEI("true and true", *_doc[0], True);
+    PARSEI("true or false", *_doc[0], True);
+    PARSEI("false or false", *_doc[0], False);
+    PARSEI("false and true or true and true", *_doc[0], True);
+    PARSEI("false or true and true or false", *_doc[0], True);
+    PARSEI("not false", *_doc[0], True);
+    PARSEI("not true", *_doc[0], False);
+    PARSEI("true and not false or false", *_doc[0], True);
+    PARSEI("((243 < 300) and (\"FOO\".lowercase() == \"foo\"))", *_doc[0], True);
+
+        // Invalid branching. testdoctype1.content = 1 is invalid
+    PARSE("testdoctype1.content = 1 and true", *_doc[0], Invalid);
+    PARSE("testdoctype1.content = 1 or true", *_doc[0], True);
+    PARSE("testdoctype1.content = 1 and false", *_doc[0], False);
+    PARSE("testdoctype1.content = 1 or false", *_doc[0], Invalid);
+    PARSE("true and testdoctype1.content = 1", *_doc[0], Invalid);
+    PARSE("true or testdoctype1.content = 1", *_doc[0], True);
+    PARSE("false and testdoctype1.content = 1", *_doc[0], False);
+    PARSE("false or testdoctype1.content = 1", *_doc[0], Invalid);
+}
+
+void DocumentSelectParserTest::testOperators5()
+{
+    createDocs();
+
+        // Functions
+    PARSE("testdoctype1.hstringval.lowercase() == \"Yet\"", *_doc[3], False);
+    PARSE("testdoctype1.hstringval.lowercase() == \"yet\"", *_doc[3], True);
+    PARSE("testdoctype1.hfloatval.lowercase() == \"yet\"", *_doc[3], Invalid);
+    PARSEI("\"bar\".hash() == -2012135647395072713", *_doc[0], True);
+    PARSEI("\"bar\".hash().abs() == 2012135647395072713", *_doc[0], True);
+    PARSEI("null.hash() == 123", *_doc[0], Invalid);
+    PARSEI("(0.234).hash() == 123", *_doc[0], False);
+    PARSEI("(0.234).lowercase() == 123", *_doc[0], Invalid);
+    PARSE("\"foo\".hash() == 123", *_doc[0], False);
+    PARSEI("(234).hash() == 123", *_doc[0], False);
+    PARSE("now() > 1311862500", *_doc[10], True);
+    PARSE("now() < 1611862500", *_doc[10], True);
+    PARSE("now() < 1311862500", *_doc[10], False);
+    PARSE("now() > 1611862500", *_doc[10], False);
+
+        // Arithmetics
+    PARSEI("id.specific.hash() % 10 = 8", *_doc[0], True);
+    PARSEI("id.specific.hash() % 10 = 2", *_doc[0], False);
+    PARSEI("\"foo\" + \"bar\" = \"foobar\"", *_doc[0], True);
+    PARSEI("\"foo\" + 4 = 25", *_doc[0], Invalid);
+    PARSEI("34.0 % 4 = 4", *_doc[0], Invalid);
+    PARSEI("-6 % 10 = -6", *_doc[0], True);
+}
+
+void DocumentSelectParserTest::testOperators6()
+{
+    createDocs();
+
+        // CJK
+        // Assuming the characters " \ ? * is not used as part of CJK tokens
+    PARSE("testdoctype1.content=\"\xE4\xB8\xBA\xE4\xBB\x80\"", *_doc[3], True);
+    PARSE("testdoctype1.content=\"\xE4\xB7\xBA\xE4\xBB\x80\"", *_doc[3], False);
+
+        // Structs and arrays
+    PARSE("testdoctype1.mystruct", *_doc[0], False);
+    PARSE("testdoctype1.mystruct", *_doc[1], True);
+    PARSE("testdoctype1.mystruct", *_doc[2], False);
+    PARSE("testdoctype1.mystruct == testdoctype1.mystruct", *_doc[0], True);
+    PARSE("testdoctype1.mystruct == testdoctype1.mystruct", *_doc[1], True);
+    PARSE("testdoctype1.mystruct != testdoctype1.mystruct", *_doc[0], False);
+    PARSE("testdoctype1.mystruct != testdoctype1.mystruct", *_doc[1], False);
+    PARSE("testdoctype1.mystruct < testdoctype1.mystruct", *_doc[0], Invalid);
+    PARSE("testdoctype1.mystruct < testdoctype1.mystruct", *_doc[1], False);
+    PARSE("testdoctype1.mystruct < 5", *_doc[1], False);
+    //  PARSE("testdoctype1.mystruct == \"foo\"", *_doc[1], Invalid);
+    PARSE("testdoctype1.mystruct.key == 14", *_doc[0], False);
+    PARSE("testdoctype1.mystruct.value == \"structval\"", *_doc[0], False);
+    PARSE("testdoctype1.mystruct.key == 14", *_doc[1], True);
+    PARSE("testdoctype1.mystruct.value == \"structval\"", *_doc[1], True);
+    PARSE("testdoctype1.structarray", *_doc[0], False);
+    PARSE("testdoctype1.structarray", *_doc[1], True);
+    PARSE("testdoctype1.structarray", *_doc[2], False);
+    PARSE("testdoctype1.structarray == testdoctype1.structarray",
+          *_doc[0], True);
+    PARSE("testdoctype1.structarray < testdoctype1.structarray",
+          *_doc[0], Invalid);
+    PARSE("testdoctype1.structarray == testdoctype1.structarray",
+          *_doc[1], True);
+    PARSE("testdoctype1.structarray < testdoctype1.structarray",
+          *_doc[1], False);
+    PARSE("testdoctype1.headerlongval<0", *_doc[6], False);
+    PARSE("testdoctype1.headerlongval<0", *_doc[7], True);
+}
+
+void DocumentSelectParserTest::testOperators7()
+{
+    createDocs();
+
+    PARSE("testdoctype1.structarray.key == 15", *_doc[0], False);
+    PARSE("testdoctype1.structarray[4].key == 15", *_doc[0], False);
+    PARSE("testdoctype1.structarray", *_doc[1], True);
+    PARSE("testdoctype1.structarray.key == 15", *_doc[1], True);
+    PARSE("testdoctype1.structarray[1].key == 16", *_doc[1], True);
+    PARSE("testdoctype1.structarray[1].key = 16", *_doc[1], True);
+    PARSE("testdoctype1.structarray.value == \"structval1\"", *_doc[0], False);
+    PARSE("testdoctype1.structarray[4].value == \"structval1\"", *_doc[0], False);
+    PARSE("testdoctype1.structarray.value == \"structval1\"", *_doc[1], True);
+    PARSE("testdoctype1.structarray[0].value == \"structval1\"", *_doc[1], True);
+    // Globbing of array-of-struct fields
+    PARSE("testdoctype1.structarray.key = 15", *_doc[0], False);
+    PARSE("testdoctype1.structarray.key = 15", *_doc[2], False);
+    PARSE("testdoctype1.structarray.key = 15", *_doc[1], True);
+    PARSE("testdoctype1.structarray.value = \"structval2\"", *_doc[2], Invalid); // Invalid due to lhs being NullValue
+    PARSE("testdoctype1.structarray.value = \"*ctval*\"", *_doc[1], True);
+    PARSE("testdoctype1.structarray[1].value = \"structval2\"", *_doc[1], True);
+    PARSE("testdoctype1.structarray[1].value = \"batman\"", *_doc[1], False);
+    // Regexp of array-of-struct fields
+    PARSE("testdoctype1.structarray.value =~ \"structval[1-9]\"", *_doc[1], True);
+    PARSE("testdoctype1.structarray.value =~ \"structval[a-z]\"", *_doc[1], False);
+    // Globbing/regexp of struct fields
+    PARSE("testdoctype1.mystruct.value = \"struc?val\"", *_doc[0], Invalid); // Invalid due to lhs being NullValue
+    PARSE("testdoctype1.mystruct.value = \"struc?val\"", *_doc[1], True);
+    PARSE("testdoctype1.mystruct.value =~ \"struct.*\"", *_doc[0], Invalid); // Ditto here
+    PARSE("testdoctype1.mystruct.value =~ \"struct.*\"", *_doc[1], True);
+
+    PARSE("testdoctype1.structarray[$x].key == 15 AND testdoctype1.structarray[$x].value == \"structval1\"", *_doc[1], True);
+    PARSE("testdoctype1.structarray[$x].key == 15 AND testdoctype1.structarray[$x].value == \"structval2\"", *_doc[1], False);
+    PARSE("testdoctype1.structarray[$x].key == 15 AND testdoctype1.structarray[$y].value == \"structval2\"", *_doc[1], True);
+}
+
+void DocumentSelectParserTest::testOperators8()
+{
+    createDocs();
+
+    PARSE("testdoctype1.mymap", *_doc[0], False);
+    PARSE("testdoctype1.mymap", *_doc[1], True);
+    PARSE("testdoctype1.mymap{3}", *_doc[1], True);
+    PARSE("testdoctype1.mymap{9}", *_doc[1], False);
+    PARSE("testdoctype1.mymap{3} == \"a\"", *_doc[1], True);
+    PARSE("testdoctype1.mymap{3} == \"b\"", *_doc[1], False);
+    PARSE("testdoctype1.mymap{9} == \"b\"", *_doc[1], False);
+    PARSE("testdoctype1.mymap.value == \"a\"", *_doc[1], True);
+    PARSE("testdoctype1.mymap.value == \"d\"", *_doc[1], False);
+    PARSE("testdoctype1.mymap{3} = \"a\"", *_doc[1], True);
+    PARSE("testdoctype1.mymap{3} = \"b\"", *_doc[1], False);
+    PARSE("testdoctype1.mymap{3} =~ \"a\"", *_doc[1], True);
+    PARSE("testdoctype1.mymap{3} =~ \"b\"", *_doc[1], False);
+    PARSE("testdoctype1.mymap.value = \"a\"", *_doc[1], True);
+    PARSE("testdoctype1.mymap.value = \"d\"", *_doc[1], False);
+    PARSE("testdoctype1.mymap.value =~ \"a\"", *_doc[1], True);
+    PARSE("testdoctype1.mymap.value =~ \"d\"", *_doc[1], False);
+    PARSE("testdoctype1.mymap == 3", *_doc[1], True);
+    PARSE("testdoctype1.mymap == 4", *_doc[1], False);
+    PARSE("testdoctype1.mymap = 3", *_doc[1], True); // Fallback to ==
+    PARSE("testdoctype1.mymap = 4", *_doc[1], False); // Fallback to ==
+
+    PARSE("testdoctype1.structarrmap{$x}[$y].key == 15 AND testdoctype1.structarrmap{$x}[$y].value == \"structval1\"", *_doc[1], True);
+    PARSE("testdoctype1.structarrmap.value[$y].key == 15 AND testdoctype1.structarrmap.value[$y].value == \"structval1\"", *_doc[1], True);
+    PARSE("testdoctype1.structarrmap{$x}[$y].key == 15 AND testdoctype1.structarrmap{$x}[$y].value == \"structval2\"", *_doc[1], False);
+    PARSE("testdoctype1.structarrmap.value[$y].key == 15 AND testdoctype1.structarrmap.value[$y].value == \"structval2\"", *_doc[1], False);
+    PARSE("testdoctype1.structarrmap{$x}[$y].key == 15 AND testdoctype1.structarrmap{$y}[$x].value == \"structval2\"", *_doc[1], False);
+}
+
+void DocumentSelectParserTest::testOperators9()
+{
+    createDocs();
+
+    PARSE("testdoctype1.stringweightedset", *_doc[1], True);
+    PARSE("testdoctype1.stringweightedset{val1}", *_doc[1], True);
+    PARSE("testdoctype1.stringweightedset{val1} == 1", *_doc[1], True);
+    PARSE("testdoctype1.stringweightedset{val1} == 2", *_doc[1], False);
+    PARSE("testdoctype1.stringweightedset == \"val1\"", *_doc[1], True);
+    PARSE("testdoctype1.stringweightedset = \"val*\"", *_doc[1], True);
+    PARSE("testdoctype1.stringweightedset =~ \"val[0-9]\"", *_doc[1], True);
+    PARSE("testdoctype1.stringweightedset == \"val5\"", *_doc[1], False);
+    PARSE("testdoctype1.stringweightedset = \"val5\"", *_doc[1], False);
+    PARSE("testdoctype1.stringweightedset =~ \"val5\"", *_doc[1], False);
+
+    PARSE("testdoctype1.structarrmap{$x}.key == 15 AND testdoctype1.stringweightedset{$x}", *_doc[1], True);
+    PARSE("testdoctype1.structarrmap{$x}.key == 17 AND testdoctype1.stringweightedset{$x}", *_doc[1], False);
+
+    PARSE("testdoctype1.structarray.key < 16", *_doc[1], True);
+    PARSE("testdoctype1.structarray.key < 15", *_doc[1], False);
+    PARSE("testdoctype1.structarray.key > 15", *_doc[1], True);
+    PARSE("testdoctype1.structarray.key > 16", *_doc[1], False);
+    PARSE("testdoctype1.structarray.key <= 15", *_doc[1], True);
+    PARSE("testdoctype1.structarray.key <= 14", *_doc[1], False);
+    PARSE("testdoctype1.structarray.key >= 16", *_doc[1], True);
+    PARSE("testdoctype1.structarray.key >= 17", *_doc[1], False);
+}
+
+namespace {
+
+    class TestVisitor : public select::Visitor {
+    private:
+        std::ostringstream data;
+
+    public:
+        virtual ~TestVisitor() {}
+
+        void visitConstant(const select::Constant& node)
+        {
+            data << "CONSTANT(" << node << ")";
+        }
+
+        virtual void
+        visitInvalidConstant(const select::InvalidConstant& node)
+        {
+            data << "INVALIDCONSTANT(" << node << ")";
+        }
+
+        void visitDocumentType(const select::DocType& node)
+        {
+            data << "DOCTYPE(" << node << ")";
+        }
+
+        void visitComparison(const select::Compare& node)
+        {
+            data << "COMPARE(" << node.getLeft() << " "
+                 << node.getOperator() << " " << node.getRight() << ")";
+        }
+
+        void visitAndBranch(const select::And& node)
+        {
+            data << "AND(";
+            node.getLeft().visit(*this);
+            data << ", ";
+            node.getRight().visit(*this);
+            data << ")";
+        }
+
+        void visitOrBranch(const select::Or& node)
+        {
+            data << "OR(";
+            node.getLeft().visit(*this);
+            data << ", ";
+            node.getRight().visit(*this);
+            data << ")";
+        }
+
+        void visitNotBranch(const select::Not& node)
+        {
+            data << "NOT(";
+            node.getChild().visit(*this);
+            data << ")";
+        }
+
+        virtual void
+        visitArithmeticValueNode(const select::ArithmeticValueNode &)
+        {
+        }
+
+        virtual void
+        visitFunctionValueNode(const select::FunctionValueNode &)
+        {
+        }
+
+        virtual void
+        visitIdValueNode(const select::IdValueNode &)
+        {
+        }
+
+        virtual void
+        visitSearchColumnValueNode(const select::SearchColumnValueNode &)
+        {
+        }
+
+        virtual void
+        visitFieldValueNode(const select::FieldValueNode &)
+        {
+        }
+
+        virtual void
+        visitFloatValueNode(const select::FloatValueNode &)
+        {
+        }
+
+        virtual void
+        visitVariableValueNode(const select::VariableValueNode &)
+        {
+        }
+
+        virtual void
+        visitIntegerValueNode(const select::IntegerValueNode &)
+        {
+        }
+
+        virtual void
+        visitCurrentTimeValueNode(const select::CurrentTimeValueNode &)
+        {
+        }
+
+        virtual void
+        visitStringValueNode(const select::StringValueNode &)
+        {
+        }
+
+        virtual void
+        visitNullValueNode(const select::NullValueNode &)
+        {
+        }
+
+        virtual void
+        visitInvalidValueNode(const select::InvalidValueNode &)
+        {
+        }
+
+        std::string getVisitString() { return data.str(); }
+    };
+
+}
+
+void DocumentSelectParserTest::testVisitor()
+{
+    createDocs();
+
+    std::unique_ptr<select::Node> root(_parser->parse(
+        "true or testdoctype1 and (not id.user = 12 or testdoctype1.hstringval = \"ola\") and "
+        "testdoctype1.headerval"));
+
+    TestVisitor v;
+    root->visit(v);
+    std::string expected =
+        "OR(CONSTANT(true), "
+            "AND(DOCTYPE(testdoctype1), "
+                "AND(OR(NOT(COMPARE(id.user = 12)), "
+                        "COMPARE(testdoctype1.hstringval = \"ola\")), "
+                    "COMPARE(testdoctype1.headerval != null)"
+                ")"
+            ")"
+        ")";
+    CPPUNIT_ASSERT_EQUAL(expected, v.getVisitString());
+}
+
+void DocumentSelectParserTest::testBodyFieldDetection()
+{
+
+    {
+        std::unique_ptr<select::Node> root(_parser->parse("testdoctype1"));
+        select::BodyFieldDetector detector(*_repo);
+
+        root->visit(detector);
+        CPPUNIT_ASSERT(!detector.foundBodyField);
+        CPPUNIT_ASSERT(detector.foundHeaderField);
+    }
+
+    {
+        std::unique_ptr<select::Node> root(_parser->parse("testdoctype1 AND id.user=1234"));
+        select::BodyFieldDetector detector(*_repo);
+
+        root->visit(detector);
+        CPPUNIT_ASSERT(!detector.foundBodyField);
+        CPPUNIT_ASSERT(detector.foundHeaderField);
+    }
+
+    {
+        std::unique_ptr<select::Node> root(_parser->parse("testdoctype1.headerval=123"));
+        select::BodyFieldDetector detector(*_repo);
+
+        root->visit(detector);
+        CPPUNIT_ASSERT(!detector.foundBodyField);
+        CPPUNIT_ASSERT(detector.foundHeaderField);
+    }
+
+    {
+        std::unique_ptr<select::Node> root(_parser->parse("testdoctype1.content"));
+        select::BodyFieldDetector detector(*_repo);
+
+        root->visit(detector);
+        CPPUNIT_ASSERT(detector.foundBodyField);
+    }
+
+    {
+        std::unique_ptr<select::Node> root(_parser->parse(
+                                                 "true or testdoctype1 and (not id.user = 12 or testdoctype1.hstringval = \"ola\") and "
+                                                 "testdoctype1.headerval"));
+
+        select::BodyFieldDetector detector(*_repo);
+
+        root->visit(detector);
+        CPPUNIT_ASSERT(!detector.foundBodyField);
+    }
+
+}
+
+void DocumentSelectParserTest::testDocumentUpdates()
+{
+    testDocumentUpdates0();
+    testDocumentUpdates1();
+    testDocumentUpdates2();
+    testDocumentUpdates3();
+    testDocumentUpdates4();
+}
+
+void DocumentSelectParserTest::testDocumentUpdates0()
+{
+    createDocs();
+
+    /* Code for tracing result to see what went wrong
+    {
+        std::ostringstream ost;
+        _parser->parse("id.bucket == 4006")->trace(*_update[0], ost);
+        ost << "\n\n";
+        _parser->parse("id.bucket == 4006")
+                ->trace(_update[0]->getId(), ost);
+        std::cerr << ost.str() << "\n";
+    } // */
+    PARSEI("", *_update[0], True);
+    PARSEI("30 < 10", *_update[0], False);
+    PARSEI("10 < 30", *_update[0], True);
+    PARSEI("30 < 10", *_update[0], False);
+    PARSEI("10 < 30", *_update[0], True);
+    PARSEI("30 <= 10", *_update[0], False);
+    PARSEI("10 <= 30", *_update[0], True);
+    PARSEI("30 <= 30", *_update[0], True);
+    PARSEI("10 >= 30", *_update[0], False);
+    PARSEI("30 >= 10", *_update[0], True);
+    PARSEI("30 >= 30", *_update[0], True);
+    PARSEI("10 > 30", *_update[0], False);
+    PARSEI("30 > 10", *_update[0], True);
+    PARSEI("30 == 10", *_update[0], False);
+    PARSEI("30 == 30", *_update[0], True);
+    PARSEI("30 != 10", *_update[0], True);
+    PARSEI("30 != 30", *_update[0], False);
+    PARSEI("\"foo\" != \"bar\"", *_update[0], True);
+    PARSEI("\"foo\" != \"foo\"", *_update[0], False);
+    PARSEI("\"foo\" == \"bar\"", *_update[0], False);
+    PARSEI("\"foo\" == \"foo\"", *_update[0], True);
+    PARSEI("\"bar\" = \"a\"", *_update[0], False);
+    PARSEI("\"bar\" = \"*a*\"", *_update[0], True);
+    PARSEI("\"bar\" = \"\"", *_update[0], False);
+    PARSEI("\"\" = \"\"", *_update[0], True);
+    PARSEI("\"bar\" =~ \"^a$\"", *_update[0], False);
+    PARSEI("\"bar\" =~ \"a\"", *_update[0], True);
+    PARSEI("\"bar\" =~ \"\"", *_update[0], True);
+    PARSEI("\"\" =~ \"\"", *_update[0], True);
+    PARSEI("30 = 10", *_update[0], False);
+    PARSEI("30 = 30", *_update[0], True);
+}
+
+void DocumentSelectParserTest::testDocumentUpdates1()
+{
+    createDocs();
+
+        // Mix of types should within numbers, but otherwise not match
+    PARSEI("30 < 10.2", *_update[0], False);
+    PARSEI("10.2 < 30", *_update[0], True);
+    PARSEI("30 < \"foo\"", *_update[0], Invalid);
+    PARSEI("30 > \"foo\"", *_update[0], Invalid);
+    PARSEI("30 != \"foo\"", *_update[0], Invalid);
+    PARSEI("14.2 <= \"foo\"", *_update[0], Invalid);
+    PARSEI("null == null", *_update[0], True);
+    PARSEI("null = null", *_update[0], True);
+    PARSEI("\"bar\" == null", *_update[0], False);
+    PARSEI("14.3 == null", *_update[0], False);
+    PARSEI("null = 0", *_update[0], False);
+
+        // Field values
+    PARSE("testdoctype1.headerval = 24", *_update[0], Invalid);
+    PARSE("testdoctype1.hfloatval = 2.0", *_update[0], Invalid);
+    PARSE("testdoctype1.content = \"bar\"", *_update[0], Invalid);
+    PARSE("testdoctype1.hstringval == testdoctype1.content", *_update[0], Invalid);
+
+        // Document types
+    PARSE("testdoctype1", *_update[0], True);
+    PARSE("testdoctype2", *_update[0], False);
+
+        // Inherited doctypes
+    PARSE("testdoctype2", *_update[4], True);
+    PARSE("testdoctype2", *_update[3], False);
+    PARSE("testdoctype1", *_update[4], True);
+    PARSE("testdoctype1.headerval = 10", *_update[4], Invalid);
+}
+
+void DocumentSelectParserTest::testDocumentUpdates2()
+{
+    createDocs();
+
+        // Id values
+    PARSEI("id == \"doc:myspace:anything\"", *_update[0], True);
+    PARSEI(" iD==  \"doc:myspace:anything\"  ", *_update[0], True);
+    PARSEI("id == \"doc:myspa:nything\"", *_update[0], False);
+    PARSEI("Id.scHeme == \"doc\"", *_update[0], True);
+    PARSEI("id.scheme == \"userdoc\"", *_update[0], False);
+    PARSEI("Id.namespaCe == \"myspace\"", *_update[0], True);
+    PARSEI("id.NaMespace == \"pace\"", *_update[0], False);
+    PARSEI("id.specific == \"anything\"", *_update[0], True);
+    PARSEI("id.user=1234", *_update[2], True);
+    PARSEI("id.user == 1234", *_update[0], Invalid);
+    PARSEI("id.group == 1234", *_update[3], Invalid);
+    PARSEI("id.group == \"yahoo\"", *_update[3], True);
+    PARSEI("id.bucket == 1234", *_update[0], False);
+    {
+        std::ostringstream ost;
+        ost << "id.bucket == " << BucketId(16, 4006).getId();
+        PARSEI(ost.str(), *_update[0], True);
+    }
+    PARSEI("id.bucket == \"foo\"", *_update[0], Invalid);
+}
+
+void DocumentSelectParserTest::testDocumentUpdates3()
+{
+    createDocs();
+
+        // Branch operators
+    PARSEI("true and false", *_update[0], False);
+    PARSEI("true and true", *_update[0], True);
+    PARSEI("true or false", *_update[0], True);
+    PARSEI("false or false", *_update[0], False);
+    PARSEI("false and true or true and true", *_update[0], True);
+    PARSEI("false or true and true or false", *_update[0], True);
+    PARSEI("not false", *_update[0], True);
+    PARSEI("not true", *_update[0], False);
+    PARSEI("true and not false or false", *_update[0], True);
+    PARSEI("((243 < 300) and (\"FOO\".lowercase() == \"foo\"))", *_update[0], True);
+
+        // Invalid branching. testdoctype1.content = 1 is invalid
+    PARSE("testdoctype1.content = 1 and true", *_update[0], Invalid);
+    PARSE("testdoctype1.content = 1 or true", *_update[0], True);
+    PARSE("testdoctype1.content = 1 and false", *_update[0], False);
+    PARSE("testdoctype1.content = 1 or false", *_update[0], Invalid);
+    PARSE("true and testdoctype1.content = 1", *_update[0], Invalid);
+    PARSE("true or testdoctype1.content = 1", *_update[0], True);
+    PARSE("false and testdoctype1.content = 1", *_update[0], False);
+    PARSE("false or testdoctype1.content = 1", *_update[0], Invalid);
+}
+
+void DocumentSelectParserTest::testDocumentUpdates4()
+{
+    createDocs();
+
+        // Functions
+    PARSEI("\"bar\".hash() == -2012135647395072713", *_update[0], True);
+    PARSEI("\"bar\".hash().abs() == 2012135647395072713", *_update[0], True);
+    PARSEI("null.hash() == 123", *_update[0], Invalid);
+    PARSEI("(0.234).hash() == 123", *_update[0], False);
+    PARSEI("(0.234).lowercase() == 123", *_update[0], Invalid);
+    PARSEI("\"foo\".hash() == 123", *_update[0], False);
+    PARSEI("(234).hash() == 123", *_update[0], False);
+
+        // Arithmetics
+    PARSEI("id.specific.hash() % 10 = 8", *_update[0], True);
+    PARSEI("id.specific.hash() % 10 = 2", *_update[0], False);
+    PARSEI("\"foo\" + \"bar\" = \"foobar\"", *_update[0], True);
+    PARSEI("\"foo\" + 4 = 25", *_update[0], Invalid);
+    PARSEI("34.0 % 4 = 4", *_update[0], Invalid);
+    PARSEI("-6 % 10 = -6", *_update[0], True);
+}
+
+void DocumentSelectParserTest::testUtf8()
+{
+    createDocs();
+    std::string utf8name(u8"H\u00e5kon");
+    CPPUNIT_ASSERT_EQUAL(size_t(6), utf8name.size());
+
+    /// \todo TODO (was warning):  UTF8 test for glob/regex support in selection language disabled. Known not to work
+//    boost::u32regex rx = boost::make_u32regex("H.kon");
+//    CPPUNIT_ASSERT_EQUAL(true, boost::u32regex_match(utf8name, rx));
+
+    _doc.push_back(createDoc(
+        "testdoctype1", "doc:myspace:utf8doc", 24, 2.0, utf8name, "bar"));
+//    PARSE("testdoctype1.hstringval = \"H?kon\"", *_doc[_doc.size()-1], True);
+//    PARSE("testdoctype1.hstringval =~ \"H.kon\"", *_doc[_doc.size()-1], True);
+}
+
+} // document
diff --git a/document/src/tests/documentselecttest.h b/document/src/tests/documentselecttest.h
new file mode 100644
index 00000000000..d8cb4d427e1
--- /dev/null
+++ b/document/src/tests/documentselecttest.h
@@ -0,0 +1,34 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/* $Id$*/
+#pragma once
+
+#include <cppunit/extensions/HelperMacros.h>
+
+class DocumentSelect_Test : public CppUnit::TestFixture {
+  CPPUNIT_TEST_SUITE( DocumentSelect_Test);
+  CPPUNIT_TEST(testEquals);
+  CPPUNIT_TEST(testLt);
+  CPPUNIT_TEST(testGt);
+  CPPUNIT_TEST(testAnd);
+  CPPUNIT_TEST(testOr);
+  CPPUNIT_TEST(testNot);
+  CPPUNIT_TEST(testConfig1);
+  CPPUNIT_TEST(testConfig2);
+  CPPUNIT_TEST_SUITE_END();
+
+public:
+  void setUp();
+
+  void tearDown();
+protected:
+  void testEquals();
+  void testLt();
+  void testGt();
+  void testAnd();
+  void testOr();
+  void testNot();
+  void testConfig1();
+  void testConfig2();
+};
+
+
diff --git a/document/src/tests/documenttestcase.cpp b/document/src/tests/documenttestcase.cpp
new file mode 100644
index 00000000000..475cf64128e
--- /dev/null
+++ b/document/src/tests/documenttestcase.cpp
@@ -0,0 +1,1434 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/document/base/testdocman.h>
+#include <vespa/document/base/testdocrepo.h>
+#include <vespa/document/fieldvalue/fieldvalues.h>
+#include <vespa/vespalib/io/fileutil.h>
+#include <vespa/vdstestlib/cppunit/macros.h>
+
+#include <vespa/document/annotation/spantree.h>
+#include <vespa/document/config/config-documenttypes.h>
+#include <vespa/document/datatype/annotationreferencedatatype.h>
+#include <vespa/document/repo/configbuilder.h>
+#include <vespa/document/repo/documenttyperepo.h>
+#include <vespa/document/serialization/vespadocumentdeserializer.h>
+#include <vespa/document/serialization/vespadocumentserializer.h>
+#include <vespa/vespalib/objects/nbostream.h>
+#include <fstream>
+
+using vespalib::nbostream;
+
+using namespace document::config_builder;
+
+namespace document {
+
+struct DocumentTest : public CppUnit::TestFixture {
+    void testTraversing();
+    void testFieldPath();
+    void testModifyDocument();
+    void testVariables();
+    void testSimpleUsage();
+    void testReadSerializedFile();
+    void testReadSerializedFileCompressed();
+    void testReadSerializedAllVersions();
+    void testGenerateSerializedFile();
+    void testGetURIFromSerialized();
+    void testBogusserialize();
+    void testCRC32();
+    void testHasChanged();
+    void testSplitSerialization();
+    void testSliceSerialize();
+    void testCompression();
+    void testCompressionConfigured();
+    void testUnknownEntries();
+    void testAnnotationDeserialization();
+    void testGetSerializedSize();
+    void testDeserializeMultiple();
+    void testSizeOf();
+
+    CPPUNIT_TEST_SUITE(DocumentTest);
+    CPPUNIT_TEST(testFieldPath);
+    CPPUNIT_TEST(testTraversing);
+    CPPUNIT_TEST(testModifyDocument);
+    CPPUNIT_TEST(testVariables);
+    CPPUNIT_TEST(testSimpleUsage);
+    CPPUNIT_TEST(testReadSerializedFile);
+    CPPUNIT_TEST(testReadSerializedFileCompressed);
+    CPPUNIT_TEST(testReadSerializedAllVersions);
+    CPPUNIT_TEST(testGenerateSerializedFile);
+    CPPUNIT_TEST(testGetURIFromSerialized);
+    CPPUNIT_TEST(testBogusserialize);
+    CPPUNIT_TEST(testCRC32);
+    CPPUNIT_TEST(testHasChanged);
+    CPPUNIT_TEST(testSplitSerialization);
+    CPPUNIT_TEST(testSliceSerialize);
+    CPPUNIT_TEST(testCompression);
+    CPPUNIT_TEST(testCompressionConfigured);
+    CPPUNIT_TEST(testUnknownEntries);
+    CPPUNIT_TEST(testAnnotationDeserialization);
+    CPPUNIT_TEST(testGetSerializedSize);
+    CPPUNIT_TEST(testDeserializeMultiple);
+    CPPUNIT_TEST(testSizeOf);
+    CPPUNIT_TEST_SUITE_END();
+};
+
+CPPUNIT_TEST_SUITE_REGISTRATION(DocumentTest);
+
+void DocumentTest::testSizeOf()
+{
+    CPPUNIT_ASSERT_EQUAL(120ul, sizeof(Document));
+    CPPUNIT_ASSERT_EQUAL(64ul, sizeof(StructFieldValue));
+    CPPUNIT_ASSERT_EQUAL(16ul, sizeof(StructuredFieldValue));
+    CPPUNIT_ASSERT_EQUAL(112ul, sizeof(SerializableArray));
+}
+
+void DocumentTest::testFieldPath()
+{
+    FieldPathEntry empty;
+    empty.asString();
+    const vespalib::string testValues[] = { "{}", "", "",
+                                       "{}r", "", "r",
+                                       "{{}}", "{", "}",
+                                       "{{}}r", "{", "}r",
+                                       "{\"{}\"}", "{}", "",
+                                       "{\"{}\"}r", "{}", "r",
+                                       "{\"{\\a}\"}r", "{a}", "r",
+                                       "{\"{\\\"}\"}r", "{\"}", "r",
+                                       "{\"{\\\\}\"}r", "{\\}", "r",
+                                       "{$x}", "$x", "",
+                                       "{$x}[$y]", "$x", "[$y]",
+                                       "{$x}.ss", "$x", ".ss",
+                                       "{\"\"}", "", ""
+                                     };
+    for (size_t i(0); i < sizeof(testValues)/sizeof(testValues[0]); i+=3) {
+        vespalib::string tmp = testValues[i];
+        vespalib::string key = FieldPathEntry::parseKey(tmp);
+        CPPUNIT_ASSERT_EQUAL(testValues[i+1], key);
+        CPPUNIT_ASSERT_EQUAL(testValues[i+2], tmp);
+    }
+}
+
+void DocumentTest::testTraversing()
+{
+    class Handler : public FieldValue::IteratorHandler {
+    public:
+        const std::string & getResult() const { return _result; }
+    private:
+        virtual void onPrimitive(const Content&)       { _result += 'P'; }
+        virtual void onCollectionStart(const Content&) { _result += '['; }
+        virtual void onCollectionEnd(const Content&)   { _result += ']'; }
+        virtual void onStructStart(const Content&)     { _result += '<'; }
+        virtual void onStructEnd(const Content&)       { _result += '>'; }
+        std::string _result;
+    };
+
+    Field primitive1("primitive1", 1, *DataType::INT, true);
+    Field primitive2("primitive2", 2, *DataType::INT, true);
+    StructDataType struct1("struct1");
+    struct1.addField(primitive1);
+    struct1.addField(primitive2);
+
+    ArrayDataType iarr(*DataType::INT);
+    ArrayDataType sarr(struct1);
+    Field iarrF("iarray", 21, iarr, true);
+    Field sarrF("sarray", 22, sarr, true);
+
+    StructDataType struct2("struct2");
+    struct2.addField(primitive1);
+    struct2.addField(primitive2);
+    struct2.addField(iarrF);
+    struct2.addField(sarrF);
+    Field s2("ss", 12, struct2, true);
+
+    StructDataType struct3("struct3");
+    struct3.addField(primitive1);
+    struct3.addField(s2);
+
+    Field structl1s1("l1s1", 11, struct3, true);
+
+    DocumentType type("test");
+    type.addField(primitive1);
+    type.addField(structl1s1);
+
+    Document doc(type, DocumentId("doc::testdoc"));
+    doc.setValue(primitive1, IntFieldValue(1));
+
+    StructFieldValue l1s1(struct3);
+    l1s1.setValue(primitive1, IntFieldValue(2));
+
+    StructFieldValue l2s1(struct2);
+    l2s1.setValue(primitive1, IntFieldValue(3));
+    l2s1.setValue(primitive2, IntFieldValue(4));
+    ArrayFieldValue iarr1(iarr);
+    iarr1.add(IntFieldValue(11));
+    iarr1.add(IntFieldValue(12));
+    iarr1.add(IntFieldValue(13));
+    ArrayFieldValue sarr1(sarr);
+    StructFieldValue l3s1(struct1);
+    l3s1.setValue(primitive1, IntFieldValue(1));
+    l3s1.setValue(primitive2, IntFieldValue(2));
+    sarr1.add(l3s1);
+    sarr1.add(l3s1);
+    l2s1.setValue(iarrF, iarr1);
+    l2s1.setValue(sarrF, sarr1);
+
+    l1s1.setValue(s2, l2s1);
+    doc.setValue(structl1s1, l1s1);
+
+    Handler fullTraverser;
+    FieldPath empty;
+    doc.iterateNested(empty.begin(), empty.end(), fullTraverser);
+    CPPUNIT_ASSERT_EQUAL(fullTraverser.getResult(),
+                         std::string("<P<P<PP[PPP][<PP><PP>]>>>"));
+}
+
+class VariableIteratorHandler : public FieldValue::IteratorHandler {
+public:
+    std::string retVal;
+
+    virtual void onPrimitive(const Content & fv) {
+        for (VariableMap::iterator iter = getVariables().begin(); iter != getVariables().end(); iter++) {
+            retVal += vespalib::make_string("%s: %s,", iter->first.c_str(), iter->second.toString().c_str());
+        }
+        retVal += " - " + fv.getValue().toString() + "\n";
+    };
+};
+
+void
+DocumentTest::testVariables()
+{
+    ArrayDataType iarr(*DataType::INT);
+    ArrayDataType iiarr(static_cast<DataType &>(iarr));
+    ArrayDataType iiiarr(static_cast<DataType &>(iiarr));
+
+    Field iiiarrF("iiiarray", 1, iiiarr, true);
+    DocumentType type("test");
+    type.addField(iiiarrF);
+
+    ArrayFieldValue iiiaV(iiiarr);
+    for (int i = 1; i < 4; i++) {
+        ArrayFieldValue iiaV(iiarr);
+        for (int j = 1; j < 4; j++) {
+            ArrayFieldValue iaV(iarr);
+            for (int k = 1; k < 4; k++) {
+                iaV.add(IntFieldValue(i * j * k));
+            }
+            iiaV.add(iaV);
+        }
+        iiiaV.add(iiaV);
+    }
+
+    Document doc(type, DocumentId("doc::testdoc"));
+    doc.setValue(iiiarrF, iiiaV);
+
+    {
+        VariableIteratorHandler handler;
+        FieldPath::UP path = type.buildFieldPath("iiiarray[$x][$y][$z]");
+        doc.iterateNested(path->begin(), path->end(), handler);
+
+        std::string fasit =
+            "x: 0,y: 0,z: 0, - 1\n"
+            "x: 0,y: 0,z: 1, - 2\n"
+            "x: 0,y: 0,z: 2, - 3\n"
+            "x: 0,y: 1,z: 0, - 2\n"
+            "x: 0,y: 1,z: 1, - 4\n"
+            "x: 0,y: 1,z: 2, - 6\n"
+            "x: 0,y: 2,z: 0, - 3\n"
+            "x: 0,y: 2,z: 1, - 6\n"
+            "x: 0,y: 2,z: 2, - 9\n"
+            "x: 1,y: 0,z: 0, - 2\n"
+            "x: 1,y: 0,z: 1, - 4\n"
+            "x: 1,y: 0,z: 2, - 6\n"
+            "x: 1,y: 1,z: 0, - 4\n"
+            "x: 1,y: 1,z: 1, - 8\n"
+            "x: 1,y: 1,z: 2, - 12\n"
+            "x: 1,y: 2,z: 0, - 6\n"
+            "x: 1,y: 2,z: 1, - 12\n"
+            "x: 1,y: 2,z: 2, - 18\n"
+            "x: 2,y: 0,z: 0, - 3\n"
+            "x: 2,y: 0,z: 1, - 6\n"
+            "x: 2,y: 0,z: 2, - 9\n"
+            "x: 2,y: 1,z: 0, - 6\n"
+            "x: 2,y: 1,z: 1, - 12\n"
+            "x: 2,y: 1,z: 2, - 18\n"
+            "x: 2,y: 2,z: 0, - 9\n"
+            "x: 2,y: 2,z: 1, - 18\n"
+            "x: 2,y: 2,z: 2, - 27\n";
+
+        CPPUNIT_ASSERT_EQUAL(fasit, handler.retVal);
+    }
+
+}
+
+class ModifyIteratorHandler : public FieldValue::IteratorHandler {
+public:
+    ModificationStatus doModify(FieldValue& fv) {
+        StringFieldValue* sfv = dynamic_cast<StringFieldValue*>(&fv);
+        if (sfv != NULL) {
+            *sfv = std::string("newvalue");
+            return MODIFIED;
+        }
+
+        return NOT_MODIFIED;
+    };
+
+    bool onComplex(const Content&) {
+        return false;
+    }
+};
+
+void
+DocumentTest::testModifyDocument()
+{
+    // Create test document type and content
+    Field primitive1("primitive1", 1, *DataType::INT, true);
+    Field primitive2("primitive2", 2, *DataType::INT, true);
+    StructDataType struct1("struct1");
+    struct1.addField(primitive1);
+    struct1.addField(primitive2);
+
+    ArrayDataType iarr(*DataType::INT);
+    ArrayDataType sarr(struct1);
+    Field iarrF("iarray", 21, iarr, true);
+    Field sarrF("sarray", 22, sarr, true);
+
+    MapDataType smap(*DataType::STRING, *DataType::STRING);
+    Field smapF("smap", 23, smap, true);
+
+    StructDataType struct2("struct2");
+    struct2.addField(primitive1);
+    struct2.addField(primitive2);
+    struct2.addField(iarrF);
+    struct2.addField(sarrF);
+    struct2.addField(smapF);
+    Field s2("ss", 12, struct2, true);
+
+    MapDataType structmap(*DataType::STRING, struct2);
+    Field structmapF("structmap", 24, structmap, true);
+
+    WeightedSetDataType wset(*DataType::STRING, false, false);
+    Field wsetF("wset", 25, wset, true);
+
+    WeightedSetDataType structwset(struct2, false, false);
+    Field structwsetF("structwset", 26, structwset, true);
+
+    StructDataType struct3("struct3");
+    struct3.addField(primitive1);
+    struct3.addField(s2);
+    struct3.addField(structmapF);
+    struct3.addField(wsetF);
+    struct3.addField(structwsetF);
+
+    Field structl1s1("l1s1", 11, struct3, true);
+
+    DocumentType type("test");
+    type.addField(primitive1);
+    type.addField(structl1s1);
+
+    Document::UP doc(new Document(type, DocumentId("doc::testdoc")));
+    doc->setValue(primitive1, IntFieldValue(1));
+
+    StructFieldValue l1s1(struct3);
+    l1s1.setValue(primitive1, IntFieldValue(2));
+
+    StructFieldValue l2s1(struct2);
+    l2s1.setValue(primitive1, IntFieldValue(3));
+    l2s1.setValue(primitive2, IntFieldValue(4));
+    StructFieldValue l2s2(struct2);
+    l2s2.setValue(primitive1, IntFieldValue(5));
+    l2s2.setValue(primitive2, IntFieldValue(6));
+    ArrayFieldValue iarr1(iarr);
+    iarr1.add(IntFieldValue(11));
+    iarr1.add(IntFieldValue(12));
+    iarr1.add(IntFieldValue(13));
+    ArrayFieldValue sarr1(sarr);
+    StructFieldValue l3s1(struct1);
+    l3s1.setValue(primitive1, IntFieldValue(1));
+    l3s1.setValue(primitive2, IntFieldValue(2));
+    sarr1.add(l3s1);
+    sarr1.add(l3s1);
+    MapFieldValue smap1(smap);
+    smap1.put(StringFieldValue("leonardo"), StringFieldValue("dicaprio"));
+    smap1.put(StringFieldValue("ellen"), StringFieldValue("page"));
+    smap1.put(StringFieldValue("joseph"), StringFieldValue("gordon-levitt"));
+    l2s1.setValue(smapF, smap1);
+    l2s1.setValue(iarrF, iarr1);
+    l2s1.setValue(sarrF, sarr1);
+
+    l1s1.setValue(s2, l2s1);
+    MapFieldValue structmap1(structmap);
+    structmap1.put(StringFieldValue("test"), l2s1);
+    l1s1.setValue(structmapF, structmap1);
+
+    WeightedSetFieldValue wset1(wset);
+    wset1.add("foo");
+    wset1.add("bar");
+    wset1.add("zoo");
+    l1s1.setValue(wsetF, wset1);
+
+    WeightedSetFieldValue wset2(structwset);
+    wset2.add(l2s1);
+    wset2.add(l2s2);
+    l1s1.setValue(structwsetF, wset2);
+
+    doc->setValue(structl1s1, l1s1);
+
+    // Begin test proper
+
+    doc->print(std::cerr, true, "");
+
+    ModifyIteratorHandler handler;
+
+    FieldPath::UP path
+        = doc->getDataType()->buildFieldPath("l1s1.structmap.value.smap{leonardo}");
+    doc->iterateNested(path->begin(), path->end(), handler);
+
+    doc->print(std::cerr, true, "");
+}
+
+void DocumentTest::testSimpleUsage()
+{
+    DocumentType::SP type(new DocumentType("test"));
+    Field intF("int", 1, *DataType::INT, true);
+    Field longF("long", 2, *DataType::LONG, true);
+    Field strF("content", 4, *DataType::STRING, false);
+
+    type->addField(intF);
+    type->addField(longF);
+    type->addField(strF);
+
+    DocumentTypeRepo repo(*type);
+    Document value(*repo.getDocumentType("test"), DocumentId("doc::testdoc"));
+
+        // Initially empty
+    CPPUNIT_ASSERT_EQUAL(size_t(0), value.getSetFieldCount());
+    CPPUNIT_ASSERT(!value.hasValue(intF));
+
+    value.setValue(intF, IntFieldValue(1));
+
+        // Not empty
+    CPPUNIT_ASSERT_EQUAL(size_t(1), value.getSetFieldCount());
+    CPPUNIT_ASSERT(value.hasValue(intF));
+
+        // Adding some more
+    value.setValue(longF, LongFieldValue(2));
+
+        // Not empty
+    CPPUNIT_ASSERT_EQUAL(size_t(2), value.getSetFieldCount());
+    CPPUNIT_ASSERT_EQUAL(1, value.getValue(intF)->getAsInt());
+    CPPUNIT_ASSERT_EQUAL(2, value.getValue(longF)->getAsInt());
+
+        // Serialize & equality
+    std::unique_ptr<ByteBuffer> buffer(value.serialize());
+    buffer->flip();
+    Document value2(*repo.getDocumentType("test"),
+                    DocumentId("userdoc::3:foo"));
+    CPPUNIT_ASSERT(value != value2);
+    value2.deserialize(repo, *buffer);
+    CPPUNIT_ASSERT(value2.hasValue(intF));
+    CPPUNIT_ASSERT_EQUAL(value, value2);
+    CPPUNIT_ASSERT_EQUAL(DocumentId("doc::testdoc"), value2.getId());
+
+        // Various ways of removing
+    {
+            // By value
+        buffer->setPos(0);
+        value2.deserialize(repo, *buffer);
+        value2.remove(intF);
+        CPPUNIT_ASSERT(!value2.hasValue(intF));
+        CPPUNIT_ASSERT_EQUAL(size_t(1), value2.getSetFieldCount());
+
+            // Clearing all
+        buffer->setPos(0);
+        value2.deserialize(repo, *buffer);
+        value2.clear();
+        CPPUNIT_ASSERT(!value2.hasValue(intF));
+        CPPUNIT_ASSERT_EQUAL(size_t(0), value2.getSetFieldCount());
+    }
+
+        // Updating
+    value2 = value;
+    CPPUNIT_ASSERT_EQUAL(value, value2);
+    value2.setValue(strF, StringFieldValue("foo"));
+    CPPUNIT_ASSERT(value2.hasValue(strF));
+    CPPUNIT_ASSERT_EQUAL(vespalib::string("foo"),
+                         value2.getValue(strF)->getAsString());
+    CPPUNIT_ASSERT(value != value2);
+    value2.assign(value);
+    CPPUNIT_ASSERT_EQUAL(value, value2);
+    Document::UP valuePtr(value2.clone());
+    CPPUNIT_ASSERT_EQUAL(value, *valuePtr);
+
+        // Iterating
+    const Document& constVal(value);
+    for(Document::const_iterator it = constVal.begin();
+        it != constVal.end(); ++it)
+    {
+        constVal.getValue(it.field());
+    }
+
+        // Comparison
+    value2 = value;
+    CPPUNIT_ASSERT_EQUAL(0, value.compare(value2));
+    value2.remove(intF);
+    CPPUNIT_ASSERT(value.compare(value2) < 0);
+    CPPUNIT_ASSERT(value2.compare(value) > 0);
+    value2 = value;
+    value2.setValue(intF, IntFieldValue(5));
+    CPPUNIT_ASSERT(value.compare(value2) < 0);
+    CPPUNIT_ASSERT(value2.compare(value) > 0);
+
+        // Output
+    CPPUNIT_ASSERT_EQUAL(
+            std::string("Document(doc::testdoc, DocumentType(test))"),
+            value.toString(false));
+    CPPUNIT_ASSERT_EQUAL(
+            std::string(
+"  Document(doc::testdoc\n"
+"    DocumentType(test, id -877171244)\n"
+"        : DocumentType(document) {\n"
+"      StructDataType(test.header, id 306916075) {\n"
+"        Field(content, id 4, PrimitiveDataType(String, id 2), body)\n"
+"        Field(int, id 1, NumericDataType(Int, id 0), header)\n"
+"        Field(long, id 2, NumericDataType(Long, id 4), header)\n"
+"      }\n"
+"    }\n"
+"    int: 1\n"
+"    long: 2\n"
+"  )"),
+            "  " + value.toString(true, "  "));
+    CPPUNIT_ASSERT_EQUAL(
+            std::string(
+                "<document documenttype=\"test\" documentid=\"doc::testdoc\">\n"
+                "  <int>1</int>\n"
+                "  <long>2</long>\n"
+                "</document>"),
+            value.toXml("  "));
+
+        // Failure situations.
+
+        // Fetch a field not existing in type
+        // (Would be nice if this failed, but whole idea to fetch by field
+        // objects is to improve performance.)
+    Field anotherIntF("int", 17, *DataType::INT, true);
+    CPPUNIT_ASSERT(!value.hasValue(anotherIntF));
+    CPPUNIT_ASSERT(value.getValue(anotherIntF).get() == 0);
+
+        // Refuse to accept non-document types
+    try{
+        StructDataType otherType("foo", 4);
+        Document value6(otherType, DocumentId("doc::"));
+        CPPUNIT_FAIL("Didn't complain about non-document type");
+    } catch (std::exception& e) {
+        CPPUNIT_ASSERT_CONTAIN("Cannot generate a document with "
+                               "non-document type", e.what());
+    }
+
+        // Refuse to set wrong types
+    try{
+        value2.setValue(intF, StringFieldValue("bar"));
+        CPPUNIT_FAIL("Failed to check type equality in setValue");
+    } catch (std::exception& e) {
+        CPPUNIT_ASSERT_CONTAIN("Cannot assign value of type", e.what());
+    }
+}
+
+void verifyJavaDocument(Document& doc)
+{
+    IntFieldValue intVal;
+    CPPUNIT_ASSERT(doc.getValue(doc.getField("intfield"), intVal));
+    CPPUNIT_ASSERT_EQUAL(5, intVal.getAsInt());
+
+    FloatFieldValue floatVal;
+    CPPUNIT_ASSERT(doc.getValue(doc.getField("floatfield"), floatVal));
+    CPPUNIT_ASSERT(floatVal.getAsFloat() == (float) -9.23);
+
+    StringFieldValue stringVal("");
+    CPPUNIT_ASSERT(doc.getValue(doc.getField("stringfield"), stringVal));
+    CPPUNIT_ASSERT_EQUAL(vespalib::string("This is a string."),
+                         stringVal.getAsString());
+
+    LongFieldValue longVal;
+    CPPUNIT_ASSERT(doc.getValue(doc.getField("longfield"), longVal));
+    CPPUNIT_ASSERT_EQUAL((int64_t)398420092938472983LL, longVal.getAsLong());
+
+    DoubleFieldValue doubleVal;
+    CPPUNIT_ASSERT(doc.getValue(doc.getField("doublefield"), doubleVal));
+    CPPUNIT_ASSERT_EQUAL(doubleVal.getAsDouble(), 98374532.398820);
+
+    ByteFieldValue byteVal;
+    CPPUNIT_ASSERT(doc.getValue(doc.getField("bytefield"), byteVal));
+    CPPUNIT_ASSERT_EQUAL(-2, byteVal.getAsInt());
+
+    RawFieldValue rawVal;
+    CPPUNIT_ASSERT(doc.getValue(doc.getField("rawfield"), rawVal));
+    CPPUNIT_ASSERT(memcmp(rawVal.getAsRaw().first, "RAW DATA", 8) == 0);
+
+    Document embedDocVal;
+    CPPUNIT_ASSERT(doc.getValue(doc.getField("docfield"), embedDocVal));
+
+    ArrayFieldValue array(doc.getField("arrayoffloatfield").getDataType());
+    CPPUNIT_ASSERT(doc.getValue(doc.getField("arrayoffloatfield"), array));
+    CPPUNIT_ASSERT_EQUAL((float)1.0, array[0].getAsFloat());
+    CPPUNIT_ASSERT_EQUAL((float)2.0, array[1].getAsFloat());
+
+    WeightedSetFieldValue wset(doc.getField("wsfield").getDataType());
+    CPPUNIT_ASSERT(doc.getValue(doc.getField("wsfield"), wset));
+    CPPUNIT_ASSERT_EQUAL(50, wset.get(StringFieldValue("Weighted 0")));
+    CPPUNIT_ASSERT_EQUAL(199, wset.get(StringFieldValue("Weighted 1")));
+
+    MapFieldValue map(doc.getField("mapfield").getDataType());
+    CPPUNIT_ASSERT(doc.getValue(doc.getField("mapfield"), map));
+    CPPUNIT_ASSERT(map.get(StringFieldValue("foo1")).get());
+    CPPUNIT_ASSERT(map.get(StringFieldValue("foo2")).get());
+    CPPUNIT_ASSERT_EQUAL(StringFieldValue("bar1"), dynamic_cast<StringFieldValue&>(*map.get(StringFieldValue("foo1"))));
+    CPPUNIT_ASSERT_EQUAL(StringFieldValue("bar2"), dynamic_cast<StringFieldValue&>(*map.get(StringFieldValue("foo2"))));
+}
+
+void DocumentTest::testReadSerializedFile()
+{
+    // Reads a file serialized from java
+    const char file_name[] = "data/crossplatform-java-cpp-doctypes.cfg";
+    DocumentTypeRepo repo(readDocumenttypesConfig(file_name));
+
+    int fd = open("data/serializejava.dat", O_RDONLY);
+
+    size_t len = lseek(fd,0,SEEK_END);
+    ByteBuffer buf(len);
+    lseek(fd,0,SEEK_SET);
+    read(fd, buf.getBuffer(), len);
+
+    close(fd);
+
+    Document doc(repo, buf);
+    verifyJavaDocument(doc);
+
+    std::unique_ptr<ByteBuffer> buf2 = doc.serialize();
+    buf2->flip();
+
+    Document doc2(repo, *buf2);
+    verifyJavaDocument(doc2);
+
+    CPPUNIT_ASSERT_EQUAL(len, buf2->getPos());
+    CPPUNIT_ASSERT(memcmp(buf2->getBuffer(), buf.getBuffer(), buf2->getPos()) == 0);
+
+    doc2.setValue("stringfield", StringFieldValue("hei"));
+
+    std::unique_ptr<ByteBuffer> buf3 = doc2.serialize();
+    CPPUNIT_ASSERT(len != buf3->getPos());
+}
+
+void DocumentTest::testReadSerializedFileCompressed()
+{
+    // Reads a file serialized from java
+    const char file_name[] = "data/crossplatform-java-cpp-doctypes.cfg";
+    DocumentTypeRepo repo(readDocumenttypesConfig(file_name));
+
+    int fd = open("data/serializejava-compressed.dat", O_RDONLY);
+
+    int len = lseek(fd,0,SEEK_END);
+    ByteBuffer buf(len);
+    lseek(fd,0,SEEK_SET);
+    read(fd, buf.getBuffer(), len);
+
+    close(fd);
+
+    Document doc(repo, buf);
+    verifyJavaDocument(doc);
+}
+
+namespace {
+    struct TestDoc {
+        std::string _dataFile;
+        /**
+         * We may add or remove types as we create new versions. If we do so,
+         * we can use the created version to know what types we no longer
+         * should check, or what fields these old documents does not contain.
+         */
+        uint32_t _createdVersion;
+
+        TestDoc(const std::string& dataFile, uint32_t version)
+            : _dataFile(dataFile),
+              _createdVersion(version)
+        {
+        }
+    };
+}
+
+/**
+ * Tests serialization of all versions.
+ *
+ * This test tests serialization and deserialization of documents of all
+ * supported types.
+ *
+ * Serialization is only supported in newest format. Deserialization should work
+ * for all formats supported, but only the part that makes sense in the new
+ * format. Thus, if new format deprecates a datatype, that datatype, when
+ * serializing old versions, must either just be dropped or converted.
+ *
+ * Thus, we create document type programmatically, because all old versions need
+ * to make sense with current config.
+ *
+ * When we create a document programmatically. This is serialized into current
+ * version files. When altering the format, after the alteration, copy the
+ * current version files to a specific version file and add those to list of
+ * files this test checks.
+ *
+ * When adding new fields to the documents, use the version tagged with each
+ * file to ignore these field for old types.
+ */
+void DocumentTest::testReadSerializedAllVersions()
+{
+    const int array_id = 1650586661;
+    const int wset_id = 1328286588;
+        // Create the datatype used for serialization test
+    DocumenttypesConfigBuilderHelper builder;
+    builder.document(1306012852, "serializetest",
+                     Struct("serializetest.header")
+                     .addField("floatfield", DataType::T_FLOAT)
+                     .addField("stringfield", DataType::T_STRING)
+                     .addField("longfield", DataType::T_LONG)
+                     .addField("urifield", DataType::T_URI),
+                     Struct("serializetest.body")
+                     .addField("intfield", DataType::T_INT)
+                     .addField("rawfield", DataType::T_RAW)
+                     .addField("doublefield", DataType::T_DOUBLE)
+                     .addField("bytefield", DataType::T_BYTE)
+                     .addField("arrayoffloatfield",
+                               Array(DataType::T_FLOAT).setId(array_id))
+                     .addField("docfield", DataType::T_DOCUMENT)
+                     .addField("wsfield",
+                               Wset(DataType::T_STRING).setId(wset_id)));
+    builder.document(1447635645, "docindoc", Struct("docindoc.header"),
+                     Struct("docindoc.body")
+                     .addField("stringindocfield", DataType::T_STRING));
+    DocumentTypeRepo repo(builder.config());
+
+    const DocumentType* docType(repo.getDocumentType("serializetest"));
+    const DocumentType* docInDocType(repo.getDocumentType("docindoc"));
+    const DataType* arrayOfFloatDataType(repo.getDataType(*docType, array_id));
+    const DataType* weightedSetDataType(repo.getDataType(*docType, wset_id));
+
+        // Create a memory instance of document
+    {
+        Document doc(*docType,
+                     DocumentId("doc:serializetest:http://test.doc.id/"));
+        doc.set("intfield", 5);
+        doc.set("floatfield", -9.23);
+        doc.set("stringfield", "This is a string.");
+        doc.set("longfield", static_cast<int64_t>(398420092938472983LL));
+        doc.set("doublefield", 98374532.398820);
+        doc.set("bytefield", -2);
+        doc.setValue("rawfield", RawFieldValue("RAW DATA", 8));
+        Document docInDoc(*docInDocType,
+                          DocumentId("doc:serializetest:http://doc.in.doc/"));
+        docInDoc.set("stringindocfield", "Elvis is dead");
+        //docInDoc.setCompression(CompressionConfig(CompressionConfig::NONE, 0, 0));
+        doc.setValue("docfield", docInDoc);
+        ArrayFieldValue floatArray(*arrayOfFloatDataType);
+        floatArray.add(1.0);
+        floatArray.add(2.0);
+        doc.setValue("arrayoffloatfield", floatArray);
+        WeightedSetFieldValue weightedSet(*weightedSetDataType);
+        weightedSet.add(StringFieldValue("Weighted 0"), 50);
+        weightedSet.add(StringFieldValue("Weighted 1"), 199);
+        doc.setValue("wsfield", weightedSet);
+
+        // Write document to disk, (when you bump version and alter stuff,
+        // you can copy this current to new test for new version)
+        {
+            //doc.setCompression(CompressionConfig(CompressionConfig::NONE, 0, 0));
+            std::unique_ptr<ByteBuffer> buf = doc.serialize();
+            CPPUNIT_ASSERT_EQUAL(buf->getLength(), buf->getPos());
+            int fd = open("data/document-cpp-currentversion-uncompressed.dat",
+                          O_WRONLY | O_CREAT | O_TRUNC, 0644);
+            CPPUNIT_ASSERT(fd > 0);
+            size_t len = write(fd, buf->getBuffer(), buf->getPos());
+            CPPUNIT_ASSERT_EQUAL(buf->getPos(), len);
+            close(fd);
+        }
+        {
+            CompressionConfig oldCfg(doc.getType().getFieldsType().getCompressionConfig());
+            CompressionConfig newCfg(CompressionConfig::LZ4, 9, 95);
+            const_cast<StructDataType &>(doc.getType().getFieldsType()).setCompressionConfig(newCfg);
+            std::unique_ptr<ByteBuffer> buf = doc.serialize();
+            CPPUNIT_ASSERT(buf->getPos() <= buf->getLength());
+            int fd = open("data/document-cpp-currentversion-lz4-9.dat",
+                          O_WRONLY | O_CREAT | O_TRUNC, 0644);
+            CPPUNIT_ASSERT(fd > 0);
+            size_t len = write(fd, buf->getBuffer(), buf->getPos());
+            CPPUNIT_ASSERT_EQUAL(buf->getPos(), len);
+            close(fd);
+            const_cast<StructDataType &>(doc.getType().getFieldsType()).setCompressionConfig(oldCfg);
+        }
+    }
+
+    std::string jpath = "../test/serializeddocuments/";
+
+    std::vector<TestDoc> tests;
+    tests.push_back(TestDoc("data/document-cpp-v8-uncompressed.dat", 8));
+    tests.push_back(TestDoc("data/document-cpp-v7-uncompressed.dat", 7));
+    tests.push_back(TestDoc("data/serializev6.dat", 6));
+    tests.push_back(TestDoc(jpath + "document-java-v8-uncompressed.dat", 8));
+    for (uint32_t i=0; i<tests.size(); ++i) {
+        int version = tests[i]._createdVersion;
+        std::string name = tests[i]._dataFile;
+        std::cerr << name << std::endl;
+        if (!vespalib::fileExists(name)) {
+            CPPUNIT_FAIL("File " + name + " does not exist.");
+        }
+        int fd = open(tests[i]._dataFile.c_str(), O_RDONLY);
+        int len = lseek(fd,0,SEEK_END);
+        ByteBuffer buf(len);
+        lseek(fd,0,SEEK_SET);
+        read(fd, buf.getBuffer(), len);
+        close(fd);
+
+        Document doc(repo, buf);
+
+        IntFieldValue intVal;
+        CPPUNIT_ASSERT(doc.getValue(doc.getField("intfield"), intVal));
+        CPPUNIT_ASSERT_EQUAL(5, intVal.getAsInt());
+
+        FloatFieldValue floatVal;
+        CPPUNIT_ASSERT(doc.getValue(doc.getField("floatfield"), floatVal));
+        CPPUNIT_ASSERT(floatVal.getAsFloat() == (float) -9.23);
+
+        StringFieldValue stringVal("");
+        CPPUNIT_ASSERT(doc.getValue(doc.getField("stringfield"), stringVal));
+        CPPUNIT_ASSERT_EQUAL(vespalib::string("This is a string."),
+                             stringVal.getAsString());
+
+        LongFieldValue longVal;
+        CPPUNIT_ASSERT(doc.getValue(doc.getField("longfield"), longVal));
+        CPPUNIT_ASSERT_EQUAL(static_cast<int64_t>(398420092938472983LL),
+                             longVal.getAsLong());
+
+        DoubleFieldValue doubleVal;
+        CPPUNIT_ASSERT(doc.getValue(doc.getField("doublefield"), doubleVal));
+        CPPUNIT_ASSERT_EQUAL(doubleVal.getAsDouble(), 98374532.398820);
+
+        ByteFieldValue byteVal;
+        CPPUNIT_ASSERT(doc.getValue(doc.getField("bytefield"), byteVal));
+        CPPUNIT_ASSERT_EQUAL(-2, byteVal.getAsInt());
+
+        RawFieldValue rawVal;
+        CPPUNIT_ASSERT(doc.getValue(doc.getField("rawfield"), rawVal));
+        CPPUNIT_ASSERT(memcmp(rawVal.getAsRaw().first, "RAW DATA", 8) == 0);
+
+        if (version > 6) {
+            Document docInDoc;
+            CPPUNIT_ASSERT(doc.getValue(doc.getField("docfield"), docInDoc));
+
+            CPPUNIT_ASSERT(docInDoc.getValue(
+                        docInDoc.getField("stringindocfield"), stringVal));
+            CPPUNIT_ASSERT_EQUAL(vespalib::string("Elvis is dead"),
+                                 stringVal.getAsString());
+        }
+
+        ArrayFieldValue array(doc.getField("arrayoffloatfield").getDataType());
+        CPPUNIT_ASSERT(doc.getValue(doc.getField("arrayoffloatfield"), array));
+        CPPUNIT_ASSERT_EQUAL((float)1.0, array[0].getAsFloat());
+        CPPUNIT_ASSERT_EQUAL((float)2.0, array[1].getAsFloat());
+
+        WeightedSetFieldValue wset(doc.getField("wsfield").getDataType());
+        CPPUNIT_ASSERT(doc.getValue(doc.getField("wsfield"), wset));
+        CPPUNIT_ASSERT_EQUAL(50, wset.get(StringFieldValue("Weighted 0")));
+        CPPUNIT_ASSERT_EQUAL(199, wset.get(StringFieldValue("Weighted 1")));
+
+        // Check that serialization doesn't cause any problems.
+        std::unique_ptr<ByteBuffer> buf2 = doc.serialize();
+        buf2->flip();
+
+        Document doc2(repo, *buf2);
+    }
+}
+
+size_t getSerializedSize(const Document &doc) {
+    return doc.serialize()->getLength();
+}
+
+size_t getSerializedSizeHeader(const Document &doc) {
+    nbostream stream;
+    doc.serializeHeader(stream);
+    return stream.size();
+}
+
+size_t getSerializedSizeBody(const Document &doc) {
+    nbostream stream;
+    doc.serializeBody(stream);
+    return stream.size();
+}
+
+void DocumentTest::testGenerateSerializedFile()
+{
+    const char file_name[] = "data/crossplatform-java-cpp-doctypes.cfg";
+    DocumentTypeRepo repo(readDocumenttypesConfig(file_name));
+    Document doc(*repo.getDocumentType("serializetest"),
+                 DocumentId(DocIdString("serializetest",
+                                        "http://test.doc.id/")));
+
+    doc.set("intfield", 5);
+    doc.set("floatfield", -9.23);
+    doc.set("stringfield", "This is a string.");
+    doc.set("longfield", (int64_t) 398420092938472983ll);
+    doc.set("doublefield", 98374532.398820);
+    doc.set("urifield", "http://this.is.a.test/");
+    doc.set("bytefield", -2);
+    doc.set("rawfield", "RAW DATA");
+
+    const DocumentType *docindoc_type = repo.getDocumentType("docindoc");
+    CPPUNIT_ASSERT(docindoc_type);
+    Document embedDoc(*docindoc_type,
+                      DocumentId(DocIdString("docindoc", "http://embedded")));
+
+    doc.setValue("docfield", embedDoc);
+
+    WeightedSetFieldValue wset(doc.getField("wsfield").getDataType());
+    wset.add(StringFieldValue("Weighted 0"), 50);
+    wset.add(StringFieldValue("Weighted 1"), 199);
+    doc.setValue("wsfield", wset);
+
+    ArrayFieldValue array(doc.getField("arrayoffloatfield").getDataType());
+    array.add(FloatFieldValue(1.0));
+    array.add(FloatFieldValue(2.0));
+    doc.setValue("arrayoffloatfield", array);
+
+    MapFieldValue map(doc.getField("mapfield").getDataType());
+    map.put(StringFieldValue("foo1"), StringFieldValue("bar1"));
+    map.put(StringFieldValue("foo2"), StringFieldValue("bar2"));
+    doc.setValue("mapfield", map);
+
+    std::unique_ptr<ByteBuffer> buf = doc.serialize();
+
+#define SERIALIZED_DIR "../../test/document/"
+    int fd = open(SERIALIZED_DIR "/serializecpp.dat",
+                  O_WRONLY | O_TRUNC | O_CREAT, 0644);
+    write(fd, buf->getBuffer(), buf->getPos());
+    close(fd);
+
+    ByteBuffer hBuf(getSerializedSizeHeader(doc));
+    doc.serializeHeader(hBuf);
+    fd = open(SERIALIZED_DIR "/serializecppsplit_header.dat",
+              O_WRONLY | O_TRUNC | O_CREAT, 0644);
+    write(fd, hBuf.getBuffer(), hBuf.getPos());
+    close(fd);
+
+    ByteBuffer bBuf(getSerializedSizeBody(doc));
+    doc.serializeBody(bBuf);
+    fd = open(SERIALIZED_DIR "/serializecppsplit_body.dat",
+              O_WRONLY | O_TRUNC | O_CREAT, 0644);
+    write(fd, bBuf.getBuffer(), bBuf.getPos());
+    close(fd);
+
+
+    CompressionConfig newCfg(CompressionConfig::LZ4, 9, 95);
+    const_cast<StructDataType &>(doc.getType().getFieldsType()).setCompressionConfig(newCfg);
+
+    ByteBuffer lz4buf(getSerializedSize(doc));
+
+    doc.serialize(lz4buf);
+    lz4buf.flip();
+
+    fd = open(SERIALIZED_DIR "/serializecpp-lz4-level9.dat",
+              O_WRONLY | O_TRUNC | O_CREAT, 0644);
+    write(fd, lz4buf.getBufferAtPos(), lz4buf.getRemaining());
+    close(fd);
+}
+void DocumentTest::testGetURIFromSerialized()
+{
+    TestDocRepo test_repo;
+    Document doc(*test_repo.getDocumentType("testdoctype1"),
+                 DocumentId("doc:ns:testdoc"));
+
+    {
+        std::unique_ptr<ByteBuffer> serialized = doc.serialize();
+        serialized->flip();
+
+        CPPUNIT_ASSERT_EQUAL(
+                vespalib::string(DocIdString("ns", "testdoc").toString()),
+                Document::getIdFromSerialized(*serialized).toString());
+
+        CPPUNIT_ASSERT_EQUAL(vespalib::string("testdoctype1"),
+                             Document::getDocTypeFromSerialized(
+                                     test_repo.getTypeRepo(),
+                                     *serialized)->getName());
+    }
+
+    {
+        std::unique_ptr<ByteBuffer> serialized = doc.serialize();
+        serialized->flip();
+
+        Document doc2(test_repo.getTypeRepo(), *serialized, false, NULL);
+        CPPUNIT_ASSERT_EQUAL(
+                vespalib::string(DocIdString("ns", "testdoc").toString()), doc2.getId().toString());
+        CPPUNIT_ASSERT_EQUAL(vespalib::string("testdoctype1"), doc2.getType().getName());
+    }
+};
+
+void DocumentTest::testBogusserialize()
+{
+    TestDocRepo test_repo;
+    try {
+        std::unique_ptr<ByteBuffer> buf(
+                new ByteBuffer("aoifjweprjwoejr203r+2+4r823++!",100));
+        Document doc(test_repo.getTypeRepo(), *buf);
+        CPPUNIT_ASSERT(false);
+    } catch (DeserializeException& e) {
+        CPPUNIT_ASSERT_CONTAIN("Unrecognized serialization version", e.what());
+    }
+
+    try {
+        std::unique_ptr<ByteBuffer> buf(new ByteBuffer("",0));
+        Document doc(test_repo.getTypeRepo(), *buf);
+        CPPUNIT_ASSERT(false);
+    } catch (DeserializeException& e) {
+        CPPUNIT_ASSERT_CONTAIN("Buffer out of bounds", e.what());
+    }
+}
+
+void DocumentTest::testCRC32()
+{
+    TestDocRepo test_repo;
+    Document doc(*test_repo.getDocumentType("testdoctype1"),
+                 DocumentId(DocIdString("crawler", "http://www.ntnu.no/")));
+
+    doc.setValue(doc.getField("hstringval"),
+                 StringFieldValue("bla bla bla bla bla"));
+
+    uint32_t crc = doc.calculateChecksum();
+    CPPUNIT_ASSERT_EQUAL(277496115u, crc);
+
+    std::unique_ptr<ByteBuffer> buf = doc.serialize();
+    buf->flip();
+
+    int pos = 20;
+
+    // Corrupt serialization.
+    buf->getBuffer()[pos] ^= 72;
+        // Create document. Byte corrupted above is in data area and
+        // shouldn't fail deserialization.
+    try {
+        Document doc2(test_repo.getTypeRepo(), *buf);
+        buf->setPos(0);
+        CPPUNIT_ASSERT(crc != doc2.calculateChecksum());
+    } catch (document::DeserializeException& e) {
+        CPPUNIT_ASSERT(false);
+    }
+        // Return original value and retry
+    buf->getBuffer()[pos] ^= 72;
+
+    /// \todo TODO (was warning):  Cannot test for in memory representation altered, as there is no syntax for getting internal refs to data from document. Add test when this is added.
+}
+
+void
+DocumentTest::testHasChanged()
+{
+    TestDocRepo test_repo;
+    Document doc(*test_repo.getDocumentType("testdoctype1"),
+                 DocumentId(DocIdString("crawler", "http://www.ntnu.no/")));
+        // Before deserialization we are changed.
+    CPPUNIT_ASSERT(doc.hasChanged());
+
+    doc.setValue(doc.getField("hstringval"),
+                 StringFieldValue("bla bla bla bla bla"));
+        // Still changed after setting a value of course.
+    CPPUNIT_ASSERT(doc.hasChanged());
+
+    std::unique_ptr<ByteBuffer> buf = doc.serialize();
+    buf->flip();
+
+        // Setting a value in doc tags us changed.
+    {
+        buf->setPos(0);
+        Document doc2(test_repo.getTypeRepo(), *buf);
+        CPPUNIT_ASSERT(!doc2.hasChanged());
+
+        doc2.set("headerval", 13);
+        CPPUNIT_ASSERT(doc2.hasChanged());
+    }
+        // Overwriting a value in doc tags us changed.
+    {
+        buf->setPos(0);
+        Document doc2(test_repo.getTypeRepo(), *buf);
+
+        doc2.set("hstringval", "bla bla bla bla bla");
+        CPPUNIT_ASSERT(doc2.hasChanged());
+    }
+        // Clearing value tags us changed.
+    {
+        buf->setPos(0);
+        Document doc2(test_repo.getTypeRepo(), *buf);
+
+        doc2.clear();
+        CPPUNIT_ASSERT(doc2.hasChanged());
+    }
+        // Add more tests here when we allow non-const refs to internals
+}
+
+void
+DocumentTest::testSplitSerialization()
+{
+    TestDocMan testDocMan;
+    Document::UP doc = testDocMan.createDocument();
+    doc->set("headerval", 50);
+
+    ByteBuffer buf(getSerializedSizeHeader(*doc));
+    doc->serializeHeader(buf);
+    buf.flip();
+
+    ByteBuffer buf2(getSerializedSizeBody(*doc));
+    doc->serializeBody(buf2);
+    buf2.flip();
+
+    CPPUNIT_ASSERT_EQUAL(size_t(65), buf.getLength());
+    CPPUNIT_ASSERT_EQUAL(size_t(73), buf2.getLength());
+
+    Document headerDoc(testDocMan.getTypeRepo(), buf);
+    CPPUNIT_ASSERT(headerDoc.hasValue("headerval"));
+    CPPUNIT_ASSERT(!headerDoc.hasValue("content"));
+
+    buf.setPos(0);
+    Document fullDoc(testDocMan.getTypeRepo(), buf, buf2);
+    CPPUNIT_ASSERT(fullDoc.hasValue("headerval"));
+    CPPUNIT_ASSERT(fullDoc.hasValue("content"));
+
+    CPPUNIT_ASSERT_EQUAL(*doc, fullDoc);
+}
+
+void DocumentTest::testSliceSerialize()
+{
+        // Test that document doesn't need its own bytebuffer, such that we
+        // can serialize multiple documents after each other in the same
+        // bytebuffer.
+    TestDocMan testDocMan;
+    Document::UP doc = testDocMan.createDocument();
+    Document::UP doc2 = testDocMan.createDocument(
+            "Some other content", "doc:test:anotherdoc");
+
+    ArrayFieldValue val(doc2->getField("rawarray").getDataType());
+    val.add(RawFieldValue("hei", 3));
+    val.add(RawFieldValue("hallo", 5));
+    val.add(RawFieldValue("hei der", 7));
+    doc2->setValue(doc2->getField("rawarray"), val);
+
+    ByteBuffer buf(getSerializedSize(*doc) + getSerializedSize(*doc2));
+    doc->serialize(buf);
+    CPPUNIT_ASSERT_EQUAL(getSerializedSize(*doc), buf.getPos());
+    doc2->serialize(buf);
+    CPPUNIT_ASSERT_EQUAL(getSerializedSize(*doc) + getSerializedSize(*doc2),
+                         buf.getPos());
+    buf.flip();
+
+    Document doc3(testDocMan.getTypeRepo(), buf);
+    CPPUNIT_ASSERT_EQUAL(getSerializedSize(*doc), buf.getPos());
+    Document doc4(testDocMan.getTypeRepo(), buf);
+    CPPUNIT_ASSERT_EQUAL(getSerializedSize(*doc) + getSerializedSize(*doc2),
+                         buf.getPos());
+
+    CPPUNIT_ASSERT_EQUAL(*doc, doc3);
+    CPPUNIT_ASSERT_EQUAL(*doc2, doc4);
+}
+
+void DocumentTest::testCompression()
+{
+    TestDocMan testDocMan;
+    Document::UP doc = testDocMan.createDocument();
+
+    std::string bigString("compress me");
+    for (int i = 0; i < 8; ++i) { bigString += bigString; }
+
+    doc->setValue("hstringval", StringFieldValue(bigString));
+
+    std::unique_ptr<ByteBuffer> buf_uncompressed = doc->serialize();
+    buf_uncompressed->flip();
+
+    CompressionConfig oldCfg(doc->getType().getFieldsType().getCompressionConfig());
+
+    CompressionConfig newCfg(CompressionConfig::LZ4, 9, 95);
+    const_cast<StructDataType &>(doc->getType().getFieldsType()).setCompressionConfig(newCfg);
+    std::unique_ptr<ByteBuffer> buf_lz4 = doc->serialize();
+    buf_lz4->flip();
+
+    const_cast<StructDataType &>(doc->getType().getFieldsType()).setCompressionConfig(oldCfg);
+
+    CPPUNIT_ASSERT(buf_lz4->getRemaining() < buf_uncompressed->getRemaining());
+
+    Document doc_lz4(testDocMan.getTypeRepo(), *buf_lz4);
+
+    CPPUNIT_ASSERT_EQUAL(*doc, doc_lz4);
+}
+
+void DocumentTest::testCompressionConfigured()
+{
+    DocumenttypesConfigBuilderHelper builder;
+    builder.document(43, "serializetest",
+                     Struct("serializetest.header").setId(44),
+                     Struct("serializetest.body").setId(45)
+                     .addField("stringfield", DataType::T_STRING));
+    DocumentTypeRepo repo(builder.config());
+    Document doc_uncompressed(
+            *repo.getDocumentType("serializetest"),
+            DocumentId("doc:test:test"));
+
+    std::string bigString("compress me");
+    for (int i = 0; i < 8; ++i) { bigString += bigString; }
+
+    doc_uncompressed.setValue("stringfield", StringFieldValue(bigString));
+    std::unique_ptr<ByteBuffer> buf_uncompressed = doc_uncompressed.serialize();
+    buf_uncompressed->flip();
+
+    size_t uncompressedSize = buf_uncompressed->getRemaining();
+
+    DocumenttypesConfigBuilderHelper builder2;
+    builder2.document(43, "serializetest",
+                      Struct("serializetest.header").setId(44),
+                      Struct("serializetest.body").setId(45)
+                      .addField("stringfield", DataType::T_STRING)
+                      .setCompression(DocumenttypesConfig::Documenttype::
+                                      Datatype::Sstruct::Compression::LZ4,
+                                      9, 99, 0));
+    DocumentTypeRepo repo2(builder2.config());
+
+    Document doc(repo2, *buf_uncompressed);
+
+    std::unique_ptr<ByteBuffer> buf_compressed = doc.serialize();
+    buf_compressed->flip();
+    size_t compressedSize = buf_compressed->getRemaining();
+
+    CPPUNIT_ASSERT(compressedSize < uncompressedSize);
+
+    Document doc2(repo2, *buf_compressed);
+
+    std::unique_ptr<ByteBuffer> buf_compressed2 = doc2.serialize();
+    buf_compressed2->flip();
+
+    CPPUNIT_ASSERT_EQUAL(compressedSize, buf_compressed2->getRemaining());
+
+    Document doc3(repo2, *buf_compressed2);
+
+    CPPUNIT_ASSERT_EQUAL(doc2, doc_uncompressed);
+    CPPUNIT_ASSERT_EQUAL(doc2, doc3);
+}
+
+void
+DocumentTest::testUnknownEntries()
+{
+    // We should be able to deserialize a document with unknown values in it.
+    DocumentType type1("test", 0);
+    DocumentType type2("test", 0);
+    Field field1("int1", *DataType::INT, true);
+    Field field2("int2", *DataType::INT, true);
+    Field field3("int3", *DataType::INT, false);
+    Field field4("int4", *DataType::INT, false);
+
+    type1.addField(field1);
+    type1.addField(field2);
+    type1.addField(field3);
+    type1.addField(field4);
+
+    type2.addField(field3);
+    type2.addField(field4);
+
+    DocumentTypeRepo repo(type2);
+
+    Document doc1(type1, DocumentId("doc::testdoc"));
+    doc1.setValue(field1, IntFieldValue(1));
+    doc1.setValue(field2, IntFieldValue(2));
+    doc1.setValue(field3, IntFieldValue(3));
+    doc1.setValue(field4, IntFieldValue(4));
+
+    uint32_t headerLen = getSerializedSizeHeader(doc1);
+    document::ByteBuffer header(headerLen);
+    doc1.serializeHeader(header);
+    header.flip();
+
+    uint32_t bodyLen = getSerializedSizeBody(doc1);
+    document::ByteBuffer body(bodyLen);
+    doc1.serializeBody(body);
+    body.flip();
+
+    uint32_t totalLen = getSerializedSize(doc1);
+    document::ByteBuffer total(totalLen);
+    doc1.serialize(total);
+    total.flip();
+
+    Document doc2;
+    doc2.deserialize(repo, total);
+
+    Document doc3;
+    doc3.deserializeHeader(repo, header);
+    doc3.deserializeBody(repo, body);
+
+    CPPUNIT_ASSERT_EQUAL(std::string(
+        "<document documenttype=\"test\" documentid=\"doc::testdoc\">\n"
+        "<int3>3</int3>\n"
+        "<int4>4</int4>\n"
+        "</document>"), doc2.toXml());
+    CPPUNIT_ASSERT_EQUAL(std::string(
+        "<document documenttype=\"test\" documentid=\"doc::testdoc\">\n"
+        "<int3>3</int3>\n"
+        "<int4>4</int4>\n"
+        "</document>"), doc3.toXml());
+
+    CPPUNIT_ASSERT_EQUAL(3, doc2.getValue(field3)->getAsInt());
+    CPPUNIT_ASSERT_EQUAL(4, doc2.getValue(field4)->getAsInt());
+    CPPUNIT_ASSERT_EQUAL(3, doc3.getValue(field3)->getAsInt());
+    CPPUNIT_ASSERT_EQUAL(4, doc3.getValue(field4)->getAsInt());
+
+    // The fields are actually accessible as long as you ask with field of
+    // correct type.
+
+    CPPUNIT_ASSERT(doc2.hasValue(field1));
+    CPPUNIT_ASSERT(doc2.hasValue(field2));
+    CPPUNIT_ASSERT(doc3.hasValue(field1));
+    CPPUNIT_ASSERT(doc3.hasValue(field2));
+
+    CPPUNIT_ASSERT_EQUAL(1, doc2.getValue(field1)->getAsInt());
+    CPPUNIT_ASSERT_EQUAL(2, doc2.getValue(field2)->getAsInt());
+    CPPUNIT_ASSERT_EQUAL(1, doc3.getValue(field1)->getAsInt());
+    CPPUNIT_ASSERT_EQUAL(2, doc3.getValue(field2)->getAsInt());
+
+    CPPUNIT_ASSERT_EQUAL(size_t(2), doc2.getSetFieldCount());
+    CPPUNIT_ASSERT_EQUAL(size_t(2), doc3.getSetFieldCount());
+
+        // Copy paste of above test to read an old version document and
+        // deserialize it with some fields lacking to see that it doesn't
+        // report failure. (Had a bug on this earlier)
+    int fd = open("data/serializev6.dat", O_RDONLY);
+    int len = lseek(fd,0,SEEK_END);
+    ByteBuffer buf(len);
+    lseek(fd,0,SEEK_SET);
+    read(fd, buf.getBuffer(), len);
+    close(fd);
+
+    DocumenttypesConfigBuilderHelper builder;
+    builder.document(42, "docindoc", Struct("docindoc.header"),
+                     Struct("docindoc.body")
+                     .addField("stringindocfield", DataType::T_STRING));
+    builder.document(43, "serializetest",
+                     Struct("serializetest.header")
+                     .addField("floatfield", DataType::T_FLOAT),
+                     Struct("serializetest.body")
+                     .addField("rawfield", DataType::T_RAW));
+    DocumentTypeRepo repo2(builder.config());
+
+    Document doc(repo2, buf);
+    doc.toXml();
+}
+
+void DocumentTest::testAnnotationDeserialization()
+{
+    DocumenttypesConfigBuilderHelper builder;
+    builder.document(-1326249427, "dokk", Struct("dokk.header"),
+                     Struct("dokk.body")
+                     .addField("age", DataType::T_BYTE)
+                     .addField("story", DataType::T_STRING)
+                     .addField("date", DataType::T_INT)
+                     .addField("friend", DataType::T_LONG))
+        .annotationType(609952424, "person", Struct("person")
+                        .addField("firstname", DataType::T_STRING)
+                        .addField("lastname", DataType::T_STRING)
+                        .addField("birthyear", DataType::T_INT)
+                        .setId(443162583))
+        .annotationType(-1695443536, "dummy", 0)
+        .annotationType(-427420193, "number", DataType::T_INT)
+        .annotationType(1616020615, "relative", Struct("relative")
+                        .addField("title", DataType::T_STRING)
+                        .addField("related", AnnotationRef(609952424))
+                        .setId(-236946034))
+        .annotationType(-269517759, "banana", 0)
+        .annotationType(-513687143, "grape", 0)
+        .annotationType(1730712959, "apple", 0);
+    DocumentTypeRepo repo(builder.config());
+
+    int fd = open("data/serializejavawithannotations.dat", O_RDONLY);
+    int len = lseek(fd,0,SEEK_END);
+    ByteBuffer buf(len);
+    lseek(fd,0,SEEK_SET);
+    read(fd, buf.getBuffer(), len);
+    close(fd);
+
+    Document doc(repo, buf);
+    StringFieldValue strVal;
+    CPPUNIT_ASSERT(doc.getValue(doc.getField("story"), strVal));
+
+    vespalib::nbostream stream;
+    VespaDocumentSerializer serializer(stream);
+    serializer.write(strVal);
+    
+    FixedTypeRepo fixedRepo(repo, doc.getType());
+    VespaDocumentDeserializer deserializer(fixedRepo, stream, 8);
+    StringFieldValue strVal2;
+    deserializer.read(strVal2);
+    CPPUNIT_ASSERT_EQUAL(strVal.toString(), strVal2.toString());
+    CPPUNIT_ASSERT_EQUAL(strVal.toString(true), strVal2.toString(true));
+
+    CPPUNIT_ASSERT_EQUAL(vespalib::string("help me help me i'm stuck inside a computer!"),
+                         strVal.getAsString());
+    StringFieldValue::SpanTrees trees = strVal.getSpanTrees();
+    const SpanTree *span_tree = StringFieldValue::findTree(trees, "fruits");
+    CPPUNIT_ASSERT(span_tree);
+    CPPUNIT_ASSERT_EQUAL(size_t(8), span_tree->numAnnotations());
+    span_tree = StringFieldValue::findTree(trees, "ballooo");
+    CPPUNIT_ASSERT(span_tree);
+    CPPUNIT_ASSERT_EQUAL(size_t(8), span_tree->numAnnotations());
+
+
+    ByteFieldValue byteVal;
+    CPPUNIT_ASSERT(doc.getValue(doc.getField("age"), byteVal));
+    CPPUNIT_ASSERT_EQUAL( 123, byteVal.getAsInt());
+
+    IntFieldValue intVal;
+    CPPUNIT_ASSERT(doc.getValue(doc.getField("date"), intVal));
+    CPPUNIT_ASSERT_EQUAL(13829297, intVal.getAsInt());
+
+    LongFieldValue longVal;
+    CPPUNIT_ASSERT(doc.getValue(doc.getField("friend"), longVal));
+    CPPUNIT_ASSERT_EQUAL((int64_t)2384LL, longVal.getAsLong());
+}
+
+void
+DocumentTest::testGetSerializedSize()
+{
+    TestDocMan testDocMan;
+    Document::UP doc = testDocMan.createDocument();
+
+    CPPUNIT_ASSERT_EQUAL(getSerializedSize(*doc), doc->getSerializedSize());
+}
+
+void
+DocumentTest::testDeserializeMultiple()
+{
+    TestDocRepo testDocRepo;
+    const DocumentTypeRepo& repo(testDocRepo.getTypeRepo());
+    const DocumentType& docType(*repo.getDocumentType("testdoctype1"));
+    CPPUNIT_ASSERT(&docType != 0);
+
+    StructFieldValue sv1(docType.getField("mystruct").getDataType());
+
+    StructFieldValue sv2(docType.getField("mystruct").getDataType());
+
+    sv1.setValue(sv1.getField("key"), IntFieldValue(1234));
+    sv2.setValue(sv2.getField("value"), StringFieldValue("badger"));
+
+    StructFieldValue sv3(docType.getField("mystruct").getDataType());
+
+    vespalib::nbostream stream;
+    VespaDocumentSerializer serializer(stream);
+    serializer.write(sv1);
+    serializer.write(sv2);
+
+    VespaDocumentDeserializer deserializer(repo, stream, 8);
+    deserializer.readStructNoReset(sv3);
+    deserializer.readStructNoReset(sv3);
+
+    StructFieldValue correct(docType.getField("mystruct").getDataType());
+
+    correct.setValue(correct.getField("key"), IntFieldValue(1234));
+    correct.setValue(correct.getField("value"), StringFieldValue("badger"));
+    CPPUNIT_ASSERT_EQUAL(correct, sv3);
+}
+
+} // document
diff --git a/document/src/tests/documenttypetestcase.cpp b/document/src/tests/documenttypetestcase.cpp
new file mode 100644
index 00000000000..e946844e867
--- /dev/null
+++ b/document/src/tests/documenttypetestcase.cpp
@@ -0,0 +1,254 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+
+#include <vespa/document/config/config-documenttypes.h>
+#include <vespa/document/datatype/documenttype.h>
+#include <vespa/document/fieldvalue/fieldvalues.h>
+#include <vespa/document/repo/configbuilder.h>
+#include <vespa/document/repo/documenttyperepo.h>
+#include <vespa/vdstestlib/cppunit/macros.h>
+#include <sstream>
+#include <string>
+
+using document::config_builder::Struct;
+using document::config_builder::Wset;
+using document::config_builder::Array;
+using document::config_builder::Map;
+
+
+namespace document {
+
+struct DocumentTypeTest : public CppUnit::TestFixture {
+    void setUp();
+    void tearDown();
+
+    void testSetGet();
+    void testHeaderContent();
+    void testFieldSet();
+    void testInheritance();
+    void testInheritanceConfig();
+    void testMultipleInheritance();
+    void testOutputOperator();
+
+    CPPUNIT_TEST_SUITE( DocumentTypeTest);
+    CPPUNIT_TEST(testSetGet);
+    CPPUNIT_TEST(testFieldSet);
+    CPPUNIT_TEST(testInheritance);
+    CPPUNIT_TEST(testInheritanceConfig);
+    CPPUNIT_TEST(testMultipleInheritance);
+    CPPUNIT_TEST(testOutputOperator);
+    CPPUNIT_TEST(testHeaderContent);
+    CPPUNIT_TEST_SUITE_END();
+
+};
+
+CPPUNIT_TEST_SUITE_REGISTRATION(DocumentTypeTest);
+
+void DocumentTypeTest::setUp()
+{
+}
+
+void DocumentTypeTest::tearDown()
+{
+}
+
+void DocumentTypeTest::testSetGet() {
+  DocumentType docType("doctypetestdoc", 0);
+
+  docType.addField(Field("stringattr", 3, *DataType::STRING, true));
+  docType.addField(Field("nalle", 0, *DataType::INT, false));
+
+  const Field& fetch1=docType.getField("stringattr");
+  const Field& fetch2=docType.getField("stringattr");
+
+  CPPUNIT_ASSERT(fetch1==fetch2);
+  CPPUNIT_ASSERT(fetch1.getName() == "stringattr");
+
+  const Field& fetch3 = docType.getField(3, Document::getNewestSerializationVersion());
+
+  CPPUNIT_ASSERT(fetch1==fetch3);
+
+  const Field& fetch4=docType.getField(0, Document::getNewestSerializationVersion());
+
+  CPPUNIT_ASSERT(fetch4!=fetch1);
+}
+
+void
+categorizeFields(const Field::Set& fields,
+                 std::vector<const Field*>& headers,
+                 std::vector<const Field*>& bodies)
+{
+    for (Field::Set::const_iterator it(fields.begin()), e(fields.end());
+         it != e; ++it)
+    {
+        if ((*it)->isHeaderField()) {
+            headers.push_back(*it);
+        } else {
+            bodies.push_back(*it);
+        }
+    }
+}
+
+void
+DocumentTypeTest::testInheritanceConfig()
+{
+    DocumentTypeRepo
+        repo(readDocumenttypesConfig("data/inheritancetest.cfg"));
+
+    {
+        const DocumentType* type(repo.getDocumentType("music"));
+        CPPUNIT_ASSERT(type != NULL);
+    }
+
+    {
+        const DocumentType* type(repo.getDocumentType("books"));
+        CPPUNIT_ASSERT(type != NULL);
+    }
+}
+
+void
+DocumentTypeTest::testHeaderContent()
+{
+    DocumentTypeRepo
+        repo(readDocumenttypesConfig("data/doctypesconfigtest.cfg"));
+
+    const DocumentType* type(repo.getDocumentType("derived"));
+
+    Field::Set fields = type->getFieldsType().getFieldSet();
+
+    std::vector<const Field*> headers;
+    std::vector<const Field*> bodies;
+    categorizeFields(fields, headers, bodies);
+
+    CPPUNIT_ASSERT(headers.size() == 4);
+    CPPUNIT_ASSERT(headers[0]->getName() == "field1");
+    CPPUNIT_ASSERT(headers[1]->getName() == "field2");
+    CPPUNIT_ASSERT(headers[2]->getName() == "field4");
+    CPPUNIT_ASSERT(headers[3]->getName() == "fieldarray1");
+
+    CPPUNIT_ASSERT(bodies.size() == 2);
+    CPPUNIT_ASSERT(bodies[0]->getName() == "field3");
+    CPPUNIT_ASSERT(bodies[1]->getName() == "field5");
+}
+
+void DocumentTypeTest::testMultipleInheritance()
+{
+    config_builder::DocumenttypesConfigBuilderHelper builder;
+    builder.document(42, "test1", Struct("test1.header"),
+                     Struct("test1.body")
+                     .addField("stringattr", DataType::T_STRING)
+                     .addField("nalle", DataType::T_INT));
+    builder.document(43, "test2", Struct("test2.header"),
+                     Struct("test2.body")
+                     .addField("stringattr", DataType::T_STRING)
+                     .addField("tmp", DataType::T_STRING)
+                     .addField("tall", DataType::T_INT));
+    builder.document(44, "test3",
+                     Struct("test3.header"), Struct("test3.body"))
+        .inherit(42).inherit(43);
+    DocumentTypeRepo repo(builder.config());
+    const DocumentType* docType3(repo.getDocumentType("test3"));
+
+    CPPUNIT_ASSERT(docType3->hasField("stringattr"));
+    CPPUNIT_ASSERT(docType3->hasField("nalle"));
+    CPPUNIT_ASSERT(docType3->hasField("tmp"));
+    CPPUNIT_ASSERT(docType3->hasField("tall"));
+
+    Document doc(*docType3, DocumentId(DocIdString("test", "test")));
+
+    IntFieldValue intVal(3);
+    doc.setValue(doc.getField("nalle"), intVal);
+
+    StringFieldValue stringVal("tmp");
+    doc.setValue(doc.getField("tmp"), stringVal);
+
+    CPPUNIT_ASSERT(doc.getValue(doc.getField("nalle"))->getAsInt()==3);
+    CPPUNIT_ASSERT_EQUAL(vespalib::string("tmp"),
+                         doc.getValue(doc.getField("tmp"))->getAsString());
+}
+
+void DocumentTypeTest::testFieldSet()
+{
+    DocumentType docType("test1");
+    docType.addField(Field("stringattr", 3, *DataType::STRING, false));
+    docType.addField(Field("nalle", 0, *DataType::INT, false));
+    DocumentType::FieldSet::Fields tmp;
+    tmp.insert("nalle");
+    tmp.insert("nulle");
+    try {
+        docType.addFieldSet("a", tmp);
+        CPPUNIT_ASSERT(false);
+    } catch (const vespalib::IllegalArgumentException & e) {
+        CPPUNIT_ASSERT_EQUAL(vespalib::string("Fieldset 'a': No field with name 'nulle' in document type 'test1'."), e.getMessage());
+    }
+}
+
+void DocumentTypeTest::testInheritance()
+{
+        // Inheritance of conflicting but equal datatype ok
+    DocumentType docType("test1");
+    docType.addField(Field("stringattr", 3, *DataType::STRING, false));
+    docType.addField(Field("nalle", 0, *DataType::INT, false));
+
+    DocumentType docType2("test2");
+    docType2.addField(Field("stringattr", 3, *DataType::STRING, false));
+    docType2.addField(Field("tmp", 5, *DataType::STRING, false));
+    docType2.addField(Field("tall", 10, *DataType::INT, false));
+
+    docType.inherit(docType2);
+    CPPUNIT_ASSERT(docType.hasField("stringattr"));
+    CPPUNIT_ASSERT(docType.hasField("nalle"));
+    CPPUNIT_ASSERT(docType.hasField("tmp"));
+    CPPUNIT_ASSERT(docType.hasField("tall"));
+
+    DocumentType docType3("test3");
+    docType3.addField(Field("stringattr", 3, *DataType::RAW, false));
+    docType3.addField(Field("tall", 10, *DataType::INT, false));
+
+    try{
+        docType2.inherit(docType3);
+        //CPPUNIT_FAIL("Supposed to fail");
+    } catch (std::exception& e) {
+        CPPUNIT_ASSERT_EQUAL(std::string("foo"), std::string(e.what()));
+    }
+
+    try{
+        docType.inherit(docType3);
+        //CPPUNIT_FAIL("Supposed to fail");
+    } catch (std::exception& e) {
+        CPPUNIT_ASSERT_EQUAL(std::string("foo"), std::string(e.what()));
+    }
+
+    DocumentType docType4("test4");
+    docType4.inherit(docType3);
+
+    CPPUNIT_ASSERT(docType4.hasField("stringattr"));
+    CPPUNIT_ASSERT(docType4.hasField("tall"));
+
+    try{
+        docType3.inherit(docType4);
+        CPPUNIT_FAIL("Accepted cyclic inheritance");
+    } catch (std::exception& e) {
+        CPPUNIT_ASSERT_CONTAIN("Cannot add cyclic dependencies", e.what());
+    }
+
+    DocumentType docType5("test5");
+    docType5.addField(Field("stringattr", 20, *DataType::RAW, false));
+
+    try{
+        docType4.inherit(docType5);
+        //CPPUNIT_FAIL("Supposed to fail");
+    } catch (std::exception& e) {
+        CPPUNIT_ASSERT_EQUAL(std::string("foo"), std::string(e.what()));
+    }
+}
+
+void DocumentTypeTest::testOutputOperator() {
+    DocumentType docType("test1");
+    std::ostringstream ost;
+    ost << docType;
+    std::string expected("DocumentType(test1)");
+    CPPUNIT_ASSERT_EQUAL(expected, ost.str());
+}
+
+} // document
diff --git a/document/src/tests/documentupdatetestcase.cpp b/document/src/tests/documentupdatetestcase.cpp
new file mode 100644
index 00000000000..be619b0f35e
--- /dev/null
+++ b/document/src/tests/documentupdatetestcase.cpp
@@ -0,0 +1,1030 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/document/fieldvalue/fieldvalues.h>
+#include <vespa/document/update/documentupdate.h>
+#include <vespa/document/base/testdocman.h>
+
+#include <vespa/document/update/addvalueupdate.h>
+#include <vespa/document/update/arithmeticvalueupdate.h>
+#include <vespa/document/update/assignvalueupdate.h>
+#include <vespa/document/update/clearvalueupdate.h>
+#include <vespa/document/update/documentupdateflags.h>
+#include <vespa/document/update/fieldupdate.h>
+#include <vespa/document/update/mapvalueupdate.h>
+#include <vespa/document/update/removevalueupdate.h>
+#include <vespa/document/update/valueupdate.h>
+#include <vespa/document/serialization/vespadocumentserializer.h>
+
+#include <vespa/document/repo/configbuilder.h>
+#include <vespa/document/repo/documenttyperepo.h>
+#include <vespa/vdstestlib/cppunit/macros.h>
+
+using namespace document::config_builder;
+
+namespace document {
+
+struct DocumentUpdateTest : public CppUnit::TestFixture {
+  void setUp() {}
+  void tearDown() {}
+
+  void testSimpleUsage();
+  void testUpdateApplySingleValue();
+  void testClearField();
+  void testUpdateArray();
+  void testUpdateWeightedSet();
+  void testIncrementNonExistingAutoCreateWSetField();
+  void testIncrementExistingWSetField();
+  void testIncrementWithZeroResultWeightIsRemoved();
+  void testReadSerializedFile();
+  void testGenerateSerializedFile();
+  void testSetBadFieldTypes();
+  void testUpdateApplyNoParams();
+  void testUpdateApplyNoArrayValues();
+  void testUpdateArrayEmptyParamValue();
+  void testUpdateWeightedSetEmptyParamValue();
+  void testUpdateArrayWrongSubtype();
+  void testUpdateWeightedSetWrongSubtype();
+  void testMapValueUpdate();
+  void testThatDocumentUpdateFlagsIsWorking();
+  void testThatCreateIfNonExistentFlagIsSerialized50AndDeserialized50();
+  void testThatCreateIfNonExistentFlagIsSerializedAndDeserialized();
+
+  CPPUNIT_TEST_SUITE(DocumentUpdateTest);
+  CPPUNIT_TEST(testSimpleUsage);
+  CPPUNIT_TEST(testUpdateApplySingleValue);
+  CPPUNIT_TEST(testClearField);
+  CPPUNIT_TEST(testUpdateArray);
+  CPPUNIT_TEST(testUpdateWeightedSet);
+  CPPUNIT_TEST(testIncrementNonExistingAutoCreateWSetField);
+  CPPUNIT_TEST(testIncrementExistingWSetField);
+  CPPUNIT_TEST(testIncrementWithZeroResultWeightIsRemoved);
+  CPPUNIT_TEST(testReadSerializedFile);
+  CPPUNIT_TEST(testGenerateSerializedFile);
+  CPPUNIT_TEST(testSetBadFieldTypes);
+  CPPUNIT_TEST(testUpdateApplyNoParams);
+  CPPUNIT_TEST(testUpdateApplyNoArrayValues);
+  CPPUNIT_TEST(testUpdateArrayEmptyParamValue);
+  CPPUNIT_TEST(testUpdateWeightedSetEmptyParamValue);
+  CPPUNIT_TEST(testUpdateArrayWrongSubtype);
+  CPPUNIT_TEST(testUpdateWeightedSetWrongSubtype);
+  CPPUNIT_TEST(testMapValueUpdate);
+  CPPUNIT_TEST(testThatDocumentUpdateFlagsIsWorking);
+  CPPUNIT_TEST(testThatCreateIfNonExistentFlagIsSerialized50AndDeserialized50);
+  CPPUNIT_TEST(testThatCreateIfNonExistentFlagIsSerializedAndDeserialized);
+  CPPUNIT_TEST_SUITE_END();
+
+};
+
+CPPUNIT_TEST_SUITE_REGISTRATION(DocumentUpdateTest);
+
+namespace {
+
+ByteBuffer::UP serializeHEAD(const DocumentUpdate & update)
+{
+    vespalib::nbostream stream;
+    VespaDocumentSerializer serializer(stream);
+    serializer.writeHEAD(update);
+    ByteBuffer::UP retVal(new ByteBuffer(stream.size()));
+    retVal->putBytes(stream.peek(), stream.size());
+    return retVal;
+}
+
+ByteBuffer::UP serialize42(const DocumentUpdate & update)
+{
+    vespalib::nbostream stream;
+    VespaDocumentSerializer serializer(stream);
+    serializer.write42(update);
+    ByteBuffer::UP retVal(new ByteBuffer(stream.size()));
+    retVal->putBytes(stream.peek(), stream.size());
+    return retVal;
+}
+
+ByteBuffer::UP serialize(const ValueUpdate & update)
+{
+    vespalib::nbostream stream;
+    VespaDocumentSerializer serializer(stream);
+    serializer.write(update);
+    ByteBuffer::UP retVal(new ByteBuffer(stream.size()));
+    retVal->putBytes(stream.peek(), stream.size());
+    return retVal;
+}
+
+ByteBuffer::UP serialize(const FieldUpdate & update)
+{
+    vespalib::nbostream stream;
+    VespaDocumentSerializer serializer(stream);
+    serializer.write(update);
+    ByteBuffer::UP retVal(new ByteBuffer(stream.size()));
+    retVal->putBytes(stream.peek(), stream.size());
+    return retVal;
+}
+
+template<typename UpdateType>
+void testValueUpdate(const UpdateType& update, const DataType &type) {
+    try{
+        DocumentTypeRepo repo;
+        ByteBuffer::UP buf = serialize(update);
+        buf->flip();
+        typename UpdateType::UP copy(dynamic_cast<UpdateType*>(
+                        ValueUpdate::createInstance(repo, type, *buf,
+                                Document::getNewestSerializationVersion())
+                        .release()));
+        CPPUNIT_ASSERT_EQUAL(update, *copy);
+    } catch (std::exception& e) {
+            std::cerr << "Failed while processing update " << update << "\n";
+    throw;
+    }
+}
+
+
+}  // namespace
+
+void
+DocumentUpdateTest::testSimpleUsage() {
+    DocumenttypesConfigBuilderHelper builder;
+    builder.document(42, "test",
+                     Struct("test.header")
+                     .addField("bytef", DataType::T_BYTE)
+                     .addField("intf", DataType::T_INT),
+                     Struct("test.body")
+                     .addField("intarr", Array(DataType::T_INT)));
+    DocumentTypeRepo repo(builder.config());
+    const DocumentType* docType(repo.getDocumentType("test"));
+    const DataType *arrayType = repo.getDataType(*docType, "Array<Int>");
+
+        // Test that primitive value updates can be serialized
+    testValueUpdate(ClearValueUpdate(), *DataType::INT);
+    testValueUpdate(AssignValueUpdate(IntFieldValue(1)), *DataType::INT);
+    testValueUpdate(ArithmeticValueUpdate(ArithmeticValueUpdate::Div, 4.3),
+                    *DataType::FLOAT);
+    testValueUpdate(AddValueUpdate(IntFieldValue(1), 4), *arrayType);
+    testValueUpdate(RemoveValueUpdate(IntFieldValue(1)), *arrayType);
+
+    FieldUpdate fieldUpdate(docType->getField("intf"));
+    fieldUpdate.addUpdate(AssignValueUpdate(IntFieldValue(1)));
+    ByteBuffer::UP fieldBuf = serialize(fieldUpdate);
+    fieldBuf->flip();
+    FieldUpdate fieldUpdateCopy(repo, *docType, *fieldBuf,
+                                Document::getNewestSerializationVersion());
+    CPPUNIT_ASSERT_EQUAL(fieldUpdate, fieldUpdateCopy);
+
+        // Test that a document update can be serialized
+    DocumentUpdate docUpdate(*docType, DocumentId("doc::testdoc"));
+    docUpdate.addUpdate(fieldUpdateCopy);
+    ByteBuffer::UP docBuf = serialize42(docUpdate);
+    docBuf->flip();
+    DocumentUpdate::UP docUpdateCopy(DocumentUpdate::create42(repo, *docBuf));
+    CPPUNIT_ASSERT(!docUpdate.affectsDocumentBody());
+    CPPUNIT_ASSERT(!docUpdateCopy->affectsDocumentBody());
+
+        // Create a test document
+    Document doc(*docType, DocumentId("doc::testdoc"));
+    doc.set("bytef", 0);
+    doc.set("intf", 5);
+    ArrayFieldValue array(*arrayType);
+    array.add(IntFieldValue(3));
+    array.add(IntFieldValue(7));
+    doc.setValue("intarr", array);
+
+        // Verify that we can apply simple updates to it
+    {
+        Document updated(doc);
+        DocumentUpdate upd(*docType, DocumentId("doc::testdoc"));
+        upd.addUpdate(FieldUpdate(docType->getField("intf"))
+                      .addUpdate(ClearValueUpdate()));
+        upd.applyTo(updated);
+        CPPUNIT_ASSERT(doc != updated);
+        CPPUNIT_ASSERT(! updated.getValue("intf"));
+    }
+    {
+        Document updated(doc);
+        DocumentUpdate upd(*docType, DocumentId("doc::testdoc"));
+        upd.addUpdate(FieldUpdate(docType->getField("intf"))
+                      .addUpdate(AssignValueUpdate(IntFieldValue(15))));
+        upd.applyTo(updated);
+        CPPUNIT_ASSERT(doc != updated);
+        CPPUNIT_ASSERT_EQUAL(15, updated.getValue("intf")->getAsInt());
+    }
+    {
+        Document updated(doc);
+        DocumentUpdate upd(*docType, DocumentId("doc::testdoc"));
+        upd.addUpdate(FieldUpdate(docType->getField("intf"))
+                      .addUpdate(ArithmeticValueUpdate(
+                                      ArithmeticValueUpdate::Add, 15)));
+        upd.applyTo(updated);
+        CPPUNIT_ASSERT(doc != updated);
+        CPPUNIT_ASSERT_EQUAL(20, updated.getValue("intf")->getAsInt());
+    }
+    {
+        Document updated(doc);
+        DocumentUpdate upd(*docType, DocumentId("doc::testdoc"));
+        upd.addUpdate(FieldUpdate(docType->getField("intarr"))
+                      .addUpdate(AddValueUpdate(IntFieldValue(4))));
+        upd.applyTo(updated);
+        CPPUNIT_ASSERT(doc != updated);
+        std::unique_ptr<ArrayFieldValue> val(dynamic_cast<ArrayFieldValue*>(
+                    updated.getValue("intarr").release()));
+        CPPUNIT_ASSERT_EQUAL(size_t(3), val->size());
+        CPPUNIT_ASSERT_EQUAL(4, (*val)[2].getAsInt());
+        CPPUNIT_ASSERT(upd.affectsDocumentBody());
+    }
+    {
+        Document updated(doc);
+        DocumentUpdate upd(*docType, DocumentId("doc::testdoc"));
+        upd.addUpdate(FieldUpdate(docType->getField("intarr"))
+                      .addUpdate(RemoveValueUpdate(IntFieldValue(3))));
+        upd.applyTo(updated);
+        CPPUNIT_ASSERT(doc != updated);
+        std::unique_ptr<ArrayFieldValue> val(dynamic_cast<ArrayFieldValue*>(
+                    updated.getValue("intarr").release()));
+        CPPUNIT_ASSERT_EQUAL(size_t(1), val->size());
+        CPPUNIT_ASSERT_EQUAL(7, (*val)[0].getAsInt());
+    }
+    {
+        Document updated(doc);
+        DocumentUpdate upd(*docType, DocumentId("doc::testdoc"));
+        upd.addUpdate(FieldUpdate(docType->getField("bytef"))
+                      .addUpdate(ArithmeticValueUpdate(
+                                      ArithmeticValueUpdate::Add, 15)));
+        upd.applyTo(updated);
+        CPPUNIT_ASSERT(doc != updated);
+        CPPUNIT_ASSERT_EQUAL(15, (int) updated.getValue("bytef")->getAsByte());
+    }
+}
+
+void
+DocumentUpdateTest::testClearField()
+{
+    // Create a document.
+    TestDocMan docMan;
+    Document::UP doc(docMan.createDocument());
+    doc->setValue(doc->getField("headerval"), IntFieldValue(4));
+    CPPUNIT_ASSERT_EQUAL(4, doc->getValue("headerval")->getAsInt());
+
+    // Apply an update.
+    DocumentUpdate(*doc->getDataType(), doc->getId())
+        .addUpdate(FieldUpdate(doc->getField("headerval"))
+                   .addUpdate(AssignValueUpdate()))
+        .applyTo(*doc);
+    CPPUNIT_ASSERT(!doc->getValue("headerval"));
+}
+
+void
+DocumentUpdateTest::testUpdateApplySingleValue()
+{
+    // Create a document.
+    TestDocMan docMan;
+    Document::UP doc(docMan.createDocument());
+    doc->setValue(doc->getField("headerval"), IntFieldValue(4));
+    CPPUNIT_ASSERT_EQUAL(4, doc->getValue("headerval")->getAsInt());
+
+    // Apply an update.
+    DocumentUpdate(*doc->getDataType(), doc->getId())
+        .addUpdate(FieldUpdate(doc->getField("headerval"))
+                   .addUpdate(AssignValueUpdate(IntFieldValue(9))))
+        .applyTo(*doc);
+    CPPUNIT_ASSERT_EQUAL(9, doc->getValue("headerval")->getAsInt());
+}
+
+void
+DocumentUpdateTest::testUpdateArray()
+{
+    // Create a document.
+    TestDocMan docMan;
+    Document::UP doc(docMan.createDocument());
+    CPPUNIT_ASSERT_EQUAL((document::FieldValue*)NULL,
+			 doc->getValue(doc->getField("tags")).get());
+
+    // Assign array field.
+    ArrayFieldValue myarray(doc->getType().getField("tags").getDataType());
+    myarray.add(StringFieldValue("foo"));
+	myarray.add(StringFieldValue("bar"));
+
+    DocumentUpdate(*doc->getDataType(), doc->getId())
+        .addUpdate(FieldUpdate(doc->getField("tags"))
+                   .addUpdate(AssignValueUpdate(myarray)))
+        .applyTo(*doc);
+    std::unique_ptr<ArrayFieldValue>
+        fval1(doc->getAs<ArrayFieldValue>(doc->getField("tags")));
+    CPPUNIT_ASSERT_EQUAL((size_t) 2, fval1->size());
+    CPPUNIT_ASSERT_EQUAL(std::string("foo"),
+			 std::string((*fval1)[0].getAsString()));
+    CPPUNIT_ASSERT_EQUAL(std::string("bar"),
+			 std::string((*fval1)[1].getAsString()));
+
+    // Append array field
+    DocumentUpdate(*doc->getDataType(), doc->getId())
+        .addUpdate(FieldUpdate(doc->getField("tags"))
+                   .addUpdate(AddValueUpdate(StringFieldValue("another")))
+                   .addUpdate(AddValueUpdate(StringFieldValue("tag"))))
+        .applyTo(*doc);
+    std::unique_ptr<ArrayFieldValue>
+        fval2(doc->getAs<ArrayFieldValue>(doc->getField("tags")));
+    CPPUNIT_ASSERT_EQUAL((size_t) 4, fval2->size());
+    CPPUNIT_ASSERT_EQUAL(std::string("foo"),
+			 std::string((*fval2)[0].getAsString()));
+    CPPUNIT_ASSERT_EQUAL(std::string("bar"),
+			 std::string((*fval2)[1].getAsString()));
+    CPPUNIT_ASSERT_EQUAL(std::string("another"),
+			 std::string((*fval2)[2].getAsString()));
+    CPPUNIT_ASSERT_EQUAL(std::string("tag"),
+			 std::string((*fval2)[3].getAsString()));
+
+    // Append single value.
+    try {
+        DocumentUpdate(*doc->getDataType(), doc->getId())
+            .addUpdate(FieldUpdate(doc->getField("tags"))
+                       .addUpdate(AssignValueUpdate(
+                                       StringFieldValue("THROW MEH!"))))
+            .applyTo(*doc);
+        CPPUNIT_FAIL("Expected exception when assinging a string value to an "
+                     "array field.");
+        }
+    catch (std::exception& e) {
+        ; // fprintf(stderr, "Got exception => OK: %s\n", e.what());
+    }
+
+    // Remove array field.
+    DocumentUpdate(*doc->getDataType(), doc->getId())
+        .addUpdate(FieldUpdate(doc->getField("tags"))
+                   .addUpdate(RemoveValueUpdate(StringFieldValue("foo")))
+                   .addUpdate(RemoveValueUpdate(StringFieldValue("tag"))))
+        .applyTo(*doc);
+    std::unique_ptr<ArrayFieldValue>
+        fval3(doc->getAs<ArrayFieldValue>(doc->getField("tags")));
+    CPPUNIT_ASSERT_EQUAL((size_t) 2, fval3->size());
+    CPPUNIT_ASSERT_EQUAL(std::string("bar"),
+			 std::string((*fval3)[0].getAsString()));
+    CPPUNIT_ASSERT_EQUAL(std::string("another"),
+			 std::string((*fval3)[1].getAsString()));
+
+    // Remove array from array.
+    ArrayFieldValue myarray2(doc->getType().getField("tags").getDataType());
+    myarray2.add(StringFieldValue("foo"));
+    myarray2.add(StringFieldValue("bar"));
+    try {
+        DocumentUpdate(*doc->getDataType(), doc->getId())
+            .addUpdate(FieldUpdate(doc->getField("tags"))
+                       .addUpdate(RemoveValueUpdate(myarray2)))
+            .applyTo(*doc);
+        CPPUNIT_FAIL("Expected exception when removing an array from a "
+                     "string array.");
+    }
+    catch (std::exception& e) {
+        ; // fprintf(stderr, "Got exception => OK: %s\n", e.what());
+    }
+}
+
+void
+DocumentUpdateTest::testUpdateWeightedSet()
+{
+    // Create a test document
+    TestDocMan docMan;
+    Document::UP doc(docMan.createDocument());
+    const Field& field(doc->getType().getField("stringweightedset"));
+    CPPUNIT_ASSERT_EQUAL((FieldValue*) 0, doc->getValue(field).get());
+	
+    // Assign weightedset field
+    WeightedSetFieldValue wset(field.getDataType());
+    wset.add(StringFieldValue("foo"), 3);
+    wset.add(StringFieldValue("bar"), 14);
+    DocumentUpdate(*doc->getDataType(), doc->getId())
+        .addUpdate(FieldUpdate(field)
+                   .addUpdate(AssignValueUpdate(wset)))
+        .applyTo(*doc);
+    std::unique_ptr<WeightedSetFieldValue>
+        fval1(doc->getAs<WeightedSetFieldValue>(field));
+    CPPUNIT_ASSERT_EQUAL((size_t) 2, fval1->size());
+    CPPUNIT_ASSERT(fval1->contains(StringFieldValue("foo")));
+    CPPUNIT_ASSERT(fval1->find(StringFieldValue("foo")) != fval1->end());
+    CPPUNIT_ASSERT_EQUAL(3, fval1->get(StringFieldValue("foo"), 0));
+    CPPUNIT_ASSERT(fval1->contains(StringFieldValue("bar")));
+    CPPUNIT_ASSERT(fval1->find(StringFieldValue("bar")) != fval1->end());
+    CPPUNIT_ASSERT_EQUAL(14, fval1->get(StringFieldValue("bar"), 0));
+
+    // Do a second assign
+    WeightedSetFieldValue wset2(field.getDataType());
+    wset2.add(StringFieldValue("foo"), 16);
+    wset2.add(StringFieldValue("bar"), 24);
+    DocumentUpdate(*doc->getDataType(), doc->getId())
+        .addUpdate(FieldUpdate(field)
+                   .addUpdate(AssignValueUpdate(wset2)))
+        .applyTo(*doc);
+    std::unique_ptr<WeightedSetFieldValue>
+        fval2(doc->getAs<WeightedSetFieldValue>(field));
+    CPPUNIT_ASSERT_EQUAL((size_t) 2, fval2->size());
+    CPPUNIT_ASSERT(fval2->contains(StringFieldValue("foo")));
+    CPPUNIT_ASSERT(fval2->find(StringFieldValue("foo")) != fval1->end());
+    CPPUNIT_ASSERT_EQUAL(16, fval2->get(StringFieldValue("foo"), 0));
+    CPPUNIT_ASSERT(fval2->contains(StringFieldValue("bar")));
+    CPPUNIT_ASSERT(fval2->find(StringFieldValue("bar")) != fval1->end());
+    CPPUNIT_ASSERT_EQUAL(24, fval2->get(StringFieldValue("bar"), 0));
+
+    // Append weighted field
+    DocumentUpdate(*doc->getDataType(), doc->getId())
+        .addUpdate(FieldUpdate(field)
+                   .addUpdate(AddValueUpdate(StringFieldValue("foo"))
+                              .setWeight(3))
+                   .addUpdate(AddValueUpdate(StringFieldValue("too"))
+                              .setWeight(14)))
+        .applyTo(*doc);
+    std::unique_ptr<WeightedSetFieldValue>
+        fval3(doc->getAs<WeightedSetFieldValue>(field));
+    CPPUNIT_ASSERT_EQUAL((size_t) 3, fval3->size());
+    CPPUNIT_ASSERT(fval3->contains(StringFieldValue("foo")));
+    CPPUNIT_ASSERT_EQUAL(3, fval3->get(StringFieldValue("foo"), 0));
+    CPPUNIT_ASSERT(fval3->contains(StringFieldValue("bar")));
+    CPPUNIT_ASSERT_EQUAL(24, fval3->get(StringFieldValue("bar"), 0));
+    CPPUNIT_ASSERT(fval3->contains(StringFieldValue("too")));
+    CPPUNIT_ASSERT_EQUAL(14, fval3->get(StringFieldValue("too"), 0));
+
+    // Remove weighted field
+    DocumentUpdate(*doc->getDataType(), doc->getId())
+        .addUpdate(FieldUpdate(field)
+                   .addUpdate(RemoveValueUpdate(StringFieldValue("foo")))
+                   .addUpdate(RemoveValueUpdate(StringFieldValue("too"))))
+        .applyTo(*doc);
+    std::unique_ptr<WeightedSetFieldValue>
+        fval4(doc->getAs<WeightedSetFieldValue>(field));
+    CPPUNIT_ASSERT_EQUAL((size_t) 1, fval4->size());
+    CPPUNIT_ASSERT(!fval4->contains(StringFieldValue("foo")));
+    CPPUNIT_ASSERT(fval4->contains(StringFieldValue("bar")));
+    CPPUNIT_ASSERT_EQUAL(24, fval4->get(StringFieldValue("bar"), 0));
+    CPPUNIT_ASSERT(!fval4->contains(StringFieldValue("too")));
+}
+
+namespace {
+
+struct WeightedSetAutoCreateFixture
+{
+    DocumentTypeRepo repo;
+    const DocumentType* docType;
+    Document doc;
+    const Field& field;
+    DocumentUpdate update;
+
+    WeightedSetAutoCreateFixture()
+        : repo(makeConfig()),
+          docType(repo.getDocumentType("test")),
+          doc(*docType, DocumentId("doc::testdoc")),
+          field(docType->getField("strwset")),
+          update(*docType, DocumentId("doc::testdoc"))
+    {
+        update.addUpdate(FieldUpdate(field)
+                  .addUpdate(MapValueUpdate(StringFieldValue("foo"),
+                       ArithmeticValueUpdate(ArithmeticValueUpdate::Add, 1))));
+    }
+
+    void applyUpdateToDocument() {
+        update.applyTo(doc);
+    }
+
+    static DocumenttypesConfig makeConfig() {
+        DocumenttypesConfigBuilderHelper builder;
+        // T_TAG is an alias for a weighted set with create-if-non-existing
+        // and remove-if-zero attributes set. Attempting to explicitly create
+        // a field matching those characteristics will in fact fail with a
+        // redefinition error.
+        builder.document(42, "test",
+                         Struct("test.header")
+                         .addField("strwset", DataType::T_TAG),
+                         Struct("test.body"));
+        return builder.config();
+    }
+};
+
+} // anon ns
+
+void
+DocumentUpdateTest::testIncrementNonExistingAutoCreateWSetField()
+{
+    WeightedSetAutoCreateFixture fixture;
+
+    fixture.applyUpdateToDocument();
+
+    std::unique_ptr<WeightedSetFieldValue> ws(
+            fixture.doc.getAs<WeightedSetFieldValue>(fixture.field));
+    CPPUNIT_ASSERT_EQUAL(size_t(1), ws->size());
+    CPPUNIT_ASSERT(ws->contains(StringFieldValue("foo")));
+    CPPUNIT_ASSERT_EQUAL(1, ws->get(StringFieldValue("foo"), 0));
+}
+
+void
+DocumentUpdateTest::testIncrementExistingWSetField()
+{
+    WeightedSetAutoCreateFixture fixture;
+    {
+        WeightedSetFieldValue wset(fixture.field.getDataType());
+        wset.add(StringFieldValue("bar"), 14);
+        fixture.doc.setValue(fixture.field, wset);
+    }
+    fixture.applyUpdateToDocument();
+
+    std::unique_ptr<WeightedSetFieldValue> ws(
+            fixture.doc.getAs<WeightedSetFieldValue>(fixture.field));
+    CPPUNIT_ASSERT_EQUAL(size_t(2), ws->size());
+    CPPUNIT_ASSERT(ws->contains(StringFieldValue("foo")));
+    CPPUNIT_ASSERT_EQUAL(1, ws->get(StringFieldValue("foo"), 0));
+}
+
+void
+DocumentUpdateTest::testIncrementWithZeroResultWeightIsRemoved()
+{
+    WeightedSetAutoCreateFixture fixture;
+    fixture.update.addUpdate(FieldUpdate(fixture.field)
+            .addUpdate(MapValueUpdate(StringFieldValue("baz"),
+                    ArithmeticValueUpdate(ArithmeticValueUpdate::Add, 0))));
+
+    fixture.applyUpdateToDocument();
+
+    std::unique_ptr<WeightedSetFieldValue> ws(
+            fixture.doc.getAs<WeightedSetFieldValue>(fixture.field));
+    CPPUNIT_ASSERT_EQUAL(size_t(1), ws->size());
+    CPPUNIT_ASSERT(ws->contains(StringFieldValue("foo")));
+    CPPUNIT_ASSERT(!ws->contains(StringFieldValue("baz")));
+}
+
+void DocumentUpdateTest::testReadSerializedFile()
+{
+    // Reads a file serialized from java
+    const char file_name[] = "data/crossplatform-java-cpp-doctypes.cfg";
+    DocumentTypeRepo repo(readDocumenttypesConfig(file_name));
+
+    int fd = open("data/serializeupdatejava.dat", O_RDONLY);
+
+    int len = lseek(fd,0,SEEK_END);
+    ByteBuffer buf(len);
+    lseek(fd,0,SEEK_SET);
+    read(fd, buf.getBuffer(), len);
+
+    close(fd);
+
+    DocumentUpdate::UP updp(DocumentUpdate::create42(repo, buf));
+    DocumentUpdate& upd(*updp);
+
+    const DocumentType *type = repo.getDocumentType("serializetest");
+    CPPUNIT_ASSERT_EQUAL(DocumentId(DocIdString("update", "test")),
+                         upd.getId());
+    CPPUNIT_ASSERT_EQUAL(*type, upd.getType());
+
+    // Verify assign value update.
+    FieldUpdate serField = upd[0];
+    CPPUNIT_ASSERT_EQUAL(serField.getField().getId(Document::getNewestSerializationVersion()),
+                         type->getField("intfield").getId(Document::getNewestSerializationVersion()));
+
+    const ValueUpdate* serValue = &serField[0];
+    CPPUNIT_ASSERT_EQUAL(serValue->getType(), ValueUpdate::Assign);
+
+    const AssignValueUpdate* assign(
+            static_cast<const AssignValueUpdate*>(serValue));
+    CPPUNIT_ASSERT_EQUAL(IntFieldValue(4),
+                         static_cast<const IntFieldValue&>(assign->getValue()));
+
+    // Verify clear field update.
+    serField = upd[1];
+    CPPUNIT_ASSERT_EQUAL(serField.getField().getId(Document::getNewestSerializationVersion()),
+                         type->getField("floatfield").getId(Document::getNewestSerializationVersion()));
+
+    serValue = &serField[0];
+    CPPUNIT_ASSERT_EQUAL(serValue->getType(), ValueUpdate::Clear);
+    CPPUNIT_ASSERT(serValue->inherits(ClearValueUpdate::classId));
+
+    // Verify add value update.
+    serField = upd[2];
+    CPPUNIT_ASSERT_EQUAL(serField.getField().getId(Document::getNewestSerializationVersion()),
+                         type->getField("arrayoffloatfield").getId(Document::getNewestSerializationVersion()));
+
+    serValue = &serField[0];
+    CPPUNIT_ASSERT_EQUAL(serValue->getType(), ValueUpdate::Add);
+
+    const AddValueUpdate* add = static_cast<const AddValueUpdate*>(serValue);
+    const FieldValue* value = &add->getValue();
+    CPPUNIT_ASSERT(value->inherits(FloatFieldValue::classId));
+    CPPUNIT_ASSERT_EQUAL(value->getAsFloat(), 5.00f);
+
+    serValue = &serField[1];
+    CPPUNIT_ASSERT_EQUAL(serValue->getType(), ValueUpdate::Add);
+
+    add = static_cast<const AddValueUpdate*>(serValue);
+    value = &add->getValue();
+    CPPUNIT_ASSERT(value->inherits(FloatFieldValue::classId));
+    CPPUNIT_ASSERT_EQUAL(value->getAsFloat(), 4.23f);
+
+    serValue = &serField[2];
+    CPPUNIT_ASSERT_EQUAL(serValue->getType(), ValueUpdate::Add);
+
+    add = static_cast<const AddValueUpdate*>(serValue);
+    value = &add->getValue();
+    CPPUNIT_ASSERT(value->inherits(FloatFieldValue::classId));
+    CPPUNIT_ASSERT_EQUAL(value->getAsFloat(), -1.00f);
+
+}
+
+void DocumentUpdateTest::testGenerateSerializedFile()
+{
+    // Tests nothing, only generates a file for java test
+    const char file_name[] = "data/crossplatform-java-cpp-doctypes.cfg";
+    DocumentTypeRepo repo(readDocumenttypesConfig(file_name));
+
+    const DocumentType *type(repo.getDocumentType("serializetest"));
+    DocumentUpdate upd(*type, DocumentId(DocIdString("update", "test")));
+    upd.addUpdate(FieldUpdate(type->getField("intfield"))
+		  .addUpdate(AssignValueUpdate(IntFieldValue(4))));
+    upd.addUpdate(FieldUpdate(type->getField("floatfield"))
+		  .addUpdate(AssignValueUpdate(FloatFieldValue(1.00f))));
+    upd.addUpdate(FieldUpdate(type->getField("arrayoffloatfield"))
+		  .addUpdate(AddValueUpdate(FloatFieldValue(5.00f)))
+		  .addUpdate(AddValueUpdate(FloatFieldValue(4.23f)))
+		  .addUpdate(AddValueUpdate(FloatFieldValue(-1.00f))));
+    upd.addUpdate(FieldUpdate(type->getField("intfield"))
+          .addUpdate(ArithmeticValueUpdate(ArithmeticValueUpdate::Add, 3)));
+    upd.addUpdate(FieldUpdate(type->getField("wsfield"))
+          .addUpdate(MapValueUpdate(StringFieldValue("foo"),
+                        ArithmeticValueUpdate(ArithmeticValueUpdate::Add, 2)))
+          .addUpdate(MapValueUpdate(StringFieldValue("foo"),
+                        ArithmeticValueUpdate(ArithmeticValueUpdate::Mul, 2))));
+    ByteBuffer::UP buf(serialize42(upd));
+
+    int fd = open("data/serializeupdatecpp.dat",
+                  O_WRONLY | O_TRUNC | O_CREAT, 0644);
+    write(fd, buf->getBuffer(), buf->getPos());
+    close(fd);
+}
+
+
+void DocumentUpdateTest::testSetBadFieldTypes()
+{
+    // Create a test document
+    TestDocMan docMan;
+    Document::UP doc(docMan.createDocument());
+    CPPUNIT_ASSERT_EQUAL((document::FieldValue*)NULL,
+			 doc->getValue(doc->getField("headerval")).get());
+
+    // Assign a float value to an int field.
+    DocumentUpdate update(*doc->getDataType(), doc->getId());
+    try {
+        update.addUpdate(FieldUpdate(doc->getField("headerval"))
+                 .addUpdate(AssignValueUpdate(FloatFieldValue(4.00f))));
+        CPPUNIT_FAIL("Expected exception when adding a float to an int field.");
+    } catch (std::exception& e) {
+        ; // fprintf(stderr, "Got exception => OK: %s\n", e.what());
+    }
+
+    // Apply update
+    update.applyTo(*doc);
+
+    // Verify that the field is NOT set in the document.
+    CPPUNIT_ASSERT_EQUAL((document::FieldValue*)NULL,
+			 doc->getValue(doc->getField("headerval")).get());
+}
+
+void
+DocumentUpdateTest::testUpdateApplyNoParams()
+{
+    // Create a test document
+    TestDocMan docMan;
+    Document::UP doc(docMan.createDocument());
+    CPPUNIT_ASSERT_EQUAL((document::FieldValue*)NULL,
+			 doc->getValue(doc->getField("tags")).get());
+
+    // Assign array field with no parameters - illegal.
+    DocumentUpdate update(*doc->getDataType(), doc->getId());
+    try {
+        update.addUpdate(FieldUpdate(doc->getField("tags"))
+                 .addUpdate(AssignValueUpdate()));
+        CPPUNIT_FAIL("Expected exception when assign a NULL value.");
+    } catch (std::exception& e) {
+        ; // fprintf(stderr, "Got exception => OK: %s\n", e.what());
+    }
+
+    // Apply update
+    update.applyTo(*doc);
+
+    // Verify that the field was cleared in the document.
+    CPPUNIT_ASSERT(!doc->hasValue(doc->getField("tags")));
+}
+
+void
+DocumentUpdateTest::testUpdateApplyNoArrayValues()
+{
+    // Create a test document
+    TestDocMan docMan;
+    Document::UP doc(docMan.createDocument());
+    const Field &field(doc->getType().getField("tags"));
+    CPPUNIT_ASSERT_EQUAL((document::FieldValue*) 0,
+                         doc->getValue(field).get());
+
+    // Assign array field with no array values = empty array
+    DocumentUpdate update(*doc->getDataType(), doc->getId());
+    update.addUpdate(FieldUpdate(field)
+                     .addUpdate(AssignValueUpdate(
+                                     ArrayFieldValue(field.getDataType()))));
+
+    // Apply update
+    update.applyTo(*doc);
+
+    // Verify that the field was set in the document
+    std::unique_ptr<ArrayFieldValue> fval(doc->getAs<ArrayFieldValue>(field));
+    CPPUNIT_ASSERT(fval.get());
+    CPPUNIT_ASSERT_EQUAL((size_t) 0, fval->size());
+}
+
+void
+DocumentUpdateTest::testUpdateArrayEmptyParamValue()
+{
+    // Create a test document.
+    TestDocMan docMan;
+    Document::UP doc(docMan.createDocument());
+    const Field &field(doc->getType().getField("tags"));
+    CPPUNIT_ASSERT_EQUAL((document::FieldValue*) 0,
+                         doc->getValue(field).get());
+
+    // Assign array field with no array values = empty array.
+    DocumentUpdate update(*doc->getDataType(), doc->getId());
+    update.addUpdate(FieldUpdate(field)
+                     .addUpdate(AssignValueUpdate(
+                                     ArrayFieldValue(field.getDataType()))));
+    update.applyTo(*doc);
+
+    // Verify that the field was set in the document.
+    std::unique_ptr<ArrayFieldValue> fval1(doc->getAs<ArrayFieldValue>(field));
+    CPPUNIT_ASSERT(fval1.get());
+    CPPUNIT_ASSERT_EQUAL((size_t) 0, fval1->size());
+
+    // Remove array field.
+    update.clear();
+    update.addUpdate(FieldUpdate(field)
+		    .addUpdate(ClearValueUpdate()));
+    update.applyTo(*doc);
+
+    // Verify that the field was cleared in the document.
+    std::unique_ptr<ArrayFieldValue> fval2(doc->getAs<ArrayFieldValue>(field));
+    CPPUNIT_ASSERT(!fval2);
+}
+
+void
+DocumentUpdateTest::testUpdateWeightedSetEmptyParamValue()
+{
+    // Create a test document
+    TestDocMan docMan;
+    Document::UP doc(docMan.createDocument());
+    const Field &field(doc->getType().getField("stringweightedset"));
+    CPPUNIT_ASSERT_EQUAL((document::FieldValue*) 0,
+			             doc->getValue(field).get());
+
+    // Assign weighted set with no items = empty set.
+    DocumentUpdate update(*doc->getDataType(), doc->getId());
+    update.addUpdate(FieldUpdate(field)
+		    .addUpdate(AssignValueUpdate(
+                    WeightedSetFieldValue(field.getDataType()))));
+    update.applyTo(*doc);
+
+    // Verify that the field was set in the document.
+    std::unique_ptr<WeightedSetFieldValue>
+        fval1(doc->getAs<WeightedSetFieldValue>(field));
+    CPPUNIT_ASSERT(fval1.get());
+    CPPUNIT_ASSERT_EQUAL((size_t) 0, fval1->size());
+
+    // Remove weighted set field.
+    update.clear();
+    update.addUpdate(FieldUpdate(field)
+		    .addUpdate(ClearValueUpdate()));
+    update.applyTo(*doc);
+
+    // Verify that the field was cleared in the document.
+    std::unique_ptr<WeightedSetFieldValue>
+        fval2(doc->getAs<WeightedSetFieldValue>(field));
+    CPPUNIT_ASSERT(!fval2);
+}
+
+void
+DocumentUpdateTest::testUpdateArrayWrongSubtype()
+{
+    // Create a test document
+    TestDocMan docMan;
+    Document::UP doc(docMan.createDocument());
+    const Field &field(doc->getType().getField("tags"));
+    CPPUNIT_ASSERT_EQUAL((document::FieldValue*) 0,
+			             doc->getValue(field).get());
+
+    // Assign int values to string array.
+    DocumentUpdate update(*doc->getDataType(), doc->getId());
+    try {
+        update.addUpdate(FieldUpdate(field)
+                 .addUpdate(AddValueUpdate(IntFieldValue(123)))
+                 .addUpdate(AddValueUpdate(IntFieldValue(456))));
+        CPPUNIT_FAIL("Expected exception when adding wrong type.");
+    } catch (std::exception& e) {
+        ; // fprintf(stderr, "Got exception => OK: %s\n", e.what());
+    }
+
+    // Apply update
+    update.applyTo(*doc);
+
+    // Verify that the field was NOT set in the document
+    FieldValue::UP fval(doc->getValue(field));
+    CPPUNIT_ASSERT_EQUAL((document::FieldValue*) 0, fval.get());
+}
+
+void
+DocumentUpdateTest::testUpdateWeightedSetWrongSubtype()
+{
+    // Create a test document
+    TestDocMan docMan;
+    Document::UP doc(docMan.createDocument());
+    const Field &field(doc->getType().getField("stringweightedset"));
+    CPPUNIT_ASSERT_EQUAL((document::FieldValue*) 0,
+			 doc->getValue(field).get());
+
+    // Assign int values to string array.
+    DocumentUpdate update(*doc->getDataType(), doc->getId());
+    try {
+        update.addUpdate(FieldUpdate(field)
+                 .addUpdate(AddValueUpdate(IntFieldValue(123))
+                        .setWeight(1000))
+                 .addUpdate(AddValueUpdate(IntFieldValue(456))
+                        .setWeight(2000)));
+        CPPUNIT_FAIL("Expected exception when adding wrong type.");
+    } catch (std::exception& e) {
+        ; // fprintf(stderr, "Got exception => OK: %s\n", e.what());
+    }
+
+    // Apply update
+    update.applyTo(*doc);
+
+    // Verify that the field was NOT set in the document
+    FieldValue::UP fval(doc->getValue(field));
+    CPPUNIT_ASSERT_EQUAL((document::FieldValue*) 0, fval.get());
+}
+
+void
+DocumentUpdateTest::testMapValueUpdate()
+{
+    // Create a test document
+    TestDocMan docMan;
+    Document::UP doc(docMan.createDocument());
+    const Field &field1 = doc->getField("stringweightedset");
+    const Field &field2 = doc->getField("stringweightedset2");
+    WeightedSetFieldValue wsval1(field1.getDataType());
+    WeightedSetFieldValue wsval2(field2.getDataType());
+    doc->setValue(field1, wsval1);
+    doc->setValue(field2, wsval2);
+
+    DocumentUpdate(*doc->getDataType(), doc->getId())
+        .addUpdate(FieldUpdate(field1)
+                   .addUpdate(MapValueUpdate(
+                        StringFieldValue("banana"),
+                        ArithmeticValueUpdate(ArithmeticValueUpdate::Add, 1.0)
+                                             )))
+        .applyTo(*doc);
+    std::unique_ptr<WeightedSetFieldValue> fv1 =
+        doc->getAs<WeightedSetFieldValue>(field1);
+    CPPUNIT_ASSERT(fv1->size() == 0);
+
+    DocumentUpdate(*doc->getDataType(), doc->getId())
+        .addUpdate(FieldUpdate(field2)
+                   .addUpdate(MapValueUpdate(
+                        StringFieldValue("banana"),
+                        ArithmeticValueUpdate(ArithmeticValueUpdate::Add, 1.0)
+                                             )))
+        .applyTo(*doc);
+    std::unique_ptr<WeightedSetFieldValue> fv2 =
+        doc->getAs<WeightedSetFieldValue>(field2);
+    CPPUNIT_ASSERT(fv2->size() == 1);
+
+    CPPUNIT_ASSERT(fv1->find(StringFieldValue("apple")) == fv1->end());
+    DocumentUpdate(*doc->getDataType(), doc->getId())
+        .addUpdate(FieldUpdate(field1)
+                   .addUpdate(ClearValueUpdate()))
+        .applyTo(*doc);
+
+    DocumentUpdate(*doc->getDataType(), doc->getId())
+        .addUpdate(FieldUpdate(field1)
+                   .addUpdate(AddValueUpdate(StringFieldValue("apple"))
+                              .setWeight(1)))
+        .applyTo(*doc);
+
+    std::unique_ptr<WeightedSetFieldValue>
+        fval3(doc->getAs<WeightedSetFieldValue>(field1));
+    CPPUNIT_ASSERT(fval3->find(StringFieldValue("apple")) != fval3->end());
+    CPPUNIT_ASSERT_EQUAL(1, fval3->get(StringFieldValue("apple")));
+
+    DocumentUpdate(*doc->getDataType(), doc->getId())
+        .addUpdate(FieldUpdate(field2)
+                   .addUpdate(AddValueUpdate(StringFieldValue("apple"))
+                              .setWeight(1)))
+        .applyTo(*doc);
+
+    std::unique_ptr<WeightedSetFieldValue>
+        fval3b(doc->getAs<WeightedSetFieldValue>(field2));
+    CPPUNIT_ASSERT(fval3b->find(StringFieldValue("apple")) != fval3b->end());
+    CPPUNIT_ASSERT_EQUAL(1, fval3b->get(StringFieldValue("apple")));
+
+    DocumentUpdate(*doc->getDataType(), doc->getId())
+        .addUpdate(FieldUpdate(field1)
+                   .addUpdate(MapValueUpdate(
+                        StringFieldValue("apple"),
+                        ArithmeticValueUpdate(ArithmeticValueUpdate::Sub, 1.0)
+                                             )))
+        .applyTo(*doc);
+
+    std::unique_ptr<WeightedSetFieldValue> fv3 =
+        doc->getAs<WeightedSetFieldValue>(field1);
+    CPPUNIT_ASSERT(fv3->find(StringFieldValue("apple")) != fv3->end());
+    CPPUNIT_ASSERT_EQUAL(0, fv3->get(StringFieldValue("apple")));
+
+    DocumentUpdate(*doc->getDataType(), doc->getId())
+        .addUpdate(FieldUpdate(field2)
+                   .addUpdate(MapValueUpdate(
+                        StringFieldValue("apple"),
+                        ArithmeticValueUpdate(ArithmeticValueUpdate::Sub, 1.0)
+                                             )))
+        .applyTo(*doc);
+
+    std::unique_ptr<WeightedSetFieldValue> fv4 =
+        doc->getAs<WeightedSetFieldValue>(field2);
+    CPPUNIT_ASSERT(fv4->find(StringFieldValue("apple")) == fv4->end());
+}
+
+
+void
+assertDocumentUpdateFlag(bool createIfNonExistent, int value)
+{
+    DocumentUpdateFlags f1;
+    f1.setCreateIfNonExistent(createIfNonExistent);
+    CPPUNIT_ASSERT_EQUAL(createIfNonExistent, f1.getCreateIfNonExistent());
+    int combined = f1.injectInto(value);
+    std::cout << "createIfNonExistent=" << createIfNonExistent << ", value=" << value << ", combined=" << combined << std::endl;
+
+    DocumentUpdateFlags f2 = DocumentUpdateFlags::extractFlags(combined);
+    int extractedValue = DocumentUpdateFlags::extractValue(combined);
+    CPPUNIT_ASSERT_EQUAL(createIfNonExistent, f2.getCreateIfNonExistent());
+    CPPUNIT_ASSERT_EQUAL(value, extractedValue);
+}
+
+void
+DocumentUpdateTest::testThatDocumentUpdateFlagsIsWorking()
+{
+    { // create-if-non-existent = true
+        assertDocumentUpdateFlag(true, 0);
+        assertDocumentUpdateFlag(true, 1);
+        assertDocumentUpdateFlag(true, 2);
+        assertDocumentUpdateFlag(true, 9999);
+        assertDocumentUpdateFlag(true, 0xFFFFFFE);
+        assertDocumentUpdateFlag(true, 0xFFFFFFF);
+    }
+    { // create-if-non-existent = false
+        assertDocumentUpdateFlag(false, 0);
+        assertDocumentUpdateFlag(false, 1);
+        assertDocumentUpdateFlag(false, 2);
+        assertDocumentUpdateFlag(false, 9999);
+        assertDocumentUpdateFlag(false, 0xFFFFFFE);
+        assertDocumentUpdateFlag(false, 0xFFFFFFF);
+    }
+}
+
+struct CreateIfNonExistentFixture
+{
+    TestDocMan docMan;
+    Document::UP document;
+    DocumentUpdate::UP update;
+    CreateIfNonExistentFixture()
+        : docMan(),
+          document(docMan.createDocument()),
+          update(new DocumentUpdate(*document->getDataType(),
+                                    document->getId()))
+    {
+        update->addUpdate(FieldUpdate(document->getField("headerval"))
+                          .addUpdate(AssignValueUpdate(IntFieldValue(1))));
+        update->setCreateIfNonExistent(true);
+    }
+};
+
+void
+DocumentUpdateTest::testThatCreateIfNonExistentFlagIsSerialized50AndDeserialized50()
+{
+    CreateIfNonExistentFixture f;
+
+    ByteBuffer::UP buf(serializeHEAD(*f.update));
+    buf->flip();
+
+    DocumentUpdate::UP deserialized = DocumentUpdate::createHEAD(f.docMan.getTypeRepo(), *buf);
+    CPPUNIT_ASSERT_EQUAL(*f.update, *deserialized);
+    CPPUNIT_ASSERT(deserialized->getCreateIfNonExistent());
+}
+
+void
+DocumentUpdateTest::testThatCreateIfNonExistentFlagIsSerializedAndDeserialized()
+{
+    CreateIfNonExistentFixture f;
+
+    ByteBuffer::UP buf(serialize42(*f.update));
+    buf->flip();
+
+    DocumentUpdate::UP deserialized = DocumentUpdate::create42(f.docMan.getTypeRepo(), *buf);
+    CPPUNIT_ASSERT_EQUAL(*f.update, *deserialized);
+    CPPUNIT_ASSERT(deserialized->getCreateIfNonExistent());
+}
+
+}  // namespace document
diff --git a/document/src/tests/documentupdatetestcase.h b/document/src/tests/documentupdatetestcase.h
new file mode 100644
index 00000000000..b8d8294ec0d
--- /dev/null
+++ b/document/src/tests/documentupdatetestcase.h
@@ -0,0 +1,50 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/* $Id$*/
+
+#pragma once
+
+#include <cppunit/extensions/HelperMacros.h>
+
+class DocumentUpdate_Test : public CppUnit::TestFixture {
+  CPPUNIT_TEST_SUITE(DocumentUpdate_Test);
+  CPPUNIT_TEST(testUpdateApplySingleValue);
+  CPPUNIT_TEST(testUpdateArray);
+  CPPUNIT_TEST(testUpdateWeightedSet);
+  CPPUNIT_TEST(testReadSerializedFile);
+  CPPUNIT_TEST(testGenerateSerializedFile);
+  CPPUNIT_TEST(testSetBadFieldTypes);
+  CPPUNIT_TEST(testUpdateApplyNoParams);
+  CPPUNIT_TEST(testUpdateApplyNoArrayValues);
+  CPPUNIT_TEST(testUpdateArrayEmptyParamValue);
+  CPPUNIT_TEST(testUpdateWeightedSetEmptyParamValue);
+  CPPUNIT_TEST(testUpdateArrayWrongSubtype);
+  CPPUNIT_TEST(testUpdateWeightedSetWrongSubtype);
+  CPPUNIT_TEST(testMapValueUpdate);
+  CPPUNIT_TEST(testThatDocumentUpdateFlagsIsWorking);
+  CPPUNIT_TEST(testThatCreateIfNonExistentFlagIsSerialized50AndDeserialized50);
+  CPPUNIT_TEST(testThatCreateIfNonExistentFlagIsSerializedAndDeserialized);
+  CPPUNIT_TEST_SUITE_END();
+
+public:
+  void setUp();
+  void tearDown();
+
+protected:
+  void testUpdateApplySingleValue();
+  void testUpdateArray();
+  void testUpdateWeightedSet();
+  void testReadSerializedFile();
+  void testGenerateSerializedFile();
+  void testSetBadFieldTypes();
+  void testUpdateApplyNoParams();
+  void testUpdateApplyNoArrayValues();
+  void testUpdateArrayEmptyParamValue();
+  void testUpdateWeightedSetEmptyParamValue();
+  void testUpdateArrayWrongSubtype();
+  void testUpdateWeightedSetWrongSubtype();
+  void testMapValueUpdate();
+  void testThatDocumentUpdateFlagsIsWorking();
+  void testThatCreateIfNonExistentFlagIsSerialized50AndDeserialized50();
+  void testThatCreateIfNonExistentFlagIsSerializedAndDeserialized();
+};
+
diff --git a/document/src/tests/fieldpathupdatetestcase.cpp b/document/src/tests/fieldpathupdatetestcase.cpp
new file mode 100644
index 00000000000..ec999d0dfe9
--- /dev/null
+++ b/document/src/tests/fieldpathupdatetestcase.cpp
@@ -0,0 +1,1310 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/document/base/testdocman.h>
+#include <vespa/document/fieldvalue/fieldvalues.h>
+#include <vespa/document/select/node.h>
+#include <vespa/vespalib/io/fileutil.h>
+#include <vespa/vdstestlib/cppunit/macros.h>
+#include <vespa/document/update/fieldpathupdates.h>
+#include <vespa/document/update/documentupdate.h>
+
+#include <vespa/document/repo/configbuilder.h>
+#include <vespa/vespalib/objects/identifiable.h>
+#include <vespa/document/serialization/vespadocumentserializer.h>
+#include <fstream>
+#include <sstream>
+
+using vespalib::Identifiable;
+using namespace document::config_builder;
+
+namespace document {
+
+struct FieldPathUpdateTestCase : public CppUnit::TestFixture {
+    DocumentTypeRepo::SP _repo;
+    DocumentType _foobar_type;
+
+    void setUp();
+    void tearDown();
+
+    void testWhereClause();
+    void testNoIterateMapValues();
+    void testRemoveField();
+    void testApplyRemoveEntireListField();
+    void testApplyRemoveMultiList();
+    void testApplyRemoveMultiWset();
+    void testApplyAssignSingle();
+    void testApplyAssignMath();
+    void testApplyAssignMathDivZero();
+    void testApplyAssignMathByteToZero();
+    void testApplyAssignMathNotModifiedOnUnderflow();
+    void testApplyAssignMathNotModifiedOnOverflow();
+    void testApplyAssignFieldNotExistingInExpression();
+    void testApplyAssignFieldNotExistingInPath();
+    void testApplyAssignTargetNotExisting();
+    void testAssignSimpleMapValueWithVariable();
+    void testApplyAssignMathRemoveIfZero();
+    void testApplyAssignMultiList();
+    void testApplyAssignMultiWset();
+    void testAssignWsetRemoveIfZero();
+    void testApplyAddMultiList();
+    void testAddAndAssignList();
+    void testAssignMap();
+    void testAssignMapStruct();
+    void testAssignMapStructVariable();
+    void testAssignMapNoExist();
+    void testAssignMapNoExistNoCreate();
+    void testQuotedStringKey();
+    void testEqualityComparison();
+    void testAffectsDocumentBody();
+    void testIncompatibleDataTypeFails();
+    void testSerializeAssign();
+    void testSerializeAdd();
+    void testSerializeRemove();
+    void testSerializeAssignMath();
+    void testReadSerializedFile();
+    void testGenerateSerializedFile();
+
+    CPPUNIT_TEST_SUITE(FieldPathUpdateTestCase);
+    CPPUNIT_TEST(testWhereClause);
+    CPPUNIT_TEST(testNoIterateMapValues);
+    CPPUNIT_TEST(testRemoveField);
+    CPPUNIT_TEST(testApplyRemoveEntireListField);
+    CPPUNIT_TEST(testApplyRemoveMultiList);
+    CPPUNIT_TEST(testApplyRemoveMultiWset);
+    CPPUNIT_TEST(testApplyAssignSingle);
+    CPPUNIT_TEST(testApplyAssignMath);
+    CPPUNIT_TEST(testApplyAssignMathDivZero);
+    CPPUNIT_TEST(testApplyAssignMathByteToZero);
+    CPPUNIT_TEST(testApplyAssignMathNotModifiedOnUnderflow);
+    CPPUNIT_TEST(testApplyAssignMathNotModifiedOnOverflow);
+    CPPUNIT_TEST(testApplyAssignFieldNotExistingInExpression);
+    CPPUNIT_TEST(testApplyAssignFieldNotExistingInPath);
+    CPPUNIT_TEST(testApplyAssignTargetNotExisting);
+    CPPUNIT_TEST(testAssignSimpleMapValueWithVariable);
+    CPPUNIT_TEST(testApplyAssignMathRemoveIfZero);
+    CPPUNIT_TEST(testApplyAssignMultiList);
+    CPPUNIT_TEST(testApplyAssignMultiWset);
+    CPPUNIT_TEST(testAssignWsetRemoveIfZero);
+    CPPUNIT_TEST(testApplyAddMultiList);
+    CPPUNIT_TEST(testAddAndAssignList);
+    CPPUNIT_TEST(testAssignMap);
+    CPPUNIT_TEST(testAssignMapStruct);
+    CPPUNIT_TEST(testAssignMapStructVariable);
+    CPPUNIT_TEST(testAssignMapNoExist);
+    CPPUNIT_TEST(testAssignMapNoExistNoCreate);
+    CPPUNIT_TEST(testQuotedStringKey);
+    CPPUNIT_TEST(testEqualityComparison);
+    CPPUNIT_TEST(testAffectsDocumentBody);
+    CPPUNIT_TEST(testIncompatibleDataTypeFails);
+    CPPUNIT_TEST(testSerializeAssign);
+    CPPUNIT_TEST(testSerializeAdd);
+    CPPUNIT_TEST(testSerializeRemove);
+    CPPUNIT_TEST(testSerializeAssignMath);
+    CPPUNIT_TEST(testReadSerializedFile);
+    CPPUNIT_TEST(testGenerateSerializedFile);
+    CPPUNIT_TEST_SUITE_END();
+private:
+    DocumentUpdate::UP
+    createDocumentUpdateForSerialization(const DocumentTypeRepo& repo);
+};
+
+CPPUNIT_TEST_SUITE_REGISTRATION(FieldPathUpdateTestCase);
+
+namespace {
+
+document::DocumenttypesConfig getRepoConfig() {
+    const int struct2_id = 64;
+    DocumenttypesConfigBuilderHelper builder;
+    builder.document(
+            42, "test",
+            Struct("test.header")
+            .addField("primitive1", DataType::T_INT)
+            .addField("l1s1", Struct("struct3")
+                      .addField("primitive1", DataType::T_INT)
+                      .addField("ss", Struct("struct2")
+                                .setId(struct2_id)
+                                .addField("primitive1", DataType::T_INT)
+                                .addField("primitive2", DataType::T_INT)
+                                .addField("iarray", Array(DataType::T_INT))
+                                .addField("sarray", Array(
+                                                Struct("struct1")
+                                                .addField("primitive1",
+                                                        DataType::T_INT)
+                                                .addField("primitive2",
+                                                        DataType::T_INT)))
+                                .addField("smap", Map(DataType::T_STRING,
+                                                DataType::T_STRING)))
+                      .addField("structmap",
+                                Map(DataType::T_STRING, struct2_id))
+                      .addField("wset", Wset(DataType::T_STRING))
+                      .addField("structwset", Wset(struct2_id))),
+            Struct("test.body"));
+    return builder.config();
+}
+
+Document::UP
+createTestDocument(const DocumentTypeRepo &repo)
+{
+    const DocumentType* type(repo.getDocumentType("test"));
+    const DataType* struct3(repo.getDataType(*type, "struct3"));
+    const DataType* struct2(repo.getDataType(*type, "struct2"));
+    const DataType* iarr(repo.getDataType(*type, "Array<Int>"));
+    const DataType* sarr(repo.getDataType(*type, "Array<struct1>"));
+    const DataType* struct1(repo.getDataType(*type, "struct1"));
+    const DataType* smap(repo.getDataType(*type, "Map<String,String>"));
+    const DataType* structmap(repo.getDataType(*type, "Map<String,struct2>"));
+    const DataType* wset(repo.getDataType(*type, "WeightedSet<String>"));
+    const DataType* structwset(repo.getDataType(*type, "WeightedSet<struct2>"));
+    Document::UP
+        doc(new Document(*type, DocumentId("doc::testdoc")));
+    doc->setValue("primitive1", IntFieldValue(1));
+    StructFieldValue l1s1(*struct3);
+    l1s1.setValue("primitive1", IntFieldValue(2));
+
+    StructFieldValue l2s1(*struct2);
+    l2s1.setValue("primitive1", IntFieldValue(3));
+    l2s1.setValue("primitive2", IntFieldValue(4));
+    StructFieldValue l2s2(*struct2);
+    l2s2.setValue("primitive1", IntFieldValue(5));
+    l2s2.setValue("primitive2", IntFieldValue(6));
+    ArrayFieldValue iarr1(*iarr);
+    iarr1.add(IntFieldValue(11));
+    iarr1.add(IntFieldValue(12));
+    iarr1.add(IntFieldValue(13));
+    ArrayFieldValue sarr1(*sarr);
+    StructFieldValue l3s1(*struct1);
+    l3s1.setValue("primitive1", IntFieldValue(1));
+    l3s1.setValue("primitive2", IntFieldValue(2));
+    sarr1.add(l3s1);
+    sarr1.add(l3s1);
+    MapFieldValue smap1(*smap);
+    smap1.put(StringFieldValue("leonardo"), StringFieldValue("dicaprio"));
+    smap1.put(StringFieldValue("ellen"), StringFieldValue("page"));
+    smap1.put(StringFieldValue("joseph"), StringFieldValue("gordon-levitt"));
+    l2s1.setValue("smap", smap1);
+    l2s1.setValue("iarray", iarr1);
+    l2s1.setValue("sarray", sarr1);
+
+    l1s1.setValue("ss", l2s1);
+    MapFieldValue structmap1(*structmap);
+    structmap1.put(StringFieldValue("test"), l2s1);
+    l1s1.setValue("structmap", structmap1);
+
+    WeightedSetFieldValue wset1(*wset);
+    wset1.add("foo");
+    wset1.add("bar");
+    wset1.add("zoo");
+    l1s1.setValue("wset", wset1);
+
+    WeightedSetFieldValue wset2(*structwset);
+    wset2.add(l2s1);
+    wset2.add(l2s2);
+    l1s1.setValue("structwset", wset2);
+
+    doc->setValue("l1s1", l1s1);
+    return doc;
+}
+
+ByteBuffer::UP serializeHEAD(const DocumentUpdate & update)
+{
+    vespalib::nbostream stream;
+    VespaDocumentSerializer serializer(stream);
+    serializer.writeHEAD(update);
+    ByteBuffer::UP retVal(new ByteBuffer(stream.size()));
+    retVal->putBytes(stream.peek(), stream.size());
+    return retVal;
+}
+
+void testSerialize(const DocumentTypeRepo& repo, const DocumentUpdate& a) {
+    try{
+        bool affectsBody = a.affectsDocumentBody();
+        ByteBuffer::UP bb(serializeHEAD(a));
+        bb->flip();
+        DocumentUpdate::UP b(DocumentUpdate::createHEAD(repo, *bb));
+
+        CPPUNIT_ASSERT_EQUAL(affectsBody, b->affectsDocumentBody());
+        CPPUNIT_ASSERT_EQUAL(size_t(0), bb->getRemaining());
+        CPPUNIT_ASSERT_EQUAL(a.getId().toString(), b->getId().toString());
+        CPPUNIT_ASSERT_EQUAL(a.getUpdates().size(), b->getUpdates().size());
+        for (size_t i(0); i < a.getUpdates().size(); i++) {
+            const FieldUpdate & ua = a.getUpdates()[i];
+            const FieldUpdate & ub = b->getUpdates()[i];
+
+            CPPUNIT_ASSERT_EQUAL(&ua.getField(), &ub.getField());
+            CPPUNIT_ASSERT_EQUAL(ua.getUpdates().size(),
+                                 ub.getUpdates().size());
+            for (size_t j(0); j < ua.getUpdates().size(); j++) {
+                CPPUNIT_ASSERT_EQUAL(ua.getUpdates()[j]->getType(),
+                                     ub.getUpdates()[j]->getType());
+            }
+        }
+        CPPUNIT_ASSERT_EQUAL(a.getFieldPathUpdates().size(), b->getFieldPathUpdates().size());
+        for (size_t i(0); i < a.getFieldPathUpdates().size(); i++) {
+            const FieldPathUpdate::CP& ua = a.getFieldPathUpdates()[i];
+            const FieldPathUpdate::CP& ub = b->getFieldPathUpdates()[i];
+
+            CPPUNIT_ASSERT_EQUAL(*ua, *ub);
+        }
+        CPPUNIT_ASSERT_EQUAL(a, *b);
+    } catch (std::exception& e) {
+        std::cerr << "Failed while testing document field path update:\n"
+                  << a.toString(true) << "\n";
+        throw;
+    }
+}
+
+} // anon ns
+
+struct TestFieldPathUpdate : FieldPathUpdate
+{
+    struct TestIteratorHandler : FieldValue::IteratorHandler
+    {
+        TestIteratorHandler(std::string& str)
+            : _str(str) {}
+
+        ModificationStatus doModify(FieldValue& value)
+        {
+            std::ostringstream ss;
+            value.print(ss, false, "");
+            if (!_str.empty()) {
+                _str += ';';
+            }
+            _str += ss.str();
+            return NOT_MODIFIED;
+        }
+
+        bool onComplex(const Content&) { return false; }
+
+        std::string& _str;
+    };
+
+    mutable std::string _str;
+
+    TestFieldPathUpdate(const DocumentTypeRepo& repo,
+                        const DataType *type,
+                        const std::string& fieldPath,
+                        const std::string& whereClause)
+        : FieldPathUpdate(repo, *type, fieldPath, whereClause)
+    {
+    }
+
+    TestFieldPathUpdate(const TestFieldPathUpdate& other)
+        : FieldPathUpdate(other)
+    {
+    }
+
+    std::unique_ptr<FieldValue::IteratorHandler> getIteratorHandler(Document&) const
+    {
+        return std::unique_ptr<FieldValue::IteratorHandler>(
+                new TestIteratorHandler(_str));
+    }
+
+    TestFieldPathUpdate* clone() const { return new TestFieldPathUpdate(*this); }
+
+    void print(std::ostream& out, bool, const std::string&) const
+    {
+        out << "TestFieldPathUpdate()";
+    }
+
+    void accept(UpdateVisitor & visitor) const override { (void) visitor; }
+    uint8_t getSerializedType() const override { assert(false); return 7; }
+};
+
+void
+FieldPathUpdateTestCase::setUp()
+{
+    DocumenttypesConfigBuilderHelper builder;
+    builder.document(42, "foobar",
+                     Struct("foobar.header")
+                     .addField("num", DataType::T_INT)
+                     .addField("byteval", DataType::T_BYTE)
+                     .addField("strfoo", DataType::T_STRING)
+                     .addField("strarray", Array(DataType::T_STRING)),
+                     Struct("foobar.body")
+                     .addField("strwset", Wset(DataType::T_STRING))
+                     .addField("structmap",
+                               Map(DataType::T_STRING, Struct("mystruct")
+                                       .addField("title", DataType::T_STRING)
+                                       .addField("rating", DataType::T_INT)))
+                     .addField("strmap",
+                               Map(DataType::T_STRING, DataType::T_STRING)));
+    _repo.reset(new DocumentTypeRepo(builder.config()));
+
+    _foobar_type = *_repo->getDocumentType("foobar");
+}
+
+void
+FieldPathUpdateTestCase::tearDown()
+{
+}
+
+void
+FieldPathUpdateTestCase::testWhereClause()
+{
+    DocumentTypeRepo repo(getRepoConfig());
+    Document::UP doc(createTestDocument(repo));
+    std::string where = "test.l1s1.structmap.value.smap{$x} == \"dicaprio\"";
+    TestFieldPathUpdate update(repo, doc->getDataType(),
+                               "l1s1.structmap.value.smap{$x}", where);
+    update.applyTo(*doc);
+    CPPUNIT_ASSERT_EQUAL(std::string("dicaprio"), update._str);
+}
+
+void
+FieldPathUpdateTestCase::testNoIterateMapValues()
+{
+    DocumentTypeRepo repo(getRepoConfig());
+    Document::UP doc(createTestDocument(repo));
+    TestFieldPathUpdate update(repo, doc->getDataType(),
+                               "l1s1.structwset.primitive1", "true");
+    update.applyTo(*doc);
+    CPPUNIT_ASSERT_EQUAL(std::string("3;5"), update._str);
+}
+
+void
+FieldPathUpdateTestCase::testRemoveField()
+{
+    Document::UP doc(
+            new Document(_foobar_type, DocumentId("doc:things:thangs")));
+    CPPUNIT_ASSERT(doc->hasValue("strfoo") == false);
+    doc->setValue("strfoo", StringFieldValue("cocacola"));
+    CPPUNIT_ASSERT_EQUAL(vespalib::string("cocacola"), doc->getValue("strfoo")->getAsString());
+    //doc->print(std::cerr, true, "");
+    DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo"));
+    docUp.addFieldPathUpdate(FieldPathUpdate::CP(new RemoveFieldPathUpdate(*_repo, *doc->getDataType(), "strfoo")));
+    docUp.applyTo(*doc);
+    CPPUNIT_ASSERT(doc->hasValue("strfoo") == false);
+}
+
+void
+FieldPathUpdateTestCase::testApplyRemoveMultiList()
+{
+     Document::UP doc(
+            new Document(_foobar_type, DocumentId("doc:things:thangs")));
+    CPPUNIT_ASSERT(doc->hasValue("strarray") == false);
+    {
+        ArrayFieldValue
+            strArray(doc->getType().getField("strarray").getDataType());
+        strArray.add(StringFieldValue("crouching tiger, hidden field"));
+        strArray.add(StringFieldValue("remove val 1"));
+        strArray.add(StringFieldValue("hello hello"));
+        doc->setValue("strarray", strArray);
+    }
+    CPPUNIT_ASSERT(doc->hasValue("strarray"));
+    //doc->print(std::cerr, true, "");
+    DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo"));
+    docUp.addFieldPathUpdate(
+            FieldPathUpdate::CP(
+                    new RemoveFieldPathUpdate(*_repo, *doc->getDataType(),
+                            "strarray[$x]", "foobar.strarray[$x] == \"remove val 1\"")));
+    docUp.applyTo(*doc);
+    {
+        std::unique_ptr<ArrayFieldValue> strArray =
+            doc->getAs<ArrayFieldValue>(doc->getField("strarray"));
+        CPPUNIT_ASSERT_EQUAL(std::size_t(2), strArray->size());
+        CPPUNIT_ASSERT_EQUAL(vespalib::string("crouching tiger, hidden field"), (*strArray)[0].getAsString());
+        CPPUNIT_ASSERT_EQUAL(vespalib::string("hello hello"), (*strArray)[1].getAsString());
+    }
+}
+
+void
+FieldPathUpdateTestCase::testApplyRemoveEntireListField()
+{
+     Document::UP doc(
+            new Document(_foobar_type, DocumentId("doc:things:thangs")));
+    CPPUNIT_ASSERT(doc->hasValue("strarray") == false);
+    {
+        ArrayFieldValue
+            strArray(doc->getType().getField("strarray").getDataType());
+        strArray.add(StringFieldValue("this list"));
+        strArray.add(StringFieldValue("should be"));
+        strArray.add(StringFieldValue("totally removed"));
+        doc->setValue("strarray", strArray);
+    }
+    //doc->print(std::cerr, true, "");
+    DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo"));
+    docUp.addFieldPathUpdate(
+            FieldPathUpdate::CP(
+                    new RemoveFieldPathUpdate(*_repo, *doc->getDataType(), "strarray", "")));
+    docUp.applyTo(*doc);
+    CPPUNIT_ASSERT(!doc->hasValue("strarray"));
+}
+
+void
+FieldPathUpdateTestCase::testApplyRemoveMultiWset()
+{
+    Document::UP doc(
+            new Document(_foobar_type, DocumentId("doc:helan:halvan")));
+    CPPUNIT_ASSERT(doc->hasValue("strwset") == false);
+    {
+        WeightedSetFieldValue
+            strWset(doc->getType().getField("strwset").getDataType());
+        strWset.add(StringFieldValue("hello hello"), 10);
+        strWset.add(StringFieldValue("remove val 1"), 20);
+        doc->setValue("strwset", strWset);
+    }
+    CPPUNIT_ASSERT(doc->hasValue("strwset"));
+    //doc->print(std::cerr, true, "");
+    DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo"));
+    docUp.addFieldPathUpdate(
+            FieldPathUpdate::CP(
+                    new RemoveFieldPathUpdate(*_repo, *doc->getDataType(),
+                            "strwset{remove val 1}")));
+    docUp.applyTo(*doc);
+    {
+        std::unique_ptr<WeightedSetFieldValue> strWset =
+            doc->getAs<WeightedSetFieldValue>(doc->getField("strwset"));
+        CPPUNIT_ASSERT_EQUAL(std::size_t(1), strWset->size());
+        CPPUNIT_ASSERT_EQUAL(10, strWset->get(StringFieldValue("hello hello")));
+    }
+}
+
+void
+FieldPathUpdateTestCase::testApplyAssignSingle()
+{
+    Document::UP doc(
+            new Document(_foobar_type, DocumentId("doc:drekka:karsk")));
+    CPPUNIT_ASSERT(doc->hasValue("strfoo") == false);
+    // Test assignment of non-existing
+    DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo"));
+    docUp.addFieldPathUpdate(
+            FieldPathUpdate::CP(
+                    new AssignFieldPathUpdate(*_repo, *doc->getDataType(),
+                            "strfoo", std::string(), StringFieldValue("himert"))));
+    docUp.applyTo(*doc);
+    CPPUNIT_ASSERT(doc->hasValue("strfoo"));
+    CPPUNIT_ASSERT_EQUAL(vespalib::string("himert"), doc->getValue("strfoo")->getAsString());
+    // Test overwriting existing
+    DocumentUpdate docUp2(_foobar_type, DocumentId("doc:barbar:foofoo"));
+    docUp2.addFieldPathUpdate(
+            FieldPathUpdate::CP(
+                    new AssignFieldPathUpdate(*_repo, *doc->getDataType(),
+                            "strfoo", std::string(), StringFieldValue("wunderbaum"))));
+    docUp2.applyTo(*doc);
+    CPPUNIT_ASSERT_EQUAL(vespalib::string("wunderbaum"), doc->getValue("strfoo")->getAsString());
+}
+
+void
+FieldPathUpdateTestCase::testApplyAssignMath()
+{
+    Document::UP doc(
+            new Document(_foobar_type, DocumentId("doc:bat:man")));
+    doc->setValue("num", IntFieldValue(34));
+
+    DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo"));
+    docUp.addFieldPathUpdate(
+            FieldPathUpdate::CP(
+                    new AssignFieldPathUpdate(*_repo, *doc->getDataType(),
+                            "num", "", "($value * 2) / $value")));
+    docUp.applyTo(*doc);
+    CPPUNIT_ASSERT_EQUAL(static_cast<const FieldValue&>(IntFieldValue(2)), *doc->getValue("num"));
+}
+
+void
+FieldPathUpdateTestCase::testApplyAssignMathByteToZero()
+{
+    Document::UP doc(
+            new Document(_foobar_type, DocumentId("doc:bat:man")));
+    doc->setValue("byteval", ByteFieldValue(3));
+
+    DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo"));
+    docUp.addFieldPathUpdate(
+            FieldPathUpdate::CP(
+                    new AssignFieldPathUpdate(*_repo, *doc->getDataType(),
+                            "byteval", "", "$value - 3")));
+    docUp.applyTo(*doc);
+    CPPUNIT_ASSERT_EQUAL(static_cast<const FieldValue&>(ByteFieldValue(0)), *doc->getValue("byteval"));
+}
+
+void
+FieldPathUpdateTestCase::testApplyAssignMathNotModifiedOnUnderflow()
+{
+    int low_value = -126;
+    Document::UP doc(
+            new Document(_foobar_type, DocumentId("doc:bat:man")));
+    doc->setValue("byteval", ByteFieldValue(low_value));
+
+    DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo"));
+    docUp.addFieldPathUpdate(
+            FieldPathUpdate::CP(
+                    new AssignFieldPathUpdate(*_repo, *doc->getDataType(),
+                            "byteval", "", "$value - 4")));
+    docUp.applyTo(*doc);
+    // Over/underflow will happen. You must have control of your data types.
+    CPPUNIT_ASSERT_EQUAL(static_cast<const FieldValue&>(ByteFieldValue((char)(low_value - 4))), *doc->getValue("byteval"));
+}
+
+void
+FieldPathUpdateTestCase::testApplyAssignMathNotModifiedOnOverflow()
+{
+    Document::UP doc(
+            new Document(_foobar_type, DocumentId("doc:bat:man")));
+    doc->setValue("byteval", ByteFieldValue(127));
+
+    DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo"));
+    docUp.addFieldPathUpdate(
+            FieldPathUpdate::CP(
+                    new AssignFieldPathUpdate(*_repo, *doc->getDataType(),
+                            "byteval", "", "$value + 200")));
+    docUp.applyTo(*doc);
+    // Over/underflow will happen. You must have control of your data types.
+    CPPUNIT_ASSERT_EQUAL(static_cast<const FieldValue&>(ByteFieldValue(static_cast<char>(static_cast<int>(127+200)))), *doc->getValue("byteval"));
+}
+
+void
+FieldPathUpdateTestCase::testApplyAssignMathDivZero()
+{
+    Document::UP doc(
+            new Document(_foobar_type, DocumentId("doc:bat:man")));
+    CPPUNIT_ASSERT(doc->hasValue("num") == false);
+    doc->setValue("num", IntFieldValue(10));
+
+    DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo"));
+    docUp.addFieldPathUpdate(
+            FieldPathUpdate::CP(
+                    new AssignFieldPathUpdate(*_repo, *doc->getDataType(),
+                            "num", "", "$value / ($value - 10)")));
+    docUp.applyTo(*doc);
+    CPPUNIT_ASSERT_EQUAL(static_cast<const FieldValue&>(IntFieldValue(10)), *doc->getValue("num"));
+}
+
+void
+FieldPathUpdateTestCase::testApplyAssignFieldNotExistingInExpression()
+{
+    Document::UP doc(
+            new Document(_foobar_type, DocumentId("doc:bat:man")));
+    CPPUNIT_ASSERT(doc->hasValue("num") == false);
+    doc->setValue("num", IntFieldValue(10));
+
+    DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo"));
+    docUp.addFieldPathUpdate(
+            FieldPathUpdate::CP(
+                    new AssignFieldPathUpdate(*_repo, *doc->getDataType(),
+                            "num", "", "foobar.num2 + $value")));
+    docUp.applyTo(*doc);
+    CPPUNIT_ASSERT_EQUAL(static_cast<const FieldValue&>(IntFieldValue(10)), *doc->getValue("num"));
+}
+
+void
+FieldPathUpdateTestCase::testApplyAssignFieldNotExistingInPath()
+{
+    Document::UP doc(
+            new Document(_foobar_type, DocumentId("doc:bat:man")));
+
+    DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo"));
+    try {
+        docUp.addFieldPathUpdate(
+                FieldPathUpdate::CP(
+                        new AssignFieldPathUpdate(*_repo, *doc->getDataType(),
+                                "nosuchnum", "", "foobar.num + $value")));
+        CPPUNIT_ASSERT(false);
+    } catch (const vespalib::IllegalArgumentException&) {
+    }
+}
+
+void
+FieldPathUpdateTestCase::testApplyAssignTargetNotExisting()
+{
+    Document::UP doc(
+            new Document(_foobar_type, DocumentId("doc:bat:man")));
+    CPPUNIT_ASSERT(doc->hasValue("num") == false);
+
+    DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo"));
+    docUp.addFieldPathUpdate(
+            FieldPathUpdate::CP(
+                    new AssignFieldPathUpdate(*_repo, *doc->getDataType(),
+                            "num", "", "$value + 5")));
+    docUp.applyTo(*doc);
+    CPPUNIT_ASSERT_EQUAL(static_cast<const FieldValue&>(IntFieldValue(5)), *doc->getValue("num"));
+}
+
+void
+FieldPathUpdateTestCase::testAssignSimpleMapValueWithVariable()
+{
+    Document::UP doc(
+            new Document(_foobar_type, DocumentId("doc:bug:hunter")));
+
+    MapFieldValue mfv(doc->getType().getField("strmap").getDataType());
+    mfv.put(StringFieldValue("foo"), StringFieldValue("bar"));
+    mfv.put(StringFieldValue("baz"), StringFieldValue("bananas"));
+    doc->setValue("strmap", mfv);
+
+    DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo"));
+    // Select on value, not key
+    docUp.addFieldPathUpdate(
+            FieldPathUpdate::CP(
+                    new AssignFieldPathUpdate(*_repo, *doc->getDataType(),
+                            "strmap{$x}", "foobar.strmap{$x} == \"bar\"", StringFieldValue("shinyvalue"))));
+    docUp.applyTo(*doc);
+
+    std::unique_ptr<MapFieldValue> valueNow(
+            doc->getAs<MapFieldValue>(doc->getField("strmap")));
+
+    CPPUNIT_ASSERT_EQUAL(std::size_t(2), valueNow->size());
+    CPPUNIT_ASSERT_EQUAL(
+            static_cast<const FieldValue&>(StringFieldValue("shinyvalue")),
+            *valueNow->get(StringFieldValue("foo")));
+    CPPUNIT_ASSERT_EQUAL(
+            static_cast<const FieldValue&>(StringFieldValue("bananas")),
+            *valueNow->get(StringFieldValue("baz")));
+}
+
+void
+FieldPathUpdateTestCase::testApplyAssignMathRemoveIfZero()
+{
+    Document::UP doc(
+            new Document(_foobar_type, DocumentId("doc:bat:man")));
+    CPPUNIT_ASSERT(doc->hasValue("num") == false);
+    doc->setValue("num", IntFieldValue(34));
+    CPPUNIT_ASSERT(doc->hasValue("num") == true);
+
+    DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo"));
+    FieldPathUpdate::CP up1(new AssignFieldPathUpdate(*_repo, *doc->getDataType(),
+                                    "num", "", "($value * 2) / $value - 2"));
+    static_cast<AssignFieldPathUpdate&>(*up1).setRemoveIfZero(true);
+    docUp.addFieldPathUpdate(up1);
+
+    docUp.applyTo(*doc);
+    CPPUNIT_ASSERT(doc->hasValue("num") == false);
+}
+
+void
+FieldPathUpdateTestCase::testApplyAssignMultiList()
+{
+    Document::UP doc(
+            new Document(_foobar_type, DocumentId("doc:fest:skinnvest")));
+    CPPUNIT_ASSERT(doc->hasValue("strarray") == false);
+
+    {
+        ArrayFieldValue
+            strArray(doc->getType().getField("strarray").getDataType());
+        strArray.add(StringFieldValue("hello hello"));
+        strArray.add(StringFieldValue("blah blargh"));
+        doc->setValue("strarray", strArray);
+        CPPUNIT_ASSERT(doc->hasValue("strarray"));
+    }
+
+    ArrayFieldValue
+        updateArray(doc->getType().getField("strarray").getDataType());
+    updateArray.add(StringFieldValue("assigned val 0"));
+    updateArray.add(StringFieldValue("assigned val 1"));
+
+    DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo"));
+    docUp.addFieldPathUpdate(
+            FieldPathUpdate::CP(
+                    new AssignFieldPathUpdate(*_repo, *doc->getDataType(),
+                            "strarray", std::string(), updateArray)));
+    docUp.applyTo(*doc);
+    {
+        std::unique_ptr<ArrayFieldValue> strArray =
+            doc->getAs<ArrayFieldValue>(doc->getField("strarray"));
+        CPPUNIT_ASSERT_EQUAL(std::size_t(2), strArray->size());
+        CPPUNIT_ASSERT_EQUAL(vespalib::string("assigned val 0"), (*strArray)[0].getAsString());
+        CPPUNIT_ASSERT_EQUAL(vespalib::string("assigned val 1"), (*strArray)[1].getAsString());
+    }
+}
+
+
+void
+FieldPathUpdateTestCase::testApplyAssignMultiWset()
+{
+    Document::UP doc(
+            new Document(_foobar_type, DocumentId("doc:fest:skinnvest")));
+    CPPUNIT_ASSERT(doc->hasValue("strarray") == false);
+
+    {
+        WeightedSetFieldValue
+            strWset(doc->getType().getField("strwset").getDataType());
+        strWset.add(StringFieldValue("hello gentlemen"), 10);
+        strWset.add(StringFieldValue("what you say"), 20);
+        doc->setValue("strwset", strWset);
+        CPPUNIT_ASSERT(doc->hasValue("strwset"));
+    }
+
+    WeightedSetFieldValue
+        assignWset(doc->getType().getField("strwset").getDataType());
+    assignWset.add(StringFieldValue("assigned val 0"), 5);
+    assignWset.add(StringFieldValue("assigned val 1"), 10);
+
+    DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo"));
+    docUp.addFieldPathUpdate(
+            FieldPathUpdate::CP(
+                    new AssignFieldPathUpdate(*_repo, *doc->getDataType(),
+                            "strwset", std::string(), assignWset)));
+    //doc->print(std::cerr, true, "");
+    docUp.applyTo(*doc);
+    //doc->print(std::cerr, true, "");
+    {
+        std::unique_ptr<WeightedSetFieldValue> strWset =
+            doc->getAs<WeightedSetFieldValue>(doc->getField("strwset"));
+        CPPUNIT_ASSERT_EQUAL(std::size_t(2), strWset->size());
+        CPPUNIT_ASSERT_EQUAL(5, strWset->get(StringFieldValue("assigned val 0")));
+        CPPUNIT_ASSERT_EQUAL(10, strWset->get(StringFieldValue("assigned val 1")));
+    }
+}
+
+void
+FieldPathUpdateTestCase::testAssignWsetRemoveIfZero()
+{
+    Document::UP doc(
+            new Document(_foobar_type, DocumentId("doc:tronder:bataljon")));
+    CPPUNIT_ASSERT(doc->hasValue("strarray") == false);
+
+    {
+        WeightedSetFieldValue
+            strWset(doc->getType().getField("strwset").getDataType());
+        strWset.add(StringFieldValue("you say goodbye"), 164);
+        strWset.add(StringFieldValue("but i say hello"), 243);
+        doc->setValue("strwset", strWset);
+        CPPUNIT_ASSERT(doc->hasValue("strwset"));
+    }
+
+    {
+        DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo"));
+        IntFieldValue zeroWeight(0);
+        FieldPathUpdate::CP assignUpdate(
+                new AssignFieldPathUpdate(*_repo, *doc->getDataType(),
+                        "strwset{you say goodbye}", std::string(), zeroWeight));
+        static_cast<AssignFieldPathUpdate&>(*assignUpdate).setRemoveIfZero(true);
+        docUp.addFieldPathUpdate(assignUpdate);
+        //doc->print(std::cerr, true, "");
+        docUp.applyTo(*doc);
+        //doc->print(std::cerr, true, "");
+        {
+            std::unique_ptr<WeightedSetFieldValue> strWset =
+                doc->getAs<WeightedSetFieldValue>(doc->getField("strwset"));
+            CPPUNIT_ASSERT_EQUAL(std::size_t(1), strWset->size());
+            CPPUNIT_ASSERT_EQUAL(243, strWset->get(StringFieldValue("but i say hello")));
+        }
+    }
+}
+
+void
+FieldPathUpdateTestCase::testApplyAddMultiList()
+{
+   Document::UP doc(
+            new Document(_foobar_type, DocumentId("doc:george:costanza")));
+    CPPUNIT_ASSERT(doc->hasValue("strarray") == false);
+
+    ArrayFieldValue adds(doc->getType().getField("strarray").getDataType());
+    adds.add(StringFieldValue("serenity now"));
+    adds.add(StringFieldValue("a festivus for the rest of us"));
+    adds.add(StringFieldValue("george is getting upset!"));
+
+    DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo"));
+    docUp.addFieldPathUpdate(
+            FieldPathUpdate::CP(
+                    new AddFieldPathUpdate(*_repo, *doc->getDataType(),
+                            "strarray", std::string(), adds)));
+    //doc->print(std::cerr, true, "");
+    docUp.applyTo(*doc);
+    //doc->print(std::cerr, true, "");
+    CPPUNIT_ASSERT(doc->hasValue("strarray"));
+}
+
+void
+FieldPathUpdateTestCase::testAddAndAssignList()
+{
+   Document::UP doc(
+            new Document(_foobar_type, DocumentId("doc:fancy:pants")));
+    CPPUNIT_ASSERT(doc->hasValue("strarray") == false);
+
+    {
+        ArrayFieldValue
+            strArray(doc->getType().getField("strarray").getDataType());
+        strArray.add(StringFieldValue("hello hello"));
+        strArray.add(StringFieldValue("blah blargh"));
+        doc->setValue("strarray", strArray);
+        CPPUNIT_ASSERT(doc->hasValue("strarray"));
+    }
+
+    DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo"));
+    docUp.addFieldPathUpdate(
+            FieldPathUpdate::CP(
+                    new AssignFieldPathUpdate(*_repo, *doc->getDataType(),
+                            "strarray[1]", std::string(), StringFieldValue("assigned val 1"))));
+
+    ArrayFieldValue adds(doc->getType().getField("strarray").getDataType());
+    adds.add(StringFieldValue("new value"));
+
+    docUp.addFieldPathUpdate(
+            FieldPathUpdate::CP(
+                    new AddFieldPathUpdate(*_repo, *doc->getDataType(),
+                            "strarray", std::string(), adds)));
+    //doc->print(std::cerr, true, "");
+    docUp.applyTo(*doc);
+    //doc->print(std::cerr, true, "");
+    {
+        std::unique_ptr<ArrayFieldValue> strArray =
+            doc->getAs<ArrayFieldValue>(doc->getField("strarray"));
+        CPPUNIT_ASSERT_EQUAL(std::size_t(3), strArray->size());
+        CPPUNIT_ASSERT_EQUAL(vespalib::string("hello hello"), (*strArray)[0].getAsString());
+        CPPUNIT_ASSERT_EQUAL(vespalib::string("assigned val 1"), (*strArray)[1].getAsString());
+        CPPUNIT_ASSERT_EQUAL(vespalib::string("new value"), (*strArray)[2].getAsString());
+    }
+}
+
+namespace {
+struct Keys {
+    vespalib::string key1;
+    vespalib::string key2;
+    vespalib::string key3;
+    Keys() : key1("foo"), key2("bar"), key3("zoo") {}
+};
+
+struct Fixture {
+    Document::UP doc;
+    MapFieldValue mfv;
+    StructFieldValue fv1, fv2, fv3, fv4;
+
+    const MapDataType &getMapType(const DocumentType &doc_type) {
+        return static_cast<const MapDataType &>(
+                doc_type.getField("structmap").getDataType());
+    }
+
+    Fixture(const DocumentType &doc_type, const Keys &k)
+        : doc(new Document(doc_type, DocumentId("doc:planet:express"))),
+          mfv(getMapType(doc_type)),
+          fv1(getMapType(doc_type).getValueType()),
+          fv2(getMapType(doc_type).getValueType()),
+          fv3(getMapType(doc_type).getValueType()),
+          fv4(getMapType(doc_type).getValueType()) {
+
+        fv1.setValue("title", StringFieldValue("fry"));
+        fv1.setValue("rating", IntFieldValue(30));
+        mfv.put(StringFieldValue(k.key1), fv1);
+
+        fv2.setValue("title", StringFieldValue("farnsworth"));
+        fv2.setValue("rating", IntFieldValue(60));
+        mfv.put(StringFieldValue(k.key2), fv2);
+
+        fv3.setValue("title", StringFieldValue("zoidberg"));
+        fv3.setValue("rating", IntFieldValue(-20));
+        mfv.put(StringFieldValue(k.key3), fv3);
+
+        doc->setValue("structmap", mfv);
+
+        fv4.setValue("title", StringFieldValue("farnsworth"));
+        fv4.setValue("rating", IntFieldValue(48));
+    }
+};
+
+}  // namespace
+
+void
+FieldPathUpdateTestCase::testAssignMap()
+{
+    Keys k;
+    Fixture f(_foobar_type, k);
+
+    DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo"));
+    docUp.addFieldPathUpdate(
+            FieldPathUpdate::CP(
+                    new AssignFieldPathUpdate(*_repo, *f.doc->getDataType(),
+                            "structmap{" + k.key2 + "}", std::string(),
+                            f.fv4)));
+    docUp.applyTo(*f.doc);
+
+    std::unique_ptr<MapFieldValue> valueNow =
+        f.doc->getAs<MapFieldValue>(f.doc->getField("structmap"));
+    CPPUNIT_ASSERT_EQUAL(std::size_t(3), valueNow->size());
+    CPPUNIT_ASSERT_EQUAL(static_cast<FieldValue&>(f.fv1),
+                         *valueNow->get(StringFieldValue(k.key1)));
+    CPPUNIT_ASSERT_EQUAL(static_cast<FieldValue&>(f.fv4),
+                         *valueNow->get(StringFieldValue(k.key2)));
+    CPPUNIT_ASSERT_EQUAL(static_cast<FieldValue&>(f.fv3),
+                         *valueNow->get(StringFieldValue(k.key3)));
+}
+
+void
+FieldPathUpdateTestCase::testAssignMapStruct()
+{
+    Keys k;
+    Fixture f(_foobar_type, k);
+
+    DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo"));
+    docUp.addFieldPathUpdate(
+            FieldPathUpdate::CP(
+                    new AssignFieldPathUpdate(*_repo, *f.doc->getDataType(),
+                            "structmap{" + k.key2 + "}.rating", std::string(),
+                            IntFieldValue(48))));
+    docUp.applyTo(*f.doc);
+
+    std::unique_ptr<MapFieldValue> valueNow =
+        f.doc->getAs<MapFieldValue>(f.doc->getField("structmap"));
+    CPPUNIT_ASSERT_EQUAL(std::size_t(3), valueNow->size());
+    CPPUNIT_ASSERT_EQUAL(static_cast<FieldValue&>(f.fv1),
+                         *valueNow->get(StringFieldValue(k.key1)));
+    CPPUNIT_ASSERT_EQUAL(static_cast<FieldValue&>(f.fv4),
+                         *valueNow->get(StringFieldValue(k.key2)));
+    CPPUNIT_ASSERT_EQUAL(static_cast<FieldValue&>(f.fv3),
+                         *valueNow->get(StringFieldValue(k.key3)));
+}
+
+void
+FieldPathUpdateTestCase::testAssignMapStructVariable()
+{
+    Keys k;
+    Fixture f(_foobar_type, k);
+
+    DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo"));
+    docUp.addFieldPathUpdate(
+            FieldPathUpdate::CP(
+                    new AssignFieldPathUpdate(*_repo, *f.doc->getDataType(),
+                            "structmap{$x}.rating",
+                            "foobar.structmap{$x}.title == \"farnsworth\"",
+                            IntFieldValue(48))));
+    docUp.applyTo(*f.doc);
+
+    std::unique_ptr<MapFieldValue> valueNow =
+        f.doc->getAs<MapFieldValue>(f.doc->getField("structmap"));
+    CPPUNIT_ASSERT_EQUAL(std::size_t(3), valueNow->size());
+    CPPUNIT_ASSERT_EQUAL(static_cast<FieldValue&>(f.fv1),
+                         *valueNow->get(StringFieldValue(k.key1)));
+    CPPUNIT_ASSERT_EQUAL(static_cast<FieldValue&>(f.fv4),
+                         *valueNow->get(StringFieldValue(k.key2)));
+    CPPUNIT_ASSERT_EQUAL(static_cast<FieldValue&>(f.fv3),
+                         *valueNow->get(StringFieldValue(k.key3)));
+}
+
+void
+FieldPathUpdateTestCase::testAssignMapNoExist()
+{
+    Document::UP doc(
+            new Document(_foobar_type, DocumentId("doc:planet:express")));
+    MapFieldValue mfv(doc->getType().getField("structmap").getDataType());
+
+    StructFieldValue fv1(dynamic_cast<const MapDataType&>(*mfv.getDataType())
+                         .getValueType());
+    fv1.setValue("title", StringFieldValue("fry"));
+    fv1.setValue("rating", IntFieldValue(30));
+
+    DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo"));
+    docUp.addFieldPathUpdate(
+            FieldPathUpdate::CP(
+                    new AssignFieldPathUpdate(*_repo, *doc->getDataType(),
+                            "structmap{foo}", std::string(), fv1)));
+    //doc->print(std::cerr, true, "");
+    docUp.applyTo(*doc);
+    //doc->print(std::cerr, true, "");
+
+    std::unique_ptr<MapFieldValue> valueNow =
+        doc->getAs<MapFieldValue>(doc->getField("structmap"));
+    CPPUNIT_ASSERT_EQUAL(std::size_t(1), valueNow->size());
+    CPPUNIT_ASSERT_EQUAL(static_cast<FieldValue&>(fv1), *valueNow->get(StringFieldValue("foo")));
+}
+
+void
+FieldPathUpdateTestCase::testAssignMapNoExistNoCreate()
+{
+    Document::UP doc(
+            new Document(_foobar_type, DocumentId("doc:planet:express")));
+    MapFieldValue mfv(doc->getType().getField("structmap").getDataType());
+
+    StructFieldValue fv1(dynamic_cast<const MapDataType&>(*mfv.getDataType())
+                         .getValueType());
+    fv1.setValue("title", StringFieldValue("fry"));
+    fv1.setValue("rating", IntFieldValue(30));
+
+    DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo"));
+    FieldPathUpdate::CP assignUpdate(
+            new AssignFieldPathUpdate(*_repo, *doc->getDataType(),
+                                      "structmap{foo}", std::string(), fv1));
+    static_cast<AssignFieldPathUpdate&>(*assignUpdate).setCreateMissingPath(false);
+    docUp.addFieldPathUpdate(assignUpdate);
+
+    //doc->print(std::cerr, true, "");
+    docUp.applyTo(*doc);
+    //doc->print(std::cerr, true, "");
+
+    std::unique_ptr<MapFieldValue> valueNow =
+        doc->getAs<MapFieldValue>(doc->getField("structmap"));
+    CPPUNIT_ASSERT(valueNow.get() == 0);
+}
+
+void
+FieldPathUpdateTestCase::testQuotedStringKey()
+{
+    Keys k;
+    k.key2 = "here is a \"fancy\" 'map' :-} key :-{";
+    const char field_path[] =
+        "structmap{\"here is a \\\"fancy\\\" 'map' :-} key :-{\"}";
+    Fixture f(_foobar_type, k);
+
+    DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo"));
+    docUp.addFieldPathUpdate(
+            FieldPathUpdate::CP(
+                    new AssignFieldPathUpdate(*_repo, *f.doc->getDataType(),
+                            field_path, std::string(), f.fv4)));
+    docUp.applyTo(*f.doc);
+
+    std::unique_ptr<MapFieldValue> valueNow =
+        f.doc->getAs<MapFieldValue>(f.doc->getField("structmap"));
+    CPPUNIT_ASSERT_EQUAL(std::size_t(3), valueNow->size());
+    CPPUNIT_ASSERT_EQUAL(static_cast<FieldValue&>(f.fv1),
+                         *valueNow->get(StringFieldValue(k.key1)));
+    CPPUNIT_ASSERT_EQUAL(static_cast<FieldValue&>(f.fv4),
+                         *valueNow->get(StringFieldValue(k.key2)));
+    CPPUNIT_ASSERT_EQUAL(static_cast<FieldValue&>(f.fv3),
+                         *valueNow->get(StringFieldValue(k.key3)));
+}
+
+void
+FieldPathUpdateTestCase::testEqualityComparison()
+{
+    Document::UP doc(
+            new Document(_foobar_type, DocumentId("doc:foo:zoo")));
+    MapFieldValue mfv(doc->getType().getField("structmap").getDataType());
+
+    StructFieldValue fv4(dynamic_cast<const MapDataType&>(*mfv.getDataType())
+                         .getValueType());
+    fv4.setValue("title", StringFieldValue("tasty cake"));
+    fv4.setValue("rating", IntFieldValue(95));
+
+    {
+        DocumentUpdate docUp1(_foobar_type, DocumentId("doc:barbar:foofoo"));
+        DocumentUpdate docUp2(_foobar_type, DocumentId("doc:barbar:foofoo"));
+        CPPUNIT_ASSERT(docUp1 == docUp2);
+
+        FieldPathUpdate::CP assignUp1(new AssignFieldPathUpdate(*_repo, *doc->getDataType(),
+                                              "structmap{here be dragons}", std::string(), fv4));
+        docUp1.addFieldPathUpdate(assignUp1);
+        CPPUNIT_ASSERT(docUp1 != docUp2);
+        docUp2.addFieldPathUpdate(assignUp1);
+        CPPUNIT_ASSERT(docUp1 == docUp2);
+    }
+    {
+        DocumentUpdate docUp1(_foobar_type, DocumentId("doc:barbar:foofoo"));
+        DocumentUpdate docUp2(_foobar_type, DocumentId("doc:barbar:foofoo"));
+        // where-clause diff
+        FieldPathUpdate::CP assignUp1(new AssignFieldPathUpdate(*_repo, *doc->getDataType(),
+                                              "structmap{here be dragons}", std::string(), fv4));
+        FieldPathUpdate::CP assignUp2(new AssignFieldPathUpdate(*_repo, *doc->getDataType(),
+                                              "structmap{here be dragons}", "false", fv4));
+        docUp1.addFieldPathUpdate(assignUp1);
+        docUp2.addFieldPathUpdate(assignUp2);
+        CPPUNIT_ASSERT(docUp1 != docUp2);
+    }
+    {
+        DocumentUpdate docUp1(_foobar_type, DocumentId("doc:barbar:foofoo"));
+        DocumentUpdate docUp2(_foobar_type, DocumentId("doc:barbar:foofoo"));
+        // fieldpath diff
+        FieldPathUpdate::CP assignUp1(new AssignFieldPathUpdate(*_repo, *doc->getDataType(),
+                                              "structmap{here be dragons}", std::string(), fv4));
+        FieldPathUpdate::CP assignUp2(new AssignFieldPathUpdate(*_repo, *doc->getDataType(),
+                                              "structmap{here be kittens}", std::string(), fv4));
+        docUp1.addFieldPathUpdate(assignUp1);
+        docUp2.addFieldPathUpdate(assignUp2);
+        CPPUNIT_ASSERT(docUp1 != docUp2);
+    }
+
+}
+
+void
+FieldPathUpdateTestCase::testAffectsDocumentBody()
+{
+    Document::UP doc(
+            new Document(_foobar_type, DocumentId("doc:things:stuff")));
+    MapFieldValue mfv(doc->getType().getField("structmap").getDataType());
+
+    StructFieldValue fv4(dynamic_cast<const MapDataType&>(*mfv.getDataType())
+                         .getValueType());
+    fv4.setValue("title", StringFieldValue("scruffy"));
+    fv4.setValue("rating", IntFieldValue(90));
+
+    // structmap is body field
+    {
+        DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo"));
+        CPPUNIT_ASSERT(!docUp.affectsDocumentBody());
+
+        FieldPathUpdate::CP update1(new AssignFieldPathUpdate(*_repo, *doc->getDataType(),
+                                            "structmap{janitor}", std::string(), fv4));
+        static_cast<AssignFieldPathUpdate&>(*update1).setCreateMissingPath(true);
+        docUp.addFieldPathUpdate(update1);
+        CPPUNIT_ASSERT(docUp.affectsDocumentBody());
+    }
+
+    // strfoo is header field
+    {
+        DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo"));
+        FieldPathUpdate::CP update1(new AssignFieldPathUpdate(*_repo, *doc->getDataType(),
+                                            "strfoo", std::string(), StringFieldValue("helloworld")));
+        static_cast<AssignFieldPathUpdate&>(*update1).setCreateMissingPath(true);
+        docUp.addFieldPathUpdate(update1);
+        CPPUNIT_ASSERT(!docUp.affectsDocumentBody());
+    }
+
+}
+
+void
+FieldPathUpdateTestCase::testIncompatibleDataTypeFails()
+{
+    Document::UP doc(
+            new Document(_foobar_type, DocumentId("doc:things:stuff")));
+    MapFieldValue mfv(doc->getType().getField("structmap").getDataType());
+
+    DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo"));
+
+    try {
+        FieldPathUpdate::CP update1(
+                new AssignFieldPathUpdate(*_repo, *doc->getDataType(),
+                        "structmap{foo}", std::string(),
+                        StringFieldValue("bad things")));
+        CPPUNIT_ASSERT(false);
+    } catch (const vespalib::IllegalArgumentException& e) {
+        // OK
+    }
+}
+
+void
+FieldPathUpdateTestCase::testSerializeAssign()
+{
+    Document::UP doc(
+            new Document(_foobar_type,
+                         DocumentId("doc:weloveto:serializestuff")));
+    MapFieldValue mfv(doc->getType().getField("structmap").getDataType());
+
+    StructFieldValue val(dynamic_cast<const MapDataType&>(*mfv.getDataType())
+                         .getValueType());
+    val.setValue("title", StringFieldValue("cool frog"));
+    val.setValue("rating", IntFieldValue(100));
+
+    DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo"));
+    CPPUNIT_ASSERT(!docUp.affectsDocumentBody());
+
+    FieldPathUpdate::CP update1(new AssignFieldPathUpdate(*_repo, *doc->getDataType(),
+                                        "structmap{ribbit}", "true", val));
+    static_cast<AssignFieldPathUpdate&>(*update1).setCreateMissingPath(true);
+    docUp.addFieldPathUpdate(update1);
+
+    testSerialize(*_repo, docUp);
+
+}
+
+void
+FieldPathUpdateTestCase::testSerializeAdd()
+{
+    Document::UP doc(
+            new Document(_foobar_type, DocumentId("doc:george:costanza")));
+    CPPUNIT_ASSERT(doc->hasValue("strarray") == false);
+
+    ArrayFieldValue adds(doc->getType().getField("strarray").getDataType());
+    adds.add(StringFieldValue("serenity now"));
+    adds.add(StringFieldValue("a festivus for the rest of us"));
+    adds.add(StringFieldValue("george is getting upset!"));
+
+    DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo"));
+    CPPUNIT_ASSERT(!docUp.affectsDocumentBody());
+
+    FieldPathUpdate::CP update1(new AddFieldPathUpdate(*_repo, *doc->getDataType(),
+                                        "strarray", std::string(), adds));
+    docUp.addFieldPathUpdate(update1);
+
+    testSerialize(*_repo, docUp);
+}
+
+void
+FieldPathUpdateTestCase::testSerializeRemove()
+{
+    Document::UP doc(
+            new Document(_foobar_type,
+                         DocumentId("doc:weloveto:serializestuff")));
+    MapFieldValue mfv(doc->getType().getField("structmap").getDataType());
+
+    DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo"));
+    CPPUNIT_ASSERT(!docUp.affectsDocumentBody());
+
+    FieldPathUpdate::CP update1(new RemoveFieldPathUpdate(*_repo, *doc->getDataType(),
+                                        "structmap{ribbit}", std::string()));
+    docUp.addFieldPathUpdate(update1);
+
+    testSerialize(*_repo, docUp);
+}
+
+void
+FieldPathUpdateTestCase::testSerializeAssignMath()
+{
+    Document::UP doc(
+            new Document(_foobar_type, DocumentId("doc:bat:man")));
+    CPPUNIT_ASSERT(doc->hasValue("num") == false);
+    doc->setValue("num", IntFieldValue(34));
+
+    DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo"));
+    docUp.addFieldPathUpdate(
+            FieldPathUpdate::CP(
+                    new AssignFieldPathUpdate(*_repo, *doc->getDataType(),
+                            "num", "", "($value * 2) / $value")));
+    testSerialize(*_repo, docUp);
+}
+
+DocumentUpdate::UP
+FieldPathUpdateTestCase::createDocumentUpdateForSerialization(const DocumentTypeRepo& repo)
+{
+    const DocumentType *docType(repo.getDocumentType("serializetest"));
+    DocumentUpdate::UP docUp(new DocumentUpdate(
+                    *docType, DocumentId("doc:serialization:xlanguage")));
+
+    FieldPathUpdate::CP
+        assign(new AssignFieldPathUpdate(repo, *docType, "intfield", "", "3"));
+    static_cast<AssignFieldPathUpdate&>(*assign).setRemoveIfZero(true);
+    static_cast<AssignFieldPathUpdate&>(*assign).setCreateMissingPath(false);
+    docUp->addFieldPathUpdate(assign);
+
+    ArrayFieldValue
+        fArray(docType->getField("arrayoffloatfield").getDataType());
+    fArray.add(FloatFieldValue(12.0));
+    fArray.add(FloatFieldValue(5.0));
+
+    docUp->addFieldPathUpdate(
+            FieldPathUpdate::CP(
+                    new AddFieldPathUpdate(repo, *docType, "arrayoffloatfield", "", fArray)));
+
+    docUp->addFieldPathUpdate(
+            FieldPathUpdate::CP(
+                    new RemoveFieldPathUpdate(repo, *docType, "intfield", "serializetest.intfield > 0")));
+
+    return docUp;
+}
+
+void
+FieldPathUpdateTestCase::testReadSerializedFile()
+{
+    // Reads a file serialized from java
+    const char cfg_file_name[] = "data/crossplatform-java-cpp-doctypes.cfg";
+    DocumentTypeRepo repo(readDocumenttypesConfig(cfg_file_name));
+
+    int fd = open("data/serialize-fieldpathupdate-java.dat", O_RDONLY);
+
+    int len = lseek(fd,0,SEEK_END);
+    ByteBuffer buf(len);
+    lseek(fd,0,SEEK_SET);
+    read(fd, buf.getBuffer(), len);
+
+    close(fd);
+
+    DocumentUpdate::UP updp(DocumentUpdate::createHEAD(repo, buf));
+    DocumentUpdate& upd(*updp);
+
+    DocumentUpdate::UP compare(
+            createDocumentUpdateForSerialization(repo));
+    CPPUNIT_ASSERT_EQUAL(*compare, upd);
+}
+
+void
+FieldPathUpdateTestCase::testGenerateSerializedFile()
+{
+    const char cfg_file_name[] = "data/crossplatform-java-cpp-doctypes.cfg";
+    DocumentTypeRepo repo(readDocumenttypesConfig(cfg_file_name));
+    // Tests nothing, only generates a file for java test
+    DocumentUpdate::UP upd(
+            createDocumentUpdateForSerialization(repo));
+
+    ByteBuffer::UP buf(serializeHEAD(*upd));
+
+    int fd = open("data/serialize-fieldpathupdate-cpp.dat",
+                  O_WRONLY | O_TRUNC | O_CREAT, 0644);
+    write(fd, buf->getBuffer(), buf->getPos());
+    close(fd);
+}
+
+} // ns document
diff --git a/document/src/tests/fieldsettest.cpp b/document/src/tests/fieldsettest.cpp
new file mode 100644
index 00000000000..41a530c4b17
--- /dev/null
+++ b/document/src/tests/fieldsettest.cpp
@@ -0,0 +1,348 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/document/base/testdocman.h>
+#include <vespa/document/fieldset/fieldsetrepo.h>
+#include <vespa/document/fieldvalue/fieldvalues.h>
+#include <vespa/vespalib/io/fileutil.h>
+#include <vespa/vdstestlib/cppunit/macros.h>
+
+#include <vespa/document/annotation/spantree.h>
+#include <vespa/document/config/config-documenttypes.h>
+#include <vespa/document/datatype/annotationreferencedatatype.h>
+#include <vespa/document/repo/configbuilder.h>
+#include <vespa/document/repo/documenttyperepo.h>
+#include <vespa/vespalib/objects/nbostream.h>
+#include <fstream>
+
+using vespalib::nbostream;
+
+using namespace document::config_builder;
+
+namespace document {
+
+struct FieldSetTest : public CppUnit::TestFixture {
+    void testParsing();
+    void testContains();
+    void testCopyDocumentFields();
+    void testDocumentSubsetCopy();
+    void testStripFields();
+    void testSerialize();
+
+    CPPUNIT_TEST_SUITE(FieldSetTest);
+    CPPUNIT_TEST(testParsing);
+    CPPUNIT_TEST(testSerialize);
+    CPPUNIT_TEST(testContains);
+    CPPUNIT_TEST(testCopyDocumentFields);
+    CPPUNIT_TEST(testDocumentSubsetCopy);
+    CPPUNIT_TEST(testStripFields);
+    CPPUNIT_TEST_SUITE_END();
+
+    std::string stringifyFields(const Document& doc) const;
+    std::string doCopyFields(const Document& src,
+                             const DocumentTypeRepo& docRepo,
+                             const std::string& fieldSetStr,
+                             Document* dest = 0) const;
+    std::string doCopyDocument(const Document& src,
+                               const DocumentTypeRepo& docRepo,
+                               const std::string& fieldSetStr);
+    std::string doStripFields(const Document& doc,
+                              const DocumentTypeRepo& docRepo,
+                              const std::string& fieldSetStr);
+    Document::UP createTestDocument(const TestDocMan& testDocMan) const;
+};
+
+CPPUNIT_TEST_SUITE_REGISTRATION(FieldSetTest);
+
+void FieldSetTest::testParsing()
+{
+    TestDocMan testDocMan;
+    const DocumentTypeRepo& docRepo = testDocMan.getTypeRepo();
+
+    FieldSetRepo repo;
+
+    dynamic_cast<AllFields&>(*repo.parse(docRepo, "[all]"));
+    dynamic_cast<NoFields&>(*repo.parse(docRepo, "[none]"));
+    dynamic_cast<DocIdOnly&>(*repo.parse(docRepo, "[id]"));
+    dynamic_cast<HeaderFields&>(*repo.parse(docRepo, "[header]"));
+    dynamic_cast<BodyFields&>(*repo.parse(docRepo, "[body]"));
+
+    FieldSet::UP set = repo.parse(docRepo, "testdoctype1:headerval,content");
+    FieldCollection& coll = dynamic_cast<FieldCollection&>(*set);
+
+    std::ostringstream ost;
+    for (Field::Set::const_iterator iter = coll.getFields().begin();
+         iter != coll.getFields().end();
+         ++iter) {
+        ost << (*iter)->getName() << " ";
+    }
+
+    CPPUNIT_ASSERT_EQUAL(std::string("content headerval "), ost.str());
+}
+
+namespace {
+
+bool checkContains(FieldSetRepo& r, const DocumentTypeRepo& repo,
+                   const std::string& str1, const std::string str2) {
+    FieldSet::UP set1 = r.parse(repo, str1);
+    FieldSet::UP set2 = r.parse(repo, str2);
+
+    return set1->contains(*set2);
+}
+
+bool checkError(FieldSetRepo& r, const DocumentTypeRepo& repo,
+                const std::string& str) {
+    try {
+        r.parse(repo, str);
+        return false;
+    } catch (...) {
+        return true;
+    }
+}
+
+}
+
+void FieldSetTest::testContains()
+{
+    TestDocMan testDocMan;
+    const DocumentTypeRepo& repo = testDocMan.getTypeRepo();
+    const DocumentType& type = *repo.getDocumentType("testdoctype1");
+
+    const Field& headerField = type.getField("headerval");
+    const Field& bodyField = type.getField("content");
+
+    NoFields none;
+    AllFields all;
+    DocIdOnly id;
+    HeaderFields h;
+    BodyFields b;
+
+    CPPUNIT_ASSERT_EQUAL(false, headerField.contains(
+                                 type.getField("headerlongval")));
+    CPPUNIT_ASSERT_EQUAL(true, headerField.contains(headerField));
+    CPPUNIT_ASSERT_EQUAL(true, headerField.contains(id));
+    CPPUNIT_ASSERT_EQUAL(false, headerField.contains(all));
+    CPPUNIT_ASSERT_EQUAL(true, headerField.contains(none));
+    CPPUNIT_ASSERT_EQUAL(false, none.contains(headerField));
+    CPPUNIT_ASSERT_EQUAL(true, all.contains(headerField));
+    CPPUNIT_ASSERT_EQUAL(true, all.contains(none));
+    CPPUNIT_ASSERT_EQUAL(false, none.contains(all));
+    CPPUNIT_ASSERT_EQUAL(true, all.contains(id));
+    CPPUNIT_ASSERT_EQUAL(false, none.contains(id));
+    CPPUNIT_ASSERT_EQUAL(true, id.contains(none));
+
+    CPPUNIT_ASSERT_EQUAL(true, h.contains(headerField));
+    CPPUNIT_ASSERT_EQUAL(false, h.contains(bodyField));
+
+    CPPUNIT_ASSERT_EQUAL(false, b.contains(headerField));
+    CPPUNIT_ASSERT_EQUAL(true, b.contains(bodyField));
+
+    FieldSetRepo r;
+    CPPUNIT_ASSERT_EQUAL(true, checkContains(r, repo, "[body]",
+                                             "testdoctype1:content"));
+    CPPUNIT_ASSERT_EQUAL(false, checkContains(r, repo, "[header]",
+                                              "testdoctype1:content"));
+    CPPUNIT_ASSERT_EQUAL(true, checkContains(r, repo,
+                                             "testdoctype1:content,headerval",
+                                             "testdoctype1:content"));
+    CPPUNIT_ASSERT_EQUAL(false, checkContains(r, repo,
+                                              "testdoctype1:content",
+                                              "testdoctype1:content,headerval"));
+    CPPUNIT_ASSERT_EQUAL(true, checkContains(r, repo,
+                                             "testdoctype1:headerval,content",
+                                             "testdoctype1:content,headerval"));
+
+    CPPUNIT_ASSERT(checkError(r, repo, "nodoctype"));
+    CPPUNIT_ASSERT(checkError(r, repo, "unknowndoctype:foo"));
+    CPPUNIT_ASSERT(checkError(r, repo, "testdoctype1:unknownfield"));
+    CPPUNIT_ASSERT(checkError(r, repo, "[badid]"));
+}
+
+std::string
+FieldSetTest::stringifyFields(const Document& doc) const
+{
+    std::vector<std::string> output;
+    const StructFieldValue& fields(doc.getFields());
+    for (StructFieldValue::const_iterator
+             it(fields.begin()), e(fields.end());
+         it != e; ++it)
+    {
+        std::ostringstream ss;
+        const Field& f(it.field());
+        ss << f.getName() << ": ";
+        FieldValue::UP val(fields.getValue(f));
+        if (val.get()) {
+            ss << val->toString();
+        } else {
+            ss << "(null)";
+        }
+        output.push_back(ss.str());
+    }
+    std::ostringstream ret;
+    std::sort(output.begin(), output.end());
+    std::copy(output.begin(), output.end(),
+              std::ostream_iterator<std::string>(ret, "\n"));
+    return ret.str();
+}
+
+std::string
+FieldSetTest::doCopyFields(const Document& src,
+                           const DocumentTypeRepo& docRepo,
+                           const std::string& fieldSetStr,
+                           Document* dest) const
+{
+    Document destDoc(src.getType(), DocumentId("doc:test:fieldsdest"));
+    if (!dest) {
+        dest = &destDoc;
+    }
+    FieldSetRepo repo;
+    FieldSet::UP fset = repo.parse(docRepo, fieldSetStr);
+    FieldSet::copyFields(*dest, src, *fset);
+    return stringifyFields(*dest);
+}
+
+std::string
+FieldSetTest::doStripFields(const Document& doc,
+                            const DocumentTypeRepo& docRepo,
+                            const std::string& fieldSetStr)
+{
+    Document::UP copy(doc.clone());
+    FieldSetRepo repo;
+    FieldSet::UP fset = repo.parse(docRepo, fieldSetStr);
+    FieldSet::stripFields(*copy, *fset);
+    return stringifyFields(*copy);
+}
+
+Document::UP
+FieldSetTest::createTestDocument(const TestDocMan& testDocMan) const
+{
+    Document::UP doc(testDocMan.createDocument("megafoo megabar",
+                                               "doc:test:fieldssrc",
+                                               "testdoctype1"));
+    doc->setValue(doc->getField("headerval"), IntFieldValue(5678));
+    doc->setValue(doc->getField("hstringval"),
+                  StringFieldValue("hello fantastic world"));
+    return doc;
+}
+
+void
+FieldSetTest::testCopyDocumentFields()
+{
+    TestDocMan testDocMan;
+    const DocumentTypeRepo& repo = testDocMan.getTypeRepo();
+    Document::UP src(createTestDocument(testDocMan));
+
+    CPPUNIT_ASSERT_EQUAL(std::string("content: megafoo megabar\n"),
+                         doCopyFields(*src, repo, "[body]"));
+    CPPUNIT_ASSERT_EQUAL(std::string(""),
+                         doCopyFields(*src, repo, "[none]"));
+    CPPUNIT_ASSERT_EQUAL(std::string("headerval: 5678\n"
+                                     "hstringval: hello fantastic world\n"),
+                         doCopyFields(*src, repo, "[header]"));
+    CPPUNIT_ASSERT_EQUAL(std::string("content: megafoo megabar\n"
+                                     "headerval: 5678\n"
+                                     "hstringval: hello fantastic world\n"),
+                         doCopyFields(*src, repo, "[all]"));
+    CPPUNIT_ASSERT_EQUAL(std::string("content: megafoo megabar\n"
+                                     "hstringval: hello fantastic world\n"),
+                         doCopyFields(*src, repo, "testdoctype1:hstringval,content"));
+    // Test that we overwrite already set fields in destination document
+    {
+        Document dest(src->getType(), DocumentId("doc:foo:bar"));
+        dest.setValue(dest.getField("content"), StringFieldValue("overwriteme"));
+        CPPUNIT_ASSERT_EQUAL(std::string("content: megafoo megabar\n"),
+                         doCopyFields(*src, repo, "[body]", &dest));
+    }
+}
+
+std::string
+FieldSetTest::doCopyDocument(const Document& src,
+                             const DocumentTypeRepo& docRepo,
+                             const std::string& fieldSetStr)
+{
+    FieldSetRepo repo;
+    FieldSet::UP fset = repo.parse(docRepo, fieldSetStr);
+    Document::UP doc(FieldSet::createDocumentSubsetCopy(src, *fset));
+    return stringifyFields(*doc);
+}
+
+
+void
+FieldSetTest::testDocumentSubsetCopy()
+{
+    TestDocMan testDocMan;
+    const DocumentTypeRepo& repo = testDocMan.getTypeRepo();
+    Document::UP src(createTestDocument(testDocMan));
+
+    {
+        Document::UP doc(FieldSet::createDocumentSubsetCopy(*src, AllFields()));
+        // Test that document id and type are copied correctly.
+        CPPUNIT_ASSERT(doc.get());
+        CPPUNIT_ASSERT_EQUAL(src->getId(), doc->getId());
+        CPPUNIT_ASSERT_EQUAL(src->getType(), doc->getType());
+        CPPUNIT_ASSERT_EQUAL(doCopyFields(*src, repo, "[all]"),
+                             stringifyFields(*doc));
+    }
+
+    const char* fieldSets[] = {
+        "[all]",
+        "[none]",
+        "[header]",
+        "[body]",
+        "testdoctype1:hstringval,content"
+    };
+    for (size_t i = 0; i < sizeof(fieldSets) / sizeof(fieldSets[0]); ++i) {
+        CPPUNIT_ASSERT_EQUAL(doCopyFields(*src, repo, fieldSets[i]),
+                             doCopyDocument(*src, repo, fieldSets[i]));
+    }
+}
+
+void
+FieldSetTest::testSerialize()
+{
+    TestDocMan testDocMan;
+    const DocumentTypeRepo& docRepo = testDocMan.getTypeRepo();
+
+    const char* fieldSets[] = {
+        "[all]",
+        "[none]",
+        "[header]",
+        "[docid]",
+        "[body]",
+        "testdoctype1:content",
+        "testdoctype1:content,hstringval"
+    };
+
+    FieldSetRepo repo;
+    for (size_t i = 0; i < sizeof(fieldSets) / sizeof(fieldSets[0]); ++i) {
+        FieldSet::UP fs = repo.parse(docRepo, fieldSets[i]);
+        CPPUNIT_ASSERT_EQUAL(vespalib::string(fieldSets[i]), repo.serialize(*fs));
+    }
+}
+
+void
+FieldSetTest::testStripFields()
+{
+    TestDocMan testDocMan;
+    const DocumentTypeRepo& repo = testDocMan.getTypeRepo();
+    Document::UP src(createTestDocument(testDocMan));
+
+    CPPUNIT_ASSERT_EQUAL(std::string("content: megafoo megabar\n"),
+                         doStripFields(*src, repo, "[body]"));
+    CPPUNIT_ASSERT_EQUAL(std::string(""),
+                         doStripFields(*src, repo, "[none]"));
+    CPPUNIT_ASSERT_EQUAL(std::string(""),
+                         doStripFields(*src, repo, "[id]"));
+    CPPUNIT_ASSERT_EQUAL(std::string("headerval: 5678\n"
+                                     "hstringval: hello fantastic world\n"),
+                         doStripFields(*src, repo, "[header]"));
+    CPPUNIT_ASSERT_EQUAL(std::string("content: megafoo megabar\n"
+                                     "headerval: 5678\n"
+                                     "hstringval: hello fantastic world\n"),
+                         doStripFields(*src, repo, "[all]"));
+    CPPUNIT_ASSERT_EQUAL(std::string("content: megafoo megabar\n"
+                                     "hstringval: hello fantastic world\n"),
+                         doStripFields(*src, repo, "testdoctype1:hstringval,content"));
+}
+
+} // document
diff --git a/document/src/tests/fieldtestcase.h b/document/src/tests/fieldtestcase.h
new file mode 100644
index 00000000000..0001b8fce1f
--- /dev/null
+++ b/document/src/tests/fieldtestcase.h
@@ -0,0 +1,20 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/* $Id$*/
+
+#pragma once
+
+#include <cppunit/extensions/HelperMacros.h>
+
+class Field_Test : public CppUnit::TestFixture {
+  CPPUNIT_TEST_SUITE( Field_Test);
+  CPPUNIT_TEST(testserialize);
+  CPPUNIT_TEST_SUITE_END();
+
+public:
+  void setUp();
+
+protected:
+  void testserialize();
+};
+
+
diff --git a/document/src/tests/fieldvalue/.gitignore b/document/src/tests/fieldvalue/.gitignore
new file mode 100644
index 00000000000..6b57b8c8d17
--- /dev/null
+++ b/document/src/tests/fieldvalue/.gitignore
@@ -0,0 +1,7 @@
+*.So
+*_test
+.depend
+Makefile
+document_document_test_app
+document_fieldvalue_test_app
+document_predicatefieldvalue_test_app
diff --git a/document/src/tests/fieldvalue/CMakeLists.txt b/document/src/tests/fieldvalue/CMakeLists.txt
new file mode 100644
index 00000000000..7b5fc637e4f
--- /dev/null
+++ b/document/src/tests/fieldvalue/CMakeLists.txt
@@ -0,0 +1,28 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(document_document_test_app
+    SOURCES
+    document_test.cpp
+    DEPENDS
+    document
+    AFTER
+    document_documentconfig
+)
+vespa_add_test(NAME document_document_test_app COMMAND document_document_test_app)
+vespa_add_executable(document_fieldvalue_test_app
+    SOURCES
+    fieldvalue_test.cpp
+    DEPENDS
+    document
+    AFTER
+    document_documentconfig
+)
+vespa_add_test(NAME document_fieldvalue_test_app COMMAND document_fieldvalue_test_app)
+vespa_add_executable(document_predicatefieldvalue_test_app
+    SOURCES
+    predicatefieldvalue_test.cpp
+    DEPENDS
+    document
+    AFTER
+    document_documentconfig
+)
+vespa_add_test(NAME document_predicatefieldvalue_test_app COMMAND document_predicatefieldvalue_test_app)
diff --git a/document/src/tests/fieldvalue/document_test.cpp b/document/src/tests/fieldvalue/document_test.cpp
new file mode 100644
index 00000000000..51216c6d77a
--- /dev/null
+++ b/document/src/tests/fieldvalue/document_test.cpp
@@ -0,0 +1,31 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Unit tests for document.
+
+#include <vespa/log/log.h>
+LOG_SETUP("document_test");
+#include <vespa/fastos/fastos.h>
+
+#include <vespa/document/base/documentid.h>
+#include <vespa/document/base/testdocrepo.h>
+#include <vespa/document/fieldvalue/document.h>
+#include <vespa/vespalib/testkit/testapp.h>
+
+using namespace document;
+
+namespace {
+
+TEST("require that document with id schema 'id' checks type") {
+    TestDocRepo repo;
+    const DataType *type = repo.getDocumentType("testdoctype1");
+    ASSERT_TRUE(type);
+
+    Document(*type, DocumentId("id:ns:testdoctype1::"));  // Should not throw
+
+    EXPECT_EXCEPTION(Document(*type, DocumentId("id:ns:type::")),
+                     vespalib::IllegalArgumentException,
+                     "testdoctype1 that don't match the id (type type)");
+}
+
+}  // namespace
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/document/src/tests/fieldvalue/fieldvalue_test.cpp b/document/src/tests/fieldvalue/fieldvalue_test.cpp
new file mode 100644
index 00000000000..7fecb42417e
--- /dev/null
+++ b/document/src/tests/fieldvalue/fieldvalue_test.cpp
@@ -0,0 +1,41 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Unit tests for fieldvalue.
+
+#include <vespa/log/log.h>
+LOG_SETUP("fieldvalue_test");
+#include <vespa/fastos/fastos.h>
+
+#include <vespa/document/fieldvalue/stringfieldvalue.h>
+#include <vespa/document/fieldvalue/longfieldvalue.h>
+#include <vespa/document/fieldvalue/intfieldvalue.h>
+
+#include <vespa/vespalib/testkit/testapp.h>
+
+using namespace document;
+
+namespace {
+
+TEST("require that StringFieldValue can be assigned primitives") {
+    StringFieldValue val;
+    val = "foo";
+    EXPECT_EQUAL("foo", val.getValue());
+    val = 1;
+    EXPECT_EQUAL("1", val.getValue());
+    val = static_cast<int64_t>(2);
+    EXPECT_EQUAL("2", val.getValue());
+    val = 3.0f;
+    EXPECT_EQUAL("3", val.getValue());
+    val = 4.0;
+    EXPECT_EQUAL("4", val.getValue());
+}
+
+TEST("require that FieldValues does not change their storage size.") {
+    EXPECT_EQUAL(8u, sizeof(FieldValue));
+    EXPECT_EQUAL(16u, sizeof(IntFieldValue));
+    EXPECT_EQUAL(24u, sizeof(LongFieldValue));
+    EXPECT_EQUAL(104u, sizeof(StringFieldValue));
+}
+
+}  // namespace
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/document/src/tests/fieldvalue/predicatefieldvalue_test.cpp b/document/src/tests/fieldvalue/predicatefieldvalue_test.cpp
new file mode 100644
index 00000000000..94b0f406696
--- /dev/null
+++ b/document/src/tests/fieldvalue/predicatefieldvalue_test.cpp
@@ -0,0 +1,70 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Unit tests for predicatefieldvalue.
+
+#include <vespa/log/log.h>
+LOG_SETUP("predicatefieldvalue_test");
+#include <vespa/fastos/fastos.h>
+
+#include <vespa/document/predicate/predicate.h>
+#include <vespa/vespalib/data/slime/slime.h>
+#include <vespa/vespalib/testkit/testapp.h>
+
+#include <vespa/document/datatype/datatype.h>
+#include <vespa/document/fieldvalue/predicatefieldvalue.h>
+#include <vespa/document/predicate/predicate_slime_builder.h>
+#include <sstream>
+#include <string>
+
+using std::ostringstream;
+using std::string;
+using vespalib::Slime;
+using namespace document;
+
+namespace {
+
+void verifyEqual(const FieldValue & a, const FieldValue & b) {
+    ostringstream o1;
+    a.print(o1, false, "");
+    ostringstream o2;
+    b.print(o2, false, "");
+    ASSERT_EQUAL(o1.str(), o2.str());
+}
+
+TEST("require that PredicateFieldValue can be cloned, assigned, and operator=") {
+    PredicateSlimeBuilder builder;
+    builder.neg().feature("foo").value("bar").value("baz");
+    PredicateFieldValue val(builder.build());
+
+    FieldValue::UP val2(val.clone());
+    verifyEqual(val, *val2);
+
+    PredicateFieldValue assigned;
+    assigned.assign(val);
+    verifyEqual(val, assigned);
+
+    PredicateFieldValue operatorAssigned;
+    operatorAssigned = val;
+    verifyEqual(val, operatorAssigned);
+}
+
+TEST("require that PredicateFieldValue can be created from datatype") {
+    FieldValue::UP val = DataType::PREDICATE->createFieldValue();
+    ASSERT_TRUE(dynamic_cast<PredicateFieldValue *>(val.get()));
+}
+
+TEST("require that PredicateFieldValue can be cloned") {
+    PredicateSlimeBuilder builder;
+    builder.neg().feature("foo").value("bar").value("baz");
+    PredicateFieldValue val(builder.build());
+    FieldValue::UP val2(val.clone());
+    ostringstream o1;
+    val.print(o1, false, "");
+    ostringstream o2;
+    val2->print(o2, false, "");
+    ASSERT_EQUAL(o1.str(), o2.str());
+}
+
+
+}  // namespace
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/document/src/tests/forcelinktest.cpp b/document/src/tests/forcelinktest.cpp
new file mode 100644
index 00000000000..c62f327585f
--- /dev/null
+++ b/document/src/tests/forcelinktest.cpp
@@ -0,0 +1,25 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/document/base/forcelink.h>
+#include <cppunit/extensions/HelperMacros.h>
+
+namespace document {
+
+struct ForceLinkTest : public CppUnit::TestFixture {
+  void testUsage();
+
+  CPPUNIT_TEST_SUITE(ForceLinkTest);
+  CPPUNIT_TEST(testUsage);
+  CPPUNIT_TEST_SUITE_END();
+
+};
+
+CPPUNIT_TEST_SUITE_REGISTRATION(ForceLinkTest);
+
+void ForceLinkTest::testUsage()
+{
+    ForceLink link;
+}
+
+} // document
diff --git a/document/src/tests/gid_filter_test.cpp b/document/src/tests/gid_filter_test.cpp
new file mode 100644
index 00000000000..9caf55f6a0b
--- /dev/null
+++ b/document/src/tests/gid_filter_test.cpp
@@ -0,0 +1,317 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2016 Yahoo! Technologies Norway AS
+
+#include <cppunit/TestFixture.h>
+#include <cppunit/extensions/HelperMacros.h>
+#include <vespa/document/select/gid_filter.h>
+#include <vespa/document/select/parser.h>
+#include <vespa/document/base/documentid.h>
+#include <vespa/document/base/testdocrepo.h>
+
+namespace document {
+namespace select {
+
+class GidFilterTest : public CppUnit::TestFixture {
+    struct Fixture {
+        TestDocRepo _repo;
+        BucketIdFactory _id_factory;
+        std::unique_ptr<Node> _root;
+        
+        Fixture(vespalib::stringref selection)
+            : _repo(),
+              _id_factory(),
+              _root(Parser(_repo.getTypeRepo(), _id_factory).parse(selection))
+        {
+        }
+
+        Fixture(Fixture&&) = default;
+        Fixture& operator=(Fixture&&) = default;
+
+        static Fixture for_selection(vespalib::stringref s) {
+            return Fixture(s);
+        }
+    };
+
+    static GlobalId id_to_gid(vespalib::stringref id_string) {
+        return DocumentId(id_string).getGlobalId();
+    }
+
+    bool might_match(vespalib::stringref selection,
+                     vespalib::stringref id_string) const
+    {
+        Fixture f(selection);
+        auto filter = GidFilter::for_selection_root_node(*f._root);
+        return filter.gid_might_match_selection(id_to_gid(id_string));
+    }
+
+    CPPUNIT_TEST_SUITE(GidFilterTest);
+    CPPUNIT_TEST(same_user_for_selection_and_gid_returns_match);
+    CPPUNIT_TEST(differing_user_for_selection_and_gid_returns_mismatch);
+    CPPUNIT_TEST(user_location_constraint_is_order_invariant);
+    CPPUNIT_TEST(non_location_selection_always_matches);
+    CPPUNIT_TEST(location_selection_does_not_match_non_location_id);
+    CPPUNIT_TEST(simple_conjunctive_location_expressions_are_filtered);
+    CPPUNIT_TEST(complex_conjunctive_location_expressions_are_filtered);
+    CPPUNIT_TEST(simple_disjunctive_location_expressions_are_not_filtered);
+    CPPUNIT_TEST(complex_disjunctive_location_expressions_are_not_filtered);
+    CPPUNIT_TEST(non_location_id_comparisons_are_not_filtered);
+    CPPUNIT_TEST(unsupported_location_comparison_operands_not_filtered);
+    CPPUNIT_TEST(default_constructed_filter_always_matches);
+    CPPUNIT_TEST(most_significant_32_bits_are_ignored);
+    CPPUNIT_TEST(gid_filters_may_be_copy_constructed);
+    CPPUNIT_TEST(gid_filters_may_be_copy_assigned);
+    CPPUNIT_TEST(same_group_for_selection_and_gid_returns_match);
+    CPPUNIT_TEST(differing_group_for_selection_and_gid_returns_mismatch);
+    CPPUNIT_TEST(composite_user_comparison_sub_expressions_not_supported);
+    CPPUNIT_TEST(composite_group_comparison_sub_expressions_not_supported);
+    CPPUNIT_TEST_SUITE_END();
+public:
+    void same_user_for_selection_and_gid_returns_match();
+    void differing_user_for_selection_and_gid_returns_mismatch();
+    void user_location_constraint_is_order_invariant();
+    void non_location_selection_always_matches();
+    void location_selection_does_not_match_non_location_id();
+    void simple_conjunctive_location_expressions_are_filtered();
+    void complex_conjunctive_location_expressions_are_filtered();
+    void simple_disjunctive_location_expressions_are_not_filtered();
+    void complex_disjunctive_location_expressions_are_not_filtered();
+    void non_location_id_comparisons_are_not_filtered();
+    void unsupported_location_comparison_operands_not_filtered();
+    void default_constructed_filter_always_matches();
+    void most_significant_32_bits_are_ignored();
+    void gid_filters_may_be_copy_constructed();
+    void gid_filters_may_be_copy_assigned();
+    void same_group_for_selection_and_gid_returns_match();
+    void differing_group_for_selection_and_gid_returns_mismatch();
+    void composite_user_comparison_sub_expressions_not_supported();
+    void composite_group_comparison_sub_expressions_not_supported();
+};
+
+CPPUNIT_TEST_SUITE_REGISTRATION(GidFilterTest);
+
+void
+GidFilterTest::same_user_for_selection_and_gid_returns_match()
+{
+    CPPUNIT_ASSERT(might_match("id.user == 12345",
+                               "id::testdoctype1:n=12345:foo"));
+    // User locations are defined over [0, 2**63-1]
+    CPPUNIT_ASSERT(might_match("id.user == 0",
+                               "id::testdoctype1:n=0:foo"));
+    CPPUNIT_ASSERT(might_match("id.user == 9223372036854775807" ,
+                               "id::testdoctype1:n=9223372036854775807:foo"));
+}
+
+void
+GidFilterTest::differing_user_for_selection_and_gid_returns_mismatch()
+{
+    CPPUNIT_ASSERT(!might_match("id.user == 1", "id::testdoctype1:n=2000:foo"));
+    // Similar, but non-identical, bit patterns
+    CPPUNIT_ASSERT(!might_match("id.user == 12345",
+                                "id::testdoctype1:n=12346:foo"));
+    CPPUNIT_ASSERT(!might_match("id.user == 12345",
+                                "id::testdoctype1:n=12344:foo"));
+}
+
+void
+GidFilterTest::user_location_constraint_is_order_invariant()
+{
+    CPPUNIT_ASSERT(might_match("12345 == id.user",
+                               "id::testdoctype1:n=12345:foo"));
+    CPPUNIT_ASSERT(!might_match("12345 == id.user",
+                                "id::testdoctype1:n=12346:foo"));
+}
+
+void
+GidFilterTest::non_location_selection_always_matches()
+{
+    CPPUNIT_ASSERT(might_match("testdoctype1.headerval == 67890",
+                               "id::testdoctype1:n=12345:foo"));
+}
+
+void
+GidFilterTest::location_selection_does_not_match_non_location_id()
+{
+    // Test name is a half-truth; the MD5-derived ID _will_ give a false
+    // positive every 2**32 or so document ID when the stars and their bit
+    // patterns align :)
+    CPPUNIT_ASSERT(!might_match("id.user == 987654321",
+                                "id::testdoctype1::foo"));
+
+    CPPUNIT_ASSERT(!might_match("id.group == 'snusmumrikk'",
+                                "id::testdoctype1::foo"));
+}
+
+void
+GidFilterTest::simple_conjunctive_location_expressions_are_filtered()
+{
+    // A conjunctive expression in this context is one where there exist a
+    // location predicate and the result of the entire expression can only
+    // be true iff the location predicate matches.
+    CPPUNIT_ASSERT(might_match("id.user == 12345 and true",
+                               "id::testdoctype1:n=12345:bar"));
+    CPPUNIT_ASSERT(might_match("true and id.user == 12345",
+                               "id::testdoctype1:n=12345:bar"));
+
+    CPPUNIT_ASSERT(!might_match("id.user == 123456 and true",
+                                "id::testdoctype1:n=12345:bar"));
+    CPPUNIT_ASSERT(!might_match("true and id.user == 123456",
+                                "id::testdoctype1:n=12345:bar"));
+}
+
+void
+GidFilterTest::complex_conjunctive_location_expressions_are_filtered()
+{
+    CPPUNIT_ASSERT(might_match("(((testdoctype1.headerval < 5) and (1 != 2)) "
+                               "and id.user == 12345)",
+                               "id::testdoctype1:n=12345:bar"));
+    CPPUNIT_ASSERT(!might_match("(((1 != 2) and (id.user==12345)) and "
+                                "(2 != 3)) and (testdoctype1.headerval < 5)",
+                                "id::testdoctype1:n=23456:bar"));
+    // In this case the expression contains a disjunction but the outcome
+    // of evaluating it still strongly depends on the location predicate.
+    CPPUNIT_ASSERT(might_match("((id.user == 12345 and true) and "
+                               "(true or false))",
+                               "id::testdoctype1:n=12345:bar"));
+    CPPUNIT_ASSERT(!might_match("((id.user == 12345 and true) and "
+                                "(true or false))",
+                                "id::testdoctype1:n=23456:bar"));
+}
+
+void
+GidFilterTest::simple_disjunctive_location_expressions_are_not_filtered()
+{
+    // Documents mismatch location but match selection as a whole.
+    CPPUNIT_ASSERT(might_match("id.user == 12345 or true",
+                               "id::testdoctype1:n=12345678:bar"));
+    CPPUNIT_ASSERT(might_match("true or id.user == 12345",
+                               "id::testdoctype1:n=12345678:bar"));
+}
+
+void
+GidFilterTest::complex_disjunctive_location_expressions_are_not_filtered()
+{
+    CPPUNIT_ASSERT(might_match("((id.user == 12345) and true) or false",
+                               "id::testdoctype1:n=12345678:bar"));
+    CPPUNIT_ASSERT(might_match("((id.user == 12345) or false) and true",
+                               "id::testdoctype1:n=12345678:bar"));
+}
+
+void
+GidFilterTest::non_location_id_comparisons_are_not_filtered()
+{
+    // Note: these selections are syntactically valid but semantically
+    // invalid (comparing strings to integers), but are used to catch any
+    // logic holes where an id node is indiscriminately treated as something
+    // from which we should derive a GID-related integer.
+    CPPUNIT_ASSERT(might_match("id.namespace == 123456",
+                               "id::testdoctype1:n=12345678:bar"));
+    CPPUNIT_ASSERT(might_match("id.type == 1234",
+                               "id::testdoctype1:n=12345678:bar"));
+    CPPUNIT_ASSERT(might_match("id.scheme == 555",
+                               "id::testdoctype1:n=12345678:bar"));
+    CPPUNIT_ASSERT(might_match("id.specific == 7654",
+                               "id::testdoctype1:n=12345678:bar"));
+}
+
+void
+GidFilterTest::unsupported_location_comparison_operands_not_filtered()
+{
+    CPPUNIT_ASSERT(might_match("id.user == 'rick & morty'",
+                               "id::testdoctype1:n=12345678:bar"));
+    CPPUNIT_ASSERT(might_match("id.group == 56789",
+                               "id::testdoctype1:n=12345678:bar"));
+    CPPUNIT_ASSERT(might_match("id.user == testdoctype1.headervalue",
+                               "id::testdoctype1:n=12345678:bar"));
+    CPPUNIT_ASSERT(might_match("id.group == testdoctype1.headervalue",
+                               "id::testdoctype1:g=helloworld:bar"));
+}
+
+void
+GidFilterTest::default_constructed_filter_always_matches()
+{
+    GidFilter filter;
+    CPPUNIT_ASSERT(filter.gid_might_match_selection(
+            DocumentId("id::testdoctype1:n=12345678:bar").getGlobalId()));
+    CPPUNIT_ASSERT(filter.gid_might_match_selection(
+            DocumentId("id::testdoctype1::foo").getGlobalId()));
+}
+
+void
+GidFilterTest::most_significant_32_bits_are_ignored()
+{
+    // The fact that the 32 MSB are effectively ignored is an artifact of
+    // how the GID location extraction is historically performed and is not
+    // necessarily the optimum (in particular, an XOR combination of the upper
+    // and lower 32 bits would likely be much better), but it's what the
+    // behavior currently is and should thus be tested.
+
+    // The following locations have the same 32 LSB:
+    CPPUNIT_ASSERT(might_match("id.user == 12345678901",
+                               "id::testdoctype1:n=29525548085:bar"));
+}
+
+void
+GidFilterTest::gid_filters_may_be_copy_constructed()
+{
+    Fixture f("id.user == 1337");
+    GidFilter filter = GidFilter::for_selection_root_node(*f._root);
+
+    GidFilter copy_constructed(filter);
+    CPPUNIT_ASSERT(copy_constructed.gid_might_match_selection(
+            id_to_gid("id::testdoctype1:n=1337:zoid")));
+    CPPUNIT_ASSERT(!copy_constructed.gid_might_match_selection(
+            id_to_gid("id::testdoctype1:n=555:zoid")));
+
+}
+
+void
+GidFilterTest::gid_filters_may_be_copy_assigned()
+{
+    Fixture f("id.user == 1337");
+    GidFilter filter = GidFilter::for_selection_root_node(*f._root);
+
+    GidFilter copy_assigned;
+    copy_assigned = filter;
+
+    CPPUNIT_ASSERT(copy_assigned.gid_might_match_selection(
+            id_to_gid("id::testdoctype1:n=1337:zoid")));
+    CPPUNIT_ASSERT(!copy_assigned.gid_might_match_selection(
+            id_to_gid("id::testdoctype1:n=555:zoid")));
+}
+
+void
+GidFilterTest::same_group_for_selection_and_gid_returns_match()
+{
+    CPPUNIT_ASSERT(might_match("id.group == 'bjarne'",
+                               "id::testdoctype1:g=bjarne:foo"));
+    CPPUNIT_ASSERT(might_match("id.group == 'andrei'",
+                               "id::testdoctype1:g=andrei:bar"));
+}
+
+void
+GidFilterTest::differing_group_for_selection_and_gid_returns_mismatch()
+{
+    CPPUNIT_ASSERT(!might_match("id.group == 'cult of bjarne'",
+                                "id::testdoctype1:g=stl:foo"));
+    CPPUNIT_ASSERT(!might_match("id.group == 'sutters mill'",
+                                "id::testdoctype1:g=andrei:bar"));
+}
+
+void
+GidFilterTest::composite_user_comparison_sub_expressions_not_supported()
+{
+    // Technically this is a mismatch, but we currently only want to support
+    // the simple, obvious cases since this is not an expected use case.
+    CPPUNIT_ASSERT(might_match("id.user == (1 + 2)",
+                               "id::testdoctype1:n=20:foo"));
+}
+
+void
+GidFilterTest::composite_group_comparison_sub_expressions_not_supported()
+{
+    CPPUNIT_ASSERT(might_match("id.group == 'foo'+'bar'",
+                               "id::testdoctype1:g=sputnik_hits:foo"));
+}
+
+} // select
+} // document
diff --git a/document/src/tests/globalidtest.cpp b/document/src/tests/globalidtest.cpp
new file mode 100644
index 00000000000..d0d066e5510
--- /dev/null
+++ b/document/src/tests/globalidtest.cpp
@@ -0,0 +1,268 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <cppunit/extensions/HelperMacros.h>
+#include <vespa/document/bucket/bucketidfactory.h>
+#include <vespa/document/base/documentid.h>
+#include <vespa/document/base/globalid.h>
+#include <vespa/vespalib/util/random.h>
+#include <vespa/vespalib/stllike/asciistream.h>
+
+namespace document {
+
+struct GlobalIdTest : public CppUnit::TestFixture {
+    void testNormalUsage();
+    void testBucketIdConversion();
+    void testGidRangeConversion();
+    void testBucketOrderCmp();
+
+    void verifyGlobalIdRange(const std::vector<DocumentId>& ids,
+                             uint32_t countBits);
+
+    CPPUNIT_TEST_SUITE(GlobalIdTest);
+    CPPUNIT_TEST(testNormalUsage);
+    CPPUNIT_TEST(testBucketIdConversion);
+    CPPUNIT_TEST(testGidRangeConversion);
+    CPPUNIT_TEST(testBucketOrderCmp);
+    CPPUNIT_TEST_SUITE_END();
+};
+
+CPPUNIT_TEST_SUITE_REGISTRATION(GlobalIdTest);
+
+void
+GlobalIdTest::testNormalUsage()
+{
+    const char* emptystring = "\0\0\0\0\0\0\0\0\0\0\0\0";
+    const char* teststring = "1234567890ABCDEF";
+    CPPUNIT_ASSERT(strlen(teststring) > GlobalId::LENGTH);
+
+    { // Test empty global id
+        GlobalId id;
+        for (uint32_t i=0; i<GlobalId::LENGTH; ++i) {
+            CPPUNIT_ASSERT_EQUAL((unsigned char) '\0', id.get()[i]);
+        }
+        GlobalId id2(emptystring);
+        CPPUNIT_ASSERT_EQUAL(id, id2);
+        CPPUNIT_ASSERT(!(id != id2));
+        CPPUNIT_ASSERT(!(id < id2) && !(id2 < id));
+        CPPUNIT_ASSERT_EQUAL(std::string(emptystring, GlobalId::LENGTH),
+                             std::string((const char*) id.get(),
+                                         GlobalId::LENGTH));
+    }
+    { // Test non-empty global id
+        GlobalId empty;
+        GlobalId initialempty;
+        initialempty.set(teststring);
+        GlobalId id(teststring);
+        CPPUNIT_ASSERT_EQUAL(id, initialempty);
+        CPPUNIT_ASSERT(!(id != initialempty));
+        CPPUNIT_ASSERT(!(id < initialempty) && !(initialempty < id));
+
+        CPPUNIT_ASSERT(id != empty);
+        CPPUNIT_ASSERT(!(id == empty));
+        CPPUNIT_ASSERT(!(id < empty) && (empty < id));
+
+        CPPUNIT_ASSERT_EQUAL(
+                std::string(teststring, GlobalId::LENGTH),
+                std::string((const char*) id.get(), GlobalId::LENGTH));
+        CPPUNIT_ASSERT_EQUAL(std::string(teststring, GlobalId::LENGTH),
+                             std::string((const char*) initialempty.get(),
+                                         GlobalId::LENGTH));
+    }
+    { // Test printing and parsing
+        GlobalId id1("LIN!#LNKASD#!MYL#&NK");
+        CPPUNIT_ASSERT_EQUAL(vespalib::string("gid(0x4c494e21234c4e4b41534423)"),
+                             id1.toString());
+        GlobalId id2 = GlobalId::parse(id1.toString());
+        CPPUNIT_ASSERT_EQUAL(id1, id2);
+            // Verify string representation too, to verify that operator== works
+        CPPUNIT_ASSERT_EQUAL(vespalib::string("gid(0x4c494e21234c4e4b41534423)"),
+                             id2.toString());
+    }
+}
+
+namespace {
+    void verifyDocumentId(const std::string& s) {
+        document::DocumentId did(s);
+        BucketIdFactory factory;
+        BucketId bid = factory.getBucketId(did);
+        GlobalId gid = did.getGlobalId();
+        BucketId generated = gid.convertToBucketId();
+        //std::cerr << bid << ", " << generated << "\n";
+        if (bid != generated) {
+            std::ostringstream ost;
+            ost << "Document id " << s << " with gid " << gid
+                << " belongs to bucket " << bid
+                << ", but globalid convert function generated bucketid "
+                << generated;
+            CPPUNIT_FAIL(ost.str());
+        }
+    }
+}
+
+void
+GlobalIdTest::testBucketIdConversion()
+{
+    verifyDocumentId("userdoc:ns:1:abc");
+    verifyDocumentId("userdoc:ns:1000:abc");
+    verifyDocumentId("userdoc:hsgf:18446744073700000000:dfdfsdfg");
+    verifyDocumentId("groupdoc:ns:somegroup:hmm");
+    verifyDocumentId("doc::test");
+    verifyDocumentId("doc:myns:http://foo.bar");
+    verifyDocumentId("doc:jsrthsdf:a234aleingzldkifvasdfgadf");
+}
+
+void
+GlobalIdTest::verifyGlobalIdRange(const std::vector<DocumentId>& ids,
+                                  uint32_t countBits)
+{
+    BucketIdFactory factory;
+    for (uint32_t i=0, n=ids.size(); i<n; ++i) {
+            // Create the bucket this document would be in with given
+            // countbits
+        BucketId bucket(factory.getBucketId(ids[i]));
+        bucket.setUsedBits(countBits);
+        bucket = bucket.stripUnused();
+            // Get the min and max GIDs for this bucket
+        GlobalId first = GlobalId::calculateFirstInBucket(bucket);
+        GlobalId last = GlobalId::calculateLastInBucket(bucket);
+            // For each document in set, verify that they are within
+            // limits if they are contained in bucket.
+        for (uint32_t j=0; j<n; ++j) {
+            BucketId bid(factory.getBucketId(ids[j]));
+            GlobalId gid(ids[j].getGlobalId());
+            uint64_t gidKey = gid.convertToBucketId().toKey();
+            if (bucket.contains(bid)) {
+                if ((gidKey < last.convertToBucketId().toKey()) || (gidKey > last.convertToBucketId().toKey())) {
+                    std::ostringstream msg;
+                    msg << gid << " should be in the range " << first
+                        << " - " << last;
+                    msg << ", as bucket " << gid.convertToBucketId()
+                        << " should be in the range "
+                        << first.convertToBucketId() << " - "
+                        << last.convertToBucketId() << "\n";
+                    msg << ", reverted " << std::hex
+                        << gid.convertToBucketId().toKey()
+                        << " should be in the range "
+                        << first.convertToBucketId().toKey() << " - "
+                        << last.convertToBucketId().toKey() << "\n";
+                    CPPUNIT_ASSERT_MESSAGE(msg.str(),
+                            gid.convertToBucketId().toKey() >=
+                                   first.convertToBucketId().toKey());
+                    CPPUNIT_ASSERT_MESSAGE(msg.str(),
+                            gid.convertToBucketId().toKey() <=
+                                   last.convertToBucketId().toKey());
+                }
+            } else {
+                if ((gidKey >= first.convertToBucketId().toKey()) && (gidKey <= last.convertToBucketId().toKey())) {
+                    std::ostringstream msg;
+                    msg << gid << " should not be in the range " << first
+                        << " - " << last;
+                    CPPUNIT_ASSERT_MESSAGE(msg.str(),
+                            (gid.convertToBucketId().toKey() <
+                                first.convertToBucketId().toKey())
+                            ||
+                            (gid.convertToBucketId().toKey() >
+                                last.convertToBucketId().toKey()));
+                }
+            }
+        }
+    }
+}
+
+void
+GlobalIdTest::testGidRangeConversion()
+{
+        // Generate a lot of random document ids used for test
+    std::vector<DocumentId> docIds;
+    vespalib::RandomGen randomizer(0xdeadbabe);
+    for (uint32_t j=0; j<100; ++j) {
+        vespalib::asciistream name_space;
+        vespalib::asciistream ost;
+        for (uint32_t i=0, n=randomizer.nextUint32(1, 5); i<n; ++i) {
+            name_space << (char) ('a' + randomizer.nextUint32(0, 25));
+        }
+        uint32_t scheme = randomizer.nextUint32(0, 2);
+        switch (scheme) {
+            case 0: ost << "doc:" << name_space.str() << ":";
+                    break;
+            case 1: ost << "userdoc:" << name_space.str() << ":";
+                    ost << randomizer.nextUint32() << ":";
+                    break;
+            case 2: ost << "groupdoc:" << name_space.str() << ":";
+                    for (uint32_t i=0, n=randomizer.nextUint32(1, 10); i<n; ++i)
+                    {
+                        ost << (char) ('a' + randomizer.nextUint32(0, 25));
+                    }
+                    ost << ":";
+                    break;
+            default: CPPUNIT_ASSERT(false);
+        }
+        ost << "http://";
+        for (uint32_t i=0, n=randomizer.nextUint32(1, 20); i<n; ++i) {
+            ost << (char) ('a' + randomizer.nextUint32(0, 25));
+        }
+        docIds.push_back(DocumentId(ost.str()));
+    }
+    //std::cerr << "\nDoing " << ((58 - 16) * docIds.size() * docIds.size())
+    //          << " tests for whether global id calculation is correct.\n";
+    for (uint32_t i=1; i<=58; ++i) {
+        //std::cerr << "Verifying with " << i << " countbits\n";
+        verifyGlobalIdRange(docIds, i);
+    }
+}
+
+void
+GlobalIdTest::testBucketOrderCmp()
+{
+    typedef GlobalId::BucketOrderCmp C;
+    CPPUNIT_ASSERT(C::compareRaw(0, 0) == 0);
+    CPPUNIT_ASSERT(C::compareRaw(0, 1) == -1);
+    CPPUNIT_ASSERT(C::compareRaw(1, 0) == 1);
+    CPPUNIT_ASSERT(C::compareRaw(255, 255) == 0);
+    CPPUNIT_ASSERT(C::compareRaw(0, 255) == -255);
+    CPPUNIT_ASSERT(C::compareRaw(255, 0) == 255);
+    CPPUNIT_ASSERT(C::compareRaw(254, 254) == 0);
+    CPPUNIT_ASSERT(C::compareRaw(254, 255) == -1);
+    CPPUNIT_ASSERT(C::compareRaw(255, 254) == 1);
+    {
+        // Test raw comparator object.
+        GlobalId foo = GlobalId::parse("gid(0x000001103330333077700000)");
+        GlobalId bar = GlobalId::parse("gid(0x000000100030003000700000)");
+        GlobalId baz = GlobalId::parse("gid(0x000000103330333000700000)");
+
+        GlobalId::BucketOrderCmp cmp;
+        CPPUNIT_ASSERT(!cmp(foo, foo));
+        CPPUNIT_ASSERT(!cmp(bar, bar));
+        CPPUNIT_ASSERT(!cmp(baz, baz));
+        CPPUNIT_ASSERT(!cmp(foo, bar));
+        CPPUNIT_ASSERT( cmp(bar, foo));
+        CPPUNIT_ASSERT(!cmp(foo, baz));
+        CPPUNIT_ASSERT( cmp(baz, foo));
+        CPPUNIT_ASSERT(!cmp(baz, bar));
+        CPPUNIT_ASSERT( cmp(bar, baz));
+    }
+    {
+        // Test sorting by bucket.
+        GlobalId foo = GlobalId::parse("gid(0x000001103330333077700000)");
+        GlobalId bar = GlobalId::parse("gid(0x000000100030003000700000)");
+        GlobalId baz = GlobalId::parse("gid(0x000000103330333000700000)");
+
+        typedef std::map<GlobalId, uint32_t, GlobalId::BucketOrderCmp> GidMap;
+        GidMap gidMap;
+        gidMap[foo] = 666;
+        gidMap[bar] = 777;
+        gidMap[baz] = 888;
+
+        GidMap::iterator it = gidMap.begin();
+        CPPUNIT_ASSERT(it->first == bar);
+        CPPUNIT_ASSERT(it->second == 777);
+        ++it;
+        CPPUNIT_ASSERT(it->first == baz);
+        CPPUNIT_ASSERT(it->second == 888);
+        ++it;
+        CPPUNIT_ASSERT(it->first == foo);
+        CPPUNIT_ASSERT(it->second == 666);
+    }
+}
+
+} // document
diff --git a/document/src/tests/heapdebugger.h b/document/src/tests/heapdebugger.h
new file mode 100644
index 00000000000..deb862fbec3
--- /dev/null
+++ b/document/src/tests/heapdebugger.h
@@ -0,0 +1,68 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * @file heapdebugger.h
+ *
+ * @author Ove Martin Malm
+ * @date Creation date: 2000-08-09
+ * @version $Id$
+ * @file mcheckhooks.h
+ *
+ * Copyright (c) : 1997-2000 Fast Search & Transfer ASA
+ * ALL RIGHTS RESERVED
+ */
+
+#pragma once
+#include <cstddef>
+
+
+/**
+ *  This function is used for counting memory usage. Must be called before any block is allocated
+ * (pt. linux only)
+ * Parameter controls operation
+ */
+extern void enableHeapUsageMonitor(int param = 0);
+
+
+/**
+ * This function return the current memory used.
+ * @return   Net number of bytes allocated on the heap
+ */
+extern size_t getHeapUsage(void);
+
+/**
+ * This enables heap debugging. Must be called before any block is allocated
+ * Parameter controls operation
+ * (pt. linux only)
+ */
+extern void enableHeapCorruptCheck(int param = 0);
+
+#define HEAPCHECKMODE_REMOVE        -1      // Deinstall
+#define HEAPCHECKMODE_NORMAL        0       // Normal
+#define HEAPCHECKMODE_EXTENSIVE     1       // All allocated blocks checked on all ops..
+#define HEAPCHECKMODE_DISABLED      2       // No checking (but extra bytes allocated)
+
+
+
+/**
+ * Run a heap check now. Will lock the heap and run through a full check. Will crash if it fails...
+ */
+extern void checkHeapNow(void);
+
+
+/**
+ * And this enables linux mcheck function with an approprate callback function. (absolutely linux only)
+ * see man mcheck
+ */
+extern void enableMCheck(void);
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/document/src/tests/heapdebuggerlinux.cpp b/document/src/tests/heapdebuggerlinux.cpp
new file mode 100644
index 00000000000..5afe583c5e1
--- /dev/null
+++ b/document/src/tests/heapdebuggerlinux.cpp
@@ -0,0 +1,708 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ *
+ * @author Ove Martin Malm
+ * @date Creation date: 2000-21-08
+ * @version $Id$
+ *
+ * Copyright (c) : 1997-2000 Fast Search & Transfer ASA
+ * ALL RIGHTS RESERVED
+ */
+
+
+// These are necessary to make it compile....
+#define _MALLOC_INTERNAL
+#define MALLOC_HOOKS
+#include <vespa/fastos/fastos.h>
+#include <stdio.h>
+#define _LIBC
+#include <malloc.h>
+#include <mcheck.h>
+#include <stdint.h>
+#include <libintl.h>
+
+#include <iostream>
+#include <pthread.h>
+#include <signal.h>
+#include <unistd.h>
+#include <string.h>
+#include <ctype.h>
+
+#include "heapdebugger.h"
+
+// Predeclaration
+static void HeapCorrupt(const char * caller, const struct elemheader *hdr, enum mcheck_status err);
+
+// -------------------------------
+// Part #1 Memory usage
+
+
+#ifndef INTERNAL_SIZE_T
+#define INTERNAL_SIZE_T size_t
+#endif
+
+#define SIZE_SZ                (sizeof(INTERNAL_SIZE_T))
+
+/* size field is or'ed with PREV_INUSE when previous adjacent chunk in use */
+
+#define PREV_INUSE 0x1
+
+/* size field is or'ed with IS_MMAPPED if the chunk was obtained with mmap() */
+
+#define IS_MMAPPED 0x2
+
+/* Bits to mask off when extracting size */
+
+#define SIZE_BITS (PREV_INUSE|IS_MMAPPED)
+
+struct malloc_chunk
+{
+    INTERNAL_SIZE_T prev_size; /* Size of previous chunk (if free). */
+    INTERNAL_SIZE_T size;      /* Size in bytes, including overhead. */
+    struct malloc_chunk* fd;   /* double links -- used only if free. */
+    struct malloc_chunk* bk;
+};
+
+typedef struct malloc_chunk* mchunkptr;
+
+/* conversion from malloc headers to user pointers, and back */
+
+#define chunk2mem(p)   ((Void_t*)((char*)(p) + 2*SIZE_SZ))
+#define mem2chunk(mem) ((mchunkptr)((char*)(mem) - 2*SIZE_SZ))
+
+/* Get size, ignoring use bits */
+
+#define chunkSize(p)          ((p)->size & ~(SIZE_BITS))
+
+
+static void (*old_free_hook_memusage)(void* buf, const void * caller );
+static void* (*old_malloc_hook_memusage)(size_t bytes, const void * caller);
+static void* (*old_realloc_hook_memusage)(void *buf, size_t bytes, const void *caller);
+static void* (*old_memalign_hook_memusage)(size_t alignment, size_t bytes,const void *caller);
+static size_t MemoryUsage = 0;
+static pthread_mutex_t memUsageMutex = PTHREAD_MUTEX_INITIALIZER;
+static bool MemUsageHookInstalled = false;
+
+
+static void me_use_free_hook(void *ptr, const void *caller)
+{
+    mchunkptr p;
+
+    pthread_mutex_lock(&memUsageMutex);
+
+    if (ptr) {
+        p = mem2chunk(ptr);
+        MemoryUsage -= chunkSize(p);
+    }
+
+    __free_hook = old_free_hook_memusage;
+
+    if (old_free_hook_memusage != NULL) {
+        (*old_free_hook_memusage) (ptr, caller);
+    } else {
+        free (ptr);
+    }
+
+    __free_hook = me_use_free_hook;
+
+    pthread_mutex_unlock(&memUsageMutex);
+}
+
+static void* me_use_malloc_hook(size_t size, const void *caller)
+{
+    void *anew;
+    mchunkptr p;
+
+    pthread_mutex_lock(&memUsageMutex);
+
+    __malloc_hook = old_malloc_hook_memusage;
+
+    if (old_malloc_hook_memusage != NULL) {
+        anew = (*old_malloc_hook_memusage) (size, caller);
+    } else {
+        anew = malloc (size);
+    }
+
+    if (anew) {
+        p = mem2chunk(anew);
+        MemoryUsage += chunkSize(p);
+    }
+
+    __malloc_hook = me_use_malloc_hook;
+    pthread_mutex_unlock(&memUsageMutex);
+
+    return(anew);
+}
+
+static void* me_use_realloc_hook(void *ptr, size_t size, const void* caller)
+{
+    void *anew;
+    mchunkptr p;
+    size_t oldsz = 0;
+    size_t newsz = 0;
+
+    pthread_mutex_lock(&memUsageMutex);
+
+    if (ptr) {
+        p = mem2chunk(ptr);
+        oldsz = chunkSize(p);
+    }
+
+    __free_hook = old_free_hook_memusage;
+    __malloc_hook = old_malloc_hook_memusage;
+    __realloc_hook = old_realloc_hook_memusage;
+
+    if (old_realloc_hook_memusage != NULL) {
+        anew = (*old_realloc_hook_memusage) (ptr, size, caller);
+    } else {
+        anew = realloc (ptr, size);
+    }
+
+    if (anew) {
+        p = mem2chunk(anew);
+        newsz = chunkSize(p);
+
+        MemoryUsage += (newsz - oldsz);
+    }
+
+    __free_hook = me_use_free_hook;
+    __malloc_hook = me_use_malloc_hook;
+    __realloc_hook = me_use_realloc_hook;
+
+    pthread_mutex_unlock(&memUsageMutex);
+
+    return anew;
+}
+
+static void* me_use_memalign_hook(size_t alignment, size_t bytes,const void *caller)
+{
+
+    void *aligned;
+
+    std::cout << "memalign" << std::endl;
+    __memalign_hook = old_memalign_hook_memusage;
+
+    if (old_memalign_hook_memusage != NULL) {
+        aligned = (*old_memalign_hook_memusage) (alignment, bytes,caller);
+    } else {
+        aligned = memalign (alignment, bytes);
+    }
+
+    __memalign_hook = me_use_memalign_hook;
+
+    return aligned;
+}
+
+
+void enableHeapUsageMonitor(int param)
+{
+    (void)param;
+    if (!MemUsageHookInstalled) {
+        old_free_hook_memusage = __free_hook;
+        __free_hook = me_use_free_hook;
+        old_malloc_hook_memusage = __malloc_hook;
+        __malloc_hook = me_use_malloc_hook;
+        old_realloc_hook_memusage = __realloc_hook;
+        __realloc_hook = me_use_realloc_hook;
+        MemUsageHookInstalled = true;
+        std::cout << "Memory usage will be counted" << std::endl;
+    }
+}
+
+size_t getHeapUsage(void)
+{
+    return(MemoryUsage);
+}
+
+
+// + Get status message....
+
+
+
+
+// -------------------
+// Part 2:  Heap corrupt
+
+static void (*old_free_hook_memcorrupt)(void*, const void *);
+static void* (*old_malloc_hook_memcorrupt)(size_t, const void *);
+static void* (*old_realloc_hook_memcorrupt)(void*, size_t, const void *);
+static void* (*old_memalign_hook_memcorrupt)(size_t alignment, size_t bytes,const void *caller);
+
+static pthread_mutex_t HeapCorruptCheckMutex = PTHREAD_MUTEX_INITIALIZER;
+
+static int HeapCorruptCheckMode = HEAPCHECKMODE_REMOVE;
+static bool PrintUsage = false;
+
+
+/* Arbitrary magical numbers.  */
+#define MAGICWORD	0xfedabeeb
+#define MAGICFREE	0xf1eef1ee
+#define MAGICBYTE	((char) 0x01)  // "End byte"
+#define MALLOCFLOOD	((char) 0x02)  // "Fresh (unused) memory"
+#define FREEFLOOD	((char) 0x03)  // "Dead" memory
+
+struct elemheader {
+    __malloc_size_t size;	        /* Exact size requested by user.  */
+    unsigned long int magic;	/* Magic number to check header integrity.  */
+    struct elemheader *prev;
+    struct elemheader *next;
+};
+
+/* This is the beginning of the list of all memory blocks allocated.
+   It is only constructed if the pedantic testing is requested.  */
+
+static struct elemheader *root;
+
+static enum mcheck_status me_corrupt_mprobe (const struct elemheader * block)
+{
+    enum mcheck_status status;
+    switch (block->magic ^ ((uintptr_t) block->prev + (uintptr_t) block->next)) {
+    case MAGICFREE:
+        status = MCHECK_FREE;
+        break;
+    case MAGICWORD:
+    {
+        const char *elem = (const char *)block;
+        status = (elem[sizeof(struct elemheader)+block->size] != MAGICBYTE) ?  MCHECK_TAIL : MCHECK_OK;
+    }
+    break;
+
+    default:
+        status = MCHECK_HEAD;
+        break;
+    }
+    return status;
+}
+
+
+
+static void unlink_blk (struct elemheader *ptr)
+{
+    if (ptr->next != NULL) {
+        ptr->next->prev = ptr->prev;
+        ptr->next->magic = MAGICWORD ^ ((uintptr_t) ptr->next->prev + (uintptr_t) ptr->next->next);
+    }
+
+    if (ptr->prev != NULL) {
+        ptr->prev->next = ptr->next;
+        ptr->prev->magic = MAGICWORD ^ ((uintptr_t) ptr->prev->prev + (uintptr_t) ptr->prev->next);
+    } else {
+        root = ptr->next;
+    }
+}
+
+static void link_blk  (struct elemheader *ptr)
+{
+    ptr->prev = NULL;
+    ptr->next = root;
+    root = ptr;
+    ptr->magic = MAGICWORD ^ (uintptr_t) ptr->next;
+
+    /* And the next block.  */
+    if (ptr->next != NULL) {
+        ptr->next->prev = ptr;
+        ptr->next->magic = MAGICWORD ^ ((uintptr_t) ptr + (uintptr_t) ptr->next->next);
+    }
+}
+
+static void heapwalker(void)
+{
+    const struct elemheader *runp = root;
+
+    while (runp)  {
+        enum mcheck_status mstatus = me_corrupt_mprobe(runp);
+        if (mstatus != MCHECK_OK) {
+            HeapCorrupt("heapwalker",runp,mstatus);
+        }
+        runp = runp->next;
+    }
+}
+
+
+static void me_corrupt_free_hook (__ptr_t ptr, const __ptr_t caller)
+{
+    enum mcheck_status mstatus;
+
+    pthread_mutex_lock(&HeapCorruptCheckMutex);
+
+    if (HeapCorruptCheckMode == HEAPCHECKMODE_EXTENSIVE) {
+        heapwalker();
+    }
+
+    if (PrintUsage) {
+        if (ptr) {
+            struct elemheader *hdr = (struct elemheader *)ptr - 1;
+            std::cout << "pid :" << getpid() << " free: Attempting to delete area with size " <<  hdr->size << " bytes" << std::endl;
+        } else {
+            std::cout << "pid :" << getpid() << " free: Attempting to delete NULL pointer " << std::endl;
+        }
+    }
+
+
+    if (ptr) {
+        struct elemheader *hdr = (struct elemheader *)ptr - 1;
+
+        mstatus = (HeapCorruptCheckMode == HEAPCHECKMODE_DISABLED) ? MCHECK_OK : me_corrupt_mprobe(hdr);
+        if (mstatus != MCHECK_OK) {
+            HeapCorrupt("free", hdr ,mstatus);
+        }
+
+        if (mstatus == MCHECK_OK) {  // ForGet element if not detected or reallocated..
+            hdr->magic = MAGICFREE;
+            unlink_blk (hdr);
+            hdr->prev = hdr->next = NULL;
+
+            char * userdata = (char *)hdr+sizeof(struct elemheader);
+            if (hdr->size > 0) {
+                memset (userdata, FREEFLOOD, hdr->size);
+            }
+
+            ptr = (__ptr_t) hdr;
+        } else {
+            ptr = NULL;    // ForGet element if not detected or reallocated..
+        }
+    }
+
+
+    __free_hook = old_free_hook_memcorrupt;
+
+    if (old_free_hook_memcorrupt != NULL) {
+        (*old_free_hook_memcorrupt) (ptr, caller);
+    } else {
+        free (ptr);
+    }
+
+    __free_hook = me_corrupt_free_hook;
+
+    pthread_mutex_unlock(&HeapCorruptCheckMutex);
+}
+
+static __ptr_t me_corrupt_malloc_hook (__malloc_size_t size, const __ptr_t caller)
+{
+    struct elemheader *hdr;
+    enum mcheck_status mstatus;
+
+    pthread_mutex_lock(&HeapCorruptCheckMutex);
+
+
+    if (HeapCorruptCheckMode == HEAPCHECKMODE_EXTENSIVE) {
+        heapwalker();
+    }
+
+    if (PrintUsage) {
+        std::cout << "pid :" << getpid() << " malloc: Attempting to allocate " <<  size << " bytes" << std::endl;
+    }
+
+
+    __malloc_hook = old_malloc_hook_memcorrupt;
+    if (old_malloc_hook_memcorrupt != NULL) {
+        hdr = (struct elemheader *) (*old_malloc_hook_memcorrupt) (sizeof (struct elemheader) + size + 1,caller);
+    } else {
+        hdr = (struct elemheader *) malloc (sizeof (struct elemheader) + size + 1);
+    }
+
+    __malloc_hook = me_corrupt_malloc_hook;
+
+    if (hdr) {
+        hdr->size = size;
+        link_blk (hdr);
+
+        char * userdata = (char *)hdr+sizeof(struct elemheader);
+        userdata[size] = MAGICBYTE;
+        if (size > 0) {
+            memset (userdata, MALLOCFLOOD, size);
+        }
+
+        // Self check!
+        mstatus = (HeapCorruptCheckMode == HEAPCHECKMODE_DISABLED) ? MCHECK_OK : me_corrupt_mprobe(hdr);
+        if (mstatus != MCHECK_OK) {
+            HeapCorrupt("malloc", hdr ,mstatus);
+        }
+    }
+
+    pthread_mutex_unlock(&HeapCorruptCheckMutex);
+
+    return hdr ? (__ptr_t) (hdr + 1) : NULL;
+}
+
+static __ptr_t me_corrupt_realloc_hook (__ptr_t ptr, __malloc_size_t size, const __ptr_t caller)
+{
+    struct elemheader *hdr;
+    __malloc_size_t osize;
+    enum mcheck_status mstatus;
+
+    pthread_mutex_lock(&HeapCorruptCheckMutex);
+
+    if (HeapCorruptCheckMode == HEAPCHECKMODE_EXTENSIVE) {
+        heapwalker();
+    }
+
+    if (PrintUsage) {
+        std::cout << "pid :" << getpid() << " realloc: Attempting to allocate area with size " <<  size << " bytes.";
+        if (ptr) {
+            hdr = (struct elemheader *)ptr - 1;
+            std::cout << " Old buffer was " << hdr->size << " bytes" << std::endl;
+        } else {
+            std::cout << " No old buffer." << std::endl;
+        }
+    }
+
+    if (ptr) {
+        hdr = ((struct elemheader *) ptr) - 1;
+        osize = hdr->size;
+
+        mstatus = (HeapCorruptCheckMode == HEAPCHECKMODE_DISABLED) ? MCHECK_OK : me_corrupt_mprobe(hdr);
+        if (mstatus != MCHECK_OK) {
+            HeapCorrupt("realloc/1", hdr ,mstatus);
+        }
+
+        unlink_blk (hdr);
+        if (size < osize) {
+            memset ((char *) ptr + size, FREEFLOOD, osize - size);
+        }
+    } else {
+        osize = 0;
+        hdr = NULL;
+    }
+
+    __free_hook = old_free_hook_memcorrupt;
+    __malloc_hook = old_malloc_hook_memcorrupt;
+    __realloc_hook = old_realloc_hook_memcorrupt;
+
+    if (old_realloc_hook_memcorrupt != NULL) {
+        hdr = (struct elemheader *) (*old_realloc_hook_memcorrupt) ((__ptr_t) hdr, sizeof (struct elemheader) + size + 1, caller);
+    } else {
+        hdr = (struct elemheader *) realloc ((__ptr_t) hdr,sizeof (struct elemheader) + size + 1);
+    }
+
+    __free_hook = me_corrupt_free_hook;
+    __malloc_hook = me_corrupt_malloc_hook;
+    __realloc_hook = me_corrupt_realloc_hook;
+
+    if (hdr) {
+        hdr->size = size;
+        link_blk (hdr);
+
+        char * userdata = (char *)hdr+sizeof(struct elemheader);
+        userdata[size] = MAGICBYTE;
+
+        if (size > osize) {
+            memset (&userdata[osize], MALLOCFLOOD, size-osize);
+        }
+
+        mstatus = (HeapCorruptCheckMode == HEAPCHECKMODE_DISABLED) ? MCHECK_OK : me_corrupt_mprobe(hdr);
+        if (mstatus != MCHECK_OK) {
+            HeapCorrupt("realloc/2", hdr ,mstatus);
+        }
+    }
+
+    pthread_mutex_unlock(&HeapCorruptCheckMutex);
+
+    return hdr ? (__ptr_t) (hdr + 1) : NULL;
+}
+
+
+static void* me_corrupt_memalign_hook(size_t alignment, size_t bytes,const void *caller)
+{
+    void *aligned;
+
+    std::cout << "memalign #2 " << std::endl;
+    __memalign_hook = old_memalign_hook_memcorrupt;
+
+    if (old_memalign_hook_memcorrupt != NULL) {
+        aligned = (*old_memalign_hook_memcorrupt) (alignment, bytes,caller);
+    } else {
+        aligned = memalign (alignment, bytes);
+    }
+
+    __memalign_hook = me_corrupt_memalign_hook;
+
+    return aligned;
+}
+
+
+
+void enableHeapCorruptCheck(int param)
+{
+
+    pthread_mutex_lock(&HeapCorruptCheckMutex);
+
+
+    // This is install..
+    if (HeapCorruptCheckMode == HEAPCHECKMODE_REMOVE && param != HEAPCHECKMODE_REMOVE) {
+        old_free_hook_memcorrupt = __free_hook;
+        __free_hook = me_corrupt_free_hook;
+        old_malloc_hook_memcorrupt = __malloc_hook;
+        __malloc_hook=me_corrupt_malloc_hook;
+        old_realloc_hook_memcorrupt = __realloc_hook;
+        __realloc_hook = me_corrupt_realloc_hook;
+        old_memalign_hook_memcorrupt = __memalign_hook;
+        __memalign_hook = me_corrupt_memalign_hook;
+        std::cout << "Heap corruption detector installed" << std::endl;
+    }
+
+    // This is deinstall..
+    if (HeapCorruptCheckMode != HEAPCHECKMODE_REMOVE && param == HEAPCHECKMODE_REMOVE) {
+        __free_hook = old_free_hook_memcorrupt;
+        __malloc_hook = old_malloc_hook_memcorrupt;
+        __realloc_hook = old_realloc_hook_memcorrupt;
+        __memalign_hook = old_memalign_hook_memcorrupt;
+        std::cout << "Heap corruption detector removed" << std::endl;
+    }
+
+    HeapCorruptCheckMode = param;
+    pthread_mutex_unlock(&HeapCorruptCheckMutex);
+}
+
+
+
+
+void checkHeapNow(void)
+{
+    pthread_mutex_lock(&HeapCorruptCheckMutex);
+    heapwalker();
+    pthread_mutex_unlock(&HeapCorruptCheckMutex);
+}
+
+
+//---------------------------------
+// Part 3: glibc's mcheck librrary
+
+static void mcheckabortfunc(enum mcheck_status err)
+{
+    HeapCorrupt("mcheckabortfunc",NULL,err);
+}
+
+
+void enableMCheck(void)
+{
+    mcheck(&mcheckabortfunc);
+}
+
+
+//---------------------------------
+// Part 4: error report & exit
+
+static void HeapDump(const char *info,const char *p,int cnt)
+{
+    char Legend[150];
+    snprintf(Legend,sizeof(Legend),"Legend: Magic (tail) [%02x]. allocated, but unused: [%02x]. Reclaimed to free pool: [%02x]",
+             (unsigned char)MAGICBYTE,
+             (unsigned char)MALLOCFLOOD,
+             (unsigned char)FREEFLOOD);
+
+    std::cout << Legend << std::endl;
+    std::cout << "Buffer dump: " <<  info << std::endl;
+
+    for (int i = 0;i < cnt;i++) {
+        char c = p[i];
+        if (isgraph(c) || c == ' ') {
+            std::cout << c;
+        } else {
+            char Tmp[10];
+            snprintf(Tmp,sizeof(Tmp),"[%02x]",(unsigned char)c);
+            std::cout << Tmp;
+        }
+    }
+    std::cout << std::endl;
+}
+
+
+static void HeapCorrupt(const char * caller, const struct elemheader *hdr, enum mcheck_status err)
+{
+
+    bool AbortNow = true;
+
+#define DUMPSIZE 100
+
+    if (!caller) {
+        caller = "?";
+    }
+    switch (err) {
+    case MCHECK_DISABLED:
+        std::cout << caller
+                  << ": Heap check was not called before the first allocation. No consistency checking can be done."
+                  << std::endl;
+        break;
+
+    case MCHECK_OK:
+        std::cout << caller
+                  << ": No inconsistency detected."
+                  << std::endl;
+        AbortNow = false;
+        break;
+
+    case MCHECK_HEAD:
+        std::cout << caller
+                  <<  ": The data immediately before the block was modified. This commonly happens when an array index or pointer is decremented too far."
+                  << std::endl
+                  << "  This block is ignored (and may appear as memory leakage)."
+                  << std::endl;
+
+        if (hdr) {
+            HeapDump("Immediately before ",((const char *)hdr) - (DUMPSIZE/2) ,DUMPSIZE/2);
+            HeapDump("The element in question ",(const char *)hdr,DUMPSIZE/2);
+        }
+        AbortNow = false;
+        break;
+
+    case MCHECK_TAIL:
+        std::cout << caller
+                  << ": The data immediately after the block was modified. This commonly happens when an array index or pointer is incremented too far."
+                  << std::endl;
+
+        if (hdr) {
+            HeapDump("Tail of element in question: ", ((const char *)hdr) + sizeof(struct elemheader) +  hdr->size + 1 - (DUMPSIZE/2),(DUMPSIZE/2));
+            HeapDump("Immediately after element: ",((const char *)hdr) + sizeof(struct elemheader) + hdr->size + 1 + 1, (DUMPSIZE/2));
+        }
+
+        break;
+
+    case MCHECK_FREE:
+        std::cout << caller
+                  << ":  The block was already freed."
+                  << std::endl;
+        break;
+
+
+    default:
+        std::cout << caller
+                  << ":  Unexpected event " << err
+                  << std::endl;
+        break;
+    }
+
+
+    if (hdr) {
+        std::cout << caller << ": The element size was " << hdr->size << " bytes." << std::endl;
+        if (err == MCHECK_HEAD) {
+            std::cout << caller << ": But size may be incorrect since other parts of the header is destroyed/undecodable." << std::endl;
+        }
+    }
+
+    if (AbortNow) {
+        std::cout << caller << ": Application will crash now." << std::flush;
+
+        // No core dump? Check ulimit -c
+
+        kill(getpid(), SIGSEGV);
+    }
+
+}
+
+// Enable this one if you want the heapdebugger to be installed "before anything else"
+// Uses same method as -lmcheck
+#if 0
+
+/* This one is called from malloc init.. */
+
+static void MallocInitHook(void)
+{
+    enableHeapUsageMonitor(0);
+    enableHeapCorruptCheck(0);
+}
+
+void (*__malloc_initialize_hook) (void) = MallocInitHook;
+
+
+#endif
diff --git a/document/src/tests/heapdebuggerother.cpp b/document/src/tests/heapdebuggerother.cpp
new file mode 100644
index 00000000000..641951f1883
--- /dev/null
+++ b/document/src/tests/heapdebuggerother.cpp
@@ -0,0 +1,44 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ *
+ * @author Ove Martin Malm
+ * @date Creation date: 2000-21-08
+ * @version $Id$
+ *
+ * Copyright (c) : 1997-2000 Fast Search & Transfer ASA
+ * ALL RIGHTS RESERVED
+ */
+
+#include <vespa/fastos/fastos.h>
+#include <stdlib.h>
+#include "heapdebugger.h"
+
+// Not implemented
+void enableHeapUsageMonitor(int param)
+{
+  (void)param;
+
+}
+
+// Not implemented
+extern size_t getHeapUsage(void)
+{
+  return 0;
+}
+
+// Not implemented
+void enableHeapCorruptCheck(int param)
+{
+  (void)param;
+
+}
+
+// Not implemented
+void enableMCheck(void)
+{
+}
+
+// Not implemented
+void checkHeapNow(void)
+{
+}
diff --git a/document/src/tests/orderingselectortest.cpp b/document/src/tests/orderingselectortest.cpp
new file mode 100644
index 00000000000..b9cf7dc358a
--- /dev/null
+++ b/document/src/tests/orderingselectortest.cpp
@@ -0,0 +1,98 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/document/select/orderingselector.h>
+#include <vespa/document/bucket/bucketidfactory.h>
+#include <vespa/document/base/testdocrepo.h>
+
+#include <vespa/vdstestlib/cppunit/macros.h>
+#include <vespa/document/select/parser.h>
+#include <memory>
+
+using document::select::Node;
+using document::select::Parser;
+
+namespace document {
+
+struct OrderingSelectorTest : public CppUnit::TestFixture {
+    void testSimple();
+
+    CPPUNIT_TEST_SUITE(OrderingSelectorTest);
+    CPPUNIT_TEST(testSimple);
+    CPPUNIT_TEST_SUITE_END();
+};
+
+CPPUNIT_TEST_SUITE_REGISTRATION(OrderingSelectorTest);
+
+#define ASSERT_MATCH(expression, ordering, correct)     \
+{ \
+    BucketIdFactory idfactory; \
+    TestDocRepo repo; \
+    OrderingSelector selector;                                \
+    Parser parser(repo.getTypeRepo(), idfactory); \
+    std::unique_ptr<Node> node(parser.parse(expression)); \
+    CPPUNIT_ASSERT(node.get() != 0); \
+    OrderingSpecification::UP spec = selector.select(*node, ordering); \
+    if (spec.get() == NULL && correct.get() == NULL) { \
+        return;\
+    }\
+    if (spec.get() == NULL && correct.get() != NULL) { \
+        CPPUNIT_ASSERT_MSG(std::string("Was NULL, expected ") + correct->toString(), false); \
+    } \
+    if (correct.get() == NULL && spec.get() != NULL) { \
+        CPPUNIT_ASSERT_MSG(std::string("Expected NULL, was ") + spec->toString(), false); \
+    } \
+    CPPUNIT_ASSERT_EQUAL(*correct, *spec); \
+}
+
+void OrderingSelectorTest::testSimple()
+{
+    ASSERT_MATCH("id.order(10,10) < 100", OrderingSpecification::DESCENDING,
+                        OrderingSpecification::UP(
+                                new OrderingSpecification(OrderingSpecification::DESCENDING, (long)99, (short)10, (short)10)));
+
+    ASSERT_MATCH("id.order(10,10) <= 100", OrderingSpecification::DESCENDING,
+                        OrderingSpecification::UP(
+                                new OrderingSpecification(OrderingSpecification::DESCENDING, (long)100, (short)10, (short)10)));
+
+    ASSERT_MATCH("id.order(10,10) > 100", OrderingSpecification::DESCENDING, OrderingSpecification::UP());
+
+    ASSERT_MATCH("id.order(10,10) > 100", OrderingSpecification::ASCENDING,
+                        OrderingSpecification::UP(
+                                new OrderingSpecification(OrderingSpecification::ASCENDING, (long)101, (short)10, (short)10)));
+
+    ASSERT_MATCH("id.user==1234 AND id.order(10,10) > 100", OrderingSpecification::ASCENDING,
+                        OrderingSpecification::UP(
+                                new OrderingSpecification(OrderingSpecification::ASCENDING, (long)101, (short)10, (short)10)));
+
+    ASSERT_MATCH("id.order(10,10) >= 100", OrderingSpecification::ASCENDING,
+                        OrderingSpecification::UP(
+                                new OrderingSpecification(OrderingSpecification::ASCENDING, (long)100, (short)10, (short)10)));
+
+    ASSERT_MATCH("id.order(10,10) == 100", OrderingSpecification::ASCENDING,
+                        OrderingSpecification::UP(
+                                new OrderingSpecification(OrderingSpecification::ASCENDING, (long)100, (short)10, (short)10)));
+
+    ASSERT_MATCH("id.order(10,10) = 100", OrderingSpecification::DESCENDING,
+                        OrderingSpecification::UP(
+                                new OrderingSpecification(OrderingSpecification::DESCENDING, (long)100, (short)10, (short)10)));
+
+    ASSERT_MATCH("id.order(10,10) > 30 AND id.order(10,10) < 100", OrderingSpecification::ASCENDING,
+                        OrderingSpecification::UP(
+                                new OrderingSpecification(OrderingSpecification::ASCENDING, (long)31, (short)10, (short)10)));
+
+    ASSERT_MATCH("id.order(10,10) > 30 AND id.order(10,10) < 100", OrderingSpecification::DESCENDING,
+                        OrderingSpecification::UP(
+                                new OrderingSpecification(OrderingSpecification::DESCENDING, (long)99, (short)10, (short)10)));
+
+    ASSERT_MATCH("id.order(10,10) > 30 OR id.order(10,10) > 70", OrderingSpecification::ASCENDING,
+                        OrderingSpecification::UP(
+                                new OrderingSpecification(OrderingSpecification::ASCENDING, (long)31, (short)10, (short)10)));
+
+    ASSERT_MATCH("id.order(10,10) < 30 OR id.order(10,10) < 70", OrderingSpecification::DESCENDING,
+                        OrderingSpecification::UP(
+                                new OrderingSpecification(OrderingSpecification::DESCENDING, (long)69, (short)10, (short)10)));
+}
+
+
+}
diff --git a/document/src/tests/positiontypetest.cpp b/document/src/tests/positiontypetest.cpp
new file mode 100644
index 00000000000..33398e9fa00
--- /dev/null
+++ b/document/src/tests/positiontypetest.cpp
@@ -0,0 +1,53 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/document/datatype/positiondatatype.h>
+#include <cppunit/extensions/HelperMacros.h>
+
+namespace document {
+
+class PositionTypeTest : public CppUnit::TestFixture {
+public:
+    void requireThatNameIsCorrect();
+    void requireThatExpectedFieldsAreThere();
+    void requireThatZCurveFieldMatchesJava();
+
+    CPPUNIT_TEST_SUITE(PositionTypeTest);
+    CPPUNIT_TEST(requireThatNameIsCorrect);
+    CPPUNIT_TEST(requireThatExpectedFieldsAreThere);
+    CPPUNIT_TEST(requireThatZCurveFieldMatchesJava);
+    CPPUNIT_TEST_SUITE_END();
+};
+
+CPPUNIT_TEST_SUITE_REGISTRATION(PositionTypeTest);
+
+void
+PositionTypeTest::requireThatNameIsCorrect()
+{
+    const StructDataType &type = PositionDataType::getInstance();
+    CPPUNIT_ASSERT_EQUAL(vespalib::string("position"), type.getName());
+}
+
+void
+PositionTypeTest::requireThatExpectedFieldsAreThere()
+{
+    const StructDataType &type = PositionDataType::getInstance();
+    Field field = type.getField("x");
+    CPPUNIT_ASSERT_EQUAL(*DataType::INT, field.getDataType());
+
+    field = type.getField("y");
+    CPPUNIT_ASSERT_EQUAL(*DataType::INT, field.getDataType());
+}
+
+void
+PositionTypeTest::requireThatZCurveFieldMatchesJava()
+{
+    CPPUNIT_ASSERT_EQUAL(vespalib::string("foo_zcurve"),
+                         PositionDataType::getZCurveFieldName("foo"));
+    CPPUNIT_ASSERT( ! PositionDataType::isZCurveFieldName("foo"));
+    CPPUNIT_ASSERT( ! PositionDataType::isZCurveFieldName("_zcurve"));
+    CPPUNIT_ASSERT( PositionDataType::isZCurveFieldName("x_zcurve"));
+    CPPUNIT_ASSERT( ! PositionDataType::isZCurveFieldName("x_zcurvex"));
+    CPPUNIT_ASSERT_EQUAL(vespalib::stringref("x"), PositionDataType::cutZCurveFieldName("x_zcurve"));
+}
+
+} // document
diff --git a/document/src/tests/predicate/.gitignore b/document/src/tests/predicate/.gitignore
new file mode 100644
index 00000000000..219b478bd6d
--- /dev/null
+++ b/document/src/tests/predicate/.gitignore
@@ -0,0 +1,7 @@
+*.So
+*_test
+.depend
+Makefile
+document_predicate_builder_test_app
+document_predicate_printer_test_app
+document_predicate_test_app
diff --git a/document/src/tests/predicate/CMakeLists.txt b/document/src/tests/predicate/CMakeLists.txt
new file mode 100644
index 00000000000..445da128498
--- /dev/null
+++ b/document/src/tests/predicate/CMakeLists.txt
@@ -0,0 +1,28 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(document_predicate_test_app
+    SOURCES
+    predicate_test.cpp
+    DEPENDS
+    document
+    AFTER
+    document_documentconfig
+)
+vespa_add_test(NAME document_predicate_test_app COMMAND document_predicate_test_app)
+vespa_add_executable(document_predicate_builder_test_app
+    SOURCES
+    predicate_builder_test.cpp
+    DEPENDS
+    document
+    AFTER
+    document_documentconfig
+)
+vespa_add_test(NAME document_predicate_builder_test_app COMMAND document_predicate_builder_test_app)
+vespa_add_executable(document_predicate_printer_test_app
+    SOURCES
+    predicate_printer_test.cpp
+    DEPENDS
+    document
+    AFTER
+    document_documentconfig
+)
+vespa_add_test(NAME document_predicate_printer_test_app COMMAND document_predicate_printer_test_app)
diff --git a/document/src/tests/predicate/predicate_builder_test.cpp b/document/src/tests/predicate/predicate_builder_test.cpp
new file mode 100644
index 00000000000..1e32aa2cc51
--- /dev/null
+++ b/document/src/tests/predicate/predicate_builder_test.cpp
@@ -0,0 +1,81 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Unit tests for predicate_builder.
+
+#include <vespa/log/log.h>
+LOG_SETUP("predicate_builder_test");
+#include <vespa/fastos/fastos.h>
+
+#include <vespa/document/predicate/predicate.h>
+#include <vespa/document/predicate/predicate_builder.h>
+#include <vespa/vespalib/data/slime/slime.h>
+#include <vespa/vespalib/testkit/testapp.h>
+
+using std::string;
+using vespalib::Slime;
+using vespalib::slime::Cursor;
+using namespace document;
+
+namespace {
+
+TEST("require that a predicate tree can be built from a slime object") {
+    const string feature_name = "feature name";
+    Slime input;
+    Cursor &obj = input.setObject();
+    obj.setLong(Predicate::NODE_TYPE, Predicate::TYPE_DISJUNCTION);
+    Cursor &children = obj.setArray(Predicate::CHILDREN);
+    {
+        Cursor &child = children.addObject();
+        child.setLong(Predicate::NODE_TYPE, Predicate::TYPE_FEATURE_SET);
+        child.setString(Predicate::KEY, feature_name);
+        Cursor &arr = child.setArray(Predicate::SET);
+        arr.addString("foo");
+        arr.addString("bar");
+    }
+    {
+        Cursor &child = children.addObject();
+        child.setLong(Predicate::NODE_TYPE, Predicate::TYPE_CONJUNCTION);
+        Cursor &and_children = child.setArray(Predicate::CHILDREN);
+        {
+            Cursor &and_child = and_children.addObject();
+            and_child.setLong(Predicate::NODE_TYPE,
+                              Predicate::TYPE_FEATURE_RANGE);
+            and_child.setString(Predicate::KEY, feature_name);
+            and_child.setLong(Predicate::RANGE_MIN, 42);
+        }
+        {
+            Cursor &and_child = and_children.addObject();
+            and_child.setLong(Predicate::NODE_TYPE, Predicate::TYPE_NEGATION);
+            Cursor &not_child =
+                and_child.setArray(Predicate::CHILDREN).addObject();
+            {
+                not_child.setLong(Predicate::NODE_TYPE,
+                                  Predicate::TYPE_FEATURE_SET);
+                not_child.setString(Predicate::KEY, feature_name);
+                Cursor &arr = not_child.setArray(Predicate::SET);
+                arr.addString("baz");
+                arr.addString("qux");
+            }
+        }
+    }
+
+    PredicateNode::UP node = PredicateBuilder().build(input.get());
+    Disjunction *disjunction = dynamic_cast<Disjunction *>(node.get());
+    ASSERT_TRUE(disjunction);
+    ASSERT_EQUAL(2u, disjunction->getSize());
+    FeatureSet *feature_set = dynamic_cast<FeatureSet *>((*disjunction)[0]);
+    ASSERT_TRUE(feature_set);
+    Conjunction *conjunction = dynamic_cast<Conjunction *>((*disjunction)[1]);
+    ASSERT_TRUE(conjunction);
+    ASSERT_EQUAL(2u, conjunction->getSize());
+    FeatureRange *feature_range =
+        dynamic_cast<FeatureRange *>((*conjunction)[0]);
+    ASSERT_TRUE(feature_range);
+    Negation *negation = dynamic_cast<Negation *>((*conjunction)[1]);
+    ASSERT_TRUE(negation);
+    feature_set = dynamic_cast<FeatureSet *>(&negation->getChild());
+    ASSERT_TRUE(feature_set);
+}
+
+}  // namespace
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/document/src/tests/predicate/predicate_printer_test.cpp b/document/src/tests/predicate/predicate_printer_test.cpp
new file mode 100644
index 00000000000..8525fb8e2db
--- /dev/null
+++ b/document/src/tests/predicate/predicate_printer_test.cpp
@@ -0,0 +1,122 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Unit tests for predicate_printer.
+
+#include <vespa/log/log.h>
+LOG_SETUP("predicate_printer_test");
+#include <vespa/fastos/fastos.h>
+
+#include <vespa/document/predicate/predicate.h>
+#include <vespa/document/predicate/predicate_printer.h>
+#include <vespa/document/predicate/predicate_slime_builder.h>
+#include <vespa/vespalib/testkit/testapp.h>
+
+using vespalib::Slime;
+using vespalib::slime::Cursor;
+using namespace document;
+
+namespace {
+
+typedef std::unique_ptr<Slime> SlimeUP;
+using namespace document::predicate_slime_builder;
+
+TEST("require that PredicatePrinter prints FeatureSets") {
+    PredicateSlimeBuilder builder;
+    builder.feature("foo").value("bar").value("baz");
+    ASSERT_EQUAL("'foo' in ['bar','baz']",
+                 PredicatePrinter::print(*builder.build()));
+
+    builder.feature("foo").value("bar");
+    ASSERT_EQUAL("'foo' in ['bar']",
+                 PredicatePrinter::print(*builder.build()));
+}
+
+TEST("require that PredicatePrinter escapes non-ascii characters") {
+    PredicateSlimeBuilder builder;
+    builder.feature("\n\t\001'").value("\xc3\xb8");
+    ASSERT_EQUAL("'\\n\\t\\x01\\x27' in ['\\xc3\\xb8']",
+                 PredicatePrinter::print(*builder.build()));
+}
+
+TEST("require that PredicatePrinter prints FeatureRanges") {
+    PredicateSlimeBuilder builder;
+    builder.feature("foo").range(-10, 42);
+    ASSERT_EQUAL("'foo' in [-10..42]",
+                 PredicatePrinter::print(*builder.build()));
+}
+
+TEST("require that PredicatePrinter prints open ended FeatureRanges") {
+    PredicateSlimeBuilder builder;
+    builder.feature("foo").greaterEqual(-10);
+    ASSERT_EQUAL("'foo' in [-10..]",
+                 PredicatePrinter::print(*builder.build()));
+
+    builder.feature("foo").lessEqual(42);
+    ASSERT_EQUAL("'foo' in [..42]", PredicatePrinter::print(*builder.build()));
+}
+
+TEST("require that PredicatePrinter prints NOT_IN FeatureSets") {
+    PredicateSlimeBuilder builder;
+    builder.neg().feature("foo").value("bar").value("baz");
+    ASSERT_EQUAL("'foo' not in ['bar','baz']",
+                 PredicatePrinter::print(*builder.build()));
+}
+
+TEST("require that PredicatePrinter can negate FeatureRanges") {
+    auto slime = neg(featureRange("foo", -10, 42));
+    ASSERT_EQUAL("'foo' not in [-10..42]", PredicatePrinter::print(*slime));
+}
+
+TEST("require that PredicatePrinter can negate open ended FeatureRanges") {
+    auto slime = neg(greaterEqual("foo", 42));
+    ASSERT_EQUAL("'foo' not in [42..]", PredicatePrinter::print(*slime));
+
+    slime = neg(lessEqual("foo", 42));
+    ASSERT_EQUAL("'foo' not in [..42]", PredicatePrinter::print(*slime));
+}
+
+TEST("require that PredicatePrinter can negate double open ended ranges") {
+    auto slime = neg(emptyRange("foo"));
+    ASSERT_EQUAL("'foo' not in [..]", PredicatePrinter::print(*slime));
+}
+
+TEST("require that PredicatePrinter prints AND expressions") {
+    auto slime = andNode({featureSet("foo", {"bar", "baz"}),
+                          featureSet("foo", {"bar", "baz"})});
+    ASSERT_EQUAL("('foo' in ['bar','baz'] and 'foo' in ['bar','baz'])",
+                 PredicatePrinter::print(*slime));
+}
+
+TEST("require that PredicatePrinter prints OR expressions") {
+    auto slime = orNode({featureSet("foo", {"bar", "baz"}),
+                         featureSet("foo", {"bar", "baz"})});
+    ASSERT_EQUAL("('foo' in ['bar','baz'] or 'foo' in ['bar','baz'])",
+                 PredicatePrinter::print(*slime));
+}
+
+TEST("require that PredicatePrinter can negate OR expressions") {
+    auto slime = neg(orNode({featureSet("foo", {"bar", "baz"}),
+                             featureSet("foo", {"bar", "baz"})}));
+    ASSERT_EQUAL("not ('foo' in ['bar','baz'] or 'foo' in ['bar','baz'])",
+                 PredicatePrinter::print(*slime));
+}
+
+TEST("require that PredicatePrinter can negate AND expressions") {
+    auto slime = neg(andNode({featureSet("foo", {"bar", "baz"}),
+                              featureSet("foo", {"bar", "baz"})}));
+    ASSERT_EQUAL("not ('foo' in ['bar','baz'] and 'foo' in ['bar','baz'])",
+                 PredicatePrinter::print(*slime));
+}
+
+TEST("require that PredicatePrinter prints True") {
+    auto slime = truePredicate();
+    ASSERT_EQUAL("true", PredicatePrinter::print(*slime));
+}
+
+TEST("require that PredicatePrinter prints False") {
+    auto slime = falsePredicate();
+    ASSERT_EQUAL("false", PredicatePrinter::print(*slime));
+}
+
+}  // namespace
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/document/src/tests/predicate/predicate_test.cpp b/document/src/tests/predicate/predicate_test.cpp
new file mode 100644
index 00000000000..3223b628c6a
--- /dev/null
+++ b/document/src/tests/predicate/predicate_test.cpp
@@ -0,0 +1,248 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Unit tests for predicate.
+
+#include <vespa/log/log.h>
+LOG_SETUP("predicate_test");
+#include <vespa/fastos/fastos.h>
+
+#include <vespa/document/predicate/predicate.h>
+#include <vespa/vespalib/data/slime/slime.h>
+#include <vespa/vespalib/testkit/testapp.h>
+
+#include <vespa/document/predicate/predicate_slime_builder.h>
+#include <string>
+
+using std::string;
+using std::vector;
+using vespalib::Slime;
+using vespalib::slime::Cursor;
+using namespace document;
+
+namespace {
+
+typedef std::unique_ptr<Slime> SlimeUP;
+
+TEST("require that predicate feature set slimes can be compared") {
+    PredicateSlimeBuilder builder;
+    builder.feature("foo").value("bar").value("baz");
+    SlimeUP s1 = builder.build();
+    builder.feature("foo").value("baz").value("bar");
+    ASSERT_EQUAL(0, Predicate::compare(*s1, *builder.build()));
+
+    builder.feature("bar").value("baz").value("bar");
+    ASSERT_EQUAL(1, Predicate::compare(*s1, *builder.build()));
+    builder.feature("qux").value("baz").value("bar");
+    ASSERT_EQUAL(-1, Predicate::compare(*s1, *builder.build()));
+
+    builder.feature("foo").value("baz");
+    ASSERT_EQUAL(1, Predicate::compare(*s1, *builder.build()));
+    builder.feature("foo").value("baz").value("qux").value("quux");
+    ASSERT_EQUAL(-1, Predicate::compare(*s1, *builder.build()));
+
+    builder.feature("foo").value("baz").value("qux");
+    ASSERT_EQUAL(-1, Predicate::compare(*s1, *builder.build()));
+    builder.feature("foo").value("baz").value("aaa");
+    ASSERT_EQUAL(1, Predicate::compare(*s1, *builder.build()));
+}
+
+TEST("require that predicate feature range slimes can be compared") {
+    PredicateSlimeBuilder builder;
+    builder.feature("foo").range(0, 10);
+    SlimeUP s1 = builder.build();
+    builder.feature("foo").range(0, 10);
+    ASSERT_EQUAL(0, Predicate::compare(*s1, *builder.build()));
+
+    builder.feature("foo").range(-1, 10);
+    ASSERT_EQUAL(1, Predicate::compare(*s1, *builder.build()));
+    builder.feature("foo").range(1, 10);
+    ASSERT_EQUAL(-1, Predicate::compare(*s1, *builder.build()));
+
+    builder.feature("foo").range(0, 9);
+    ASSERT_EQUAL(1, Predicate::compare(*s1, *builder.build()));
+    builder.feature("foo").range(0, 11);
+    ASSERT_EQUAL(-1, Predicate::compare(*s1, *builder.build()));
+
+    builder.feature("foo").greaterEqual(0);
+    ASSERT_EQUAL(-1, Predicate::compare(*s1, *builder.build()));
+    builder.feature("foo").lessEqual(10);
+    ASSERT_EQUAL(-1, Predicate::compare(*s1, *builder.build()));
+}
+
+TEST("require that predicate open feature range slimes can be compared") {
+    PredicateSlimeBuilder builder;
+    builder.feature("foo").greaterEqual(10);
+    SlimeUP s1 = builder.build();
+    builder.feature("foo").greaterEqual(10);
+    ASSERT_EQUAL(0, Predicate::compare(*s1, *builder.build()));
+
+    builder.feature("foo").greaterEqual(9);
+    ASSERT_EQUAL(1, Predicate::compare(*s1, *builder.build()));
+    builder.feature("foo").greaterEqual(11);
+    ASSERT_EQUAL(-1, Predicate::compare(*s1, *builder.build()));
+
+    builder.feature("foo").lessEqual(10);
+    ASSERT_EQUAL(-1, Predicate::compare(*s1, *builder.build()));
+}
+
+TEST("require that predicate 'not' slimes can be compared") {
+    PredicateSlimeBuilder builder;
+    builder.neg().feature("foo").range(0, 10);
+    SlimeUP s1 = builder.build();
+    builder.neg().feature("foo").range(0, 10);
+    ASSERT_EQUAL(0, Predicate::compare(*s1, *builder.build()));
+
+    builder.neg().feature("foo").range(0, 11);
+    ASSERT_EQUAL(-1, Predicate::compare(*s1, *builder.build()));
+
+    builder.feature("foo").range(0, 10);
+    ASSERT_EQUAL(-1, Predicate::compare(*s1, *builder.build()));
+}
+
+TEST("require that predicate 'and' slimes can be compared") {
+    PredicateSlimeBuilder builder;
+    SlimeUP s1 = builder.feature("foo").value("bar").value("baz").build();
+    SlimeUP s2 = builder.feature("foo").value("bar").value("qux").build();
+    SlimeUP and_node = builder.and_node(std::move(s1), std::move(s2)).build();
+
+    s1 = builder.feature("foo").value("bar").value("baz").build();
+    s2 = builder.feature("foo").value("bar").value("qux").build();
+    builder.and_node(std::move(s1), std::move(s2));
+    ASSERT_EQUAL(0, Predicate::compare(*and_node, *builder.build()));
+
+    s1 = builder.feature("foo").value("bar").value("baz").build();
+    s2 = builder.feature("foo").value("bar").value("qux").build();
+    builder.and_node(std::move(s2), std::move(s1));
+    ASSERT_EQUAL(-1, Predicate::compare(*and_node, *builder.build()));
+}
+
+TEST("require that predicate 'or' slimes can be compared") {
+    PredicateSlimeBuilder builder;
+    SlimeUP s1 = builder.feature("foo").value("bar").value("baz").build();
+    SlimeUP s2 = builder.feature("foo").value("bar").value("qux").build();
+    SlimeUP or_node = builder.or_node(std::move(s1), std::move(s2)).build();
+
+    s1 = builder.feature("foo").value("bar").value("baz").build();
+    s2 = builder.feature("foo").value("bar").value("qux").build();
+    builder.or_node(std::move(s1), std::move(s2));
+    ASSERT_EQUAL(0, Predicate::compare(*or_node, *builder.build()));
+
+    s1 = builder.feature("foo").value("bar").value("baz").build();
+    s2 = builder.feature("foo").value("bar").value("qux").build();
+    builder.or_node(std::move(s2), std::move(s1));
+    ASSERT_EQUAL(-1, Predicate::compare(*or_node, *builder.build()));
+}
+
+TEST("require that predicate 'true' slimes can be compared") {
+    PredicateSlimeBuilder builder;
+    builder.true_predicate();
+    SlimeUP s1 = builder.build();
+    builder.true_predicate();
+    ASSERT_EQUAL(0, Predicate::compare(*s1, *builder.build()));
+
+    builder.false_predicate();
+    ASSERT_EQUAL(-1, Predicate::compare(*s1, *builder.build()));
+}
+
+TEST("require that predicate 'false' slimes can be compared") {
+    PredicateSlimeBuilder builder;
+    builder.false_predicate();
+    SlimeUP s1 = builder.build();
+    builder.false_predicate();
+    ASSERT_EQUAL(0, Predicate::compare(*s1, *builder.build()));
+
+    builder.true_predicate();
+    ASSERT_EQUAL(1, Predicate::compare(*s1, *builder.build()));
+}
+
+TEST("require that feature set can be created") {
+    const string feature_name = "feature name";
+    Slime input;
+    Cursor &obj = input.setObject();
+    obj.setString(Predicate::KEY, feature_name);
+    Cursor &arr = obj.setArray(Predicate::SET);
+    arr.addString("foo");
+    arr.addString("bar");
+    FeatureSet set(input.get());
+    EXPECT_EQUAL(feature_name, set.getKey());
+    ASSERT_EQUAL(2u, set.getSize());
+    EXPECT_EQUAL("foo", set[0]);
+    EXPECT_EQUAL("bar", set[1]);
+}
+
+TEST("require that feature range can be created") {
+    const string feature_name = "feature name";
+    const long min = 0;
+    const long max = 42;
+    Slime input;
+    Cursor &obj = input.setObject();
+    obj.setString(Predicate::KEY, feature_name);
+    obj.setLong(Predicate::RANGE_MIN, min);
+    obj.setLong(Predicate::RANGE_MAX, max);
+    FeatureRange set(input.get());
+    EXPECT_EQUAL(feature_name, set.getKey());
+    EXPECT_TRUE(set.hasMin());
+    EXPECT_TRUE(set.hasMax());
+    EXPECT_EQUAL(min, set.getMin());
+    EXPECT_EQUAL(max, set.getMax());
+}
+
+TEST("require that feature range can be open") {
+    const string feature_name = "feature name";
+    Slime input;
+    Cursor &obj = input.setObject();
+    obj.setString(Predicate::KEY, feature_name);
+    FeatureRange set(input.get());
+    EXPECT_EQUAL(feature_name, set.getKey());
+    EXPECT_FALSE(set.hasMin());
+    EXPECT_FALSE(set.hasMax());
+    EXPECT_EQUAL(LLONG_MIN, set.getMin());
+    EXPECT_EQUAL(LLONG_MAX, set.getMax());
+}
+
+PredicateNode::UP getPredicateNode() {
+    const string feature_name = "feature name";
+    Slime input;
+    Cursor &obj = input.setObject();
+    obj.setString(Predicate::KEY, feature_name);
+    Cursor &arr = obj.setArray(Predicate::SET);
+    arr.addString("foo");
+    arr.addString("bar");
+
+    PredicateNode::UP node(new FeatureSet(input.get()));
+    return node;
+}
+
+TEST("require that negation nodes holds a child") {
+    PredicateNode::UP node(getPredicateNode());
+    PredicateNode *expected = node.get();
+    Negation neg(std::move(node));
+
+    EXPECT_EQUAL(expected, &neg.getChild());
+}
+
+TEST("require that conjunction nodes holds several children") {
+    vector<PredicateNode *> nodes;
+    nodes.push_back(getPredicateNode().release());
+    nodes.push_back(getPredicateNode().release());
+    Conjunction and_node(nodes);
+
+    ASSERT_EQUAL(2u, and_node.getSize());
+    EXPECT_EQUAL(nodes[0], and_node[0]);
+    EXPECT_EQUAL(nodes[1], and_node[1]);
+}
+
+TEST("require that disjunction nodes holds several children") {
+    vector<PredicateNode *> nodes;
+    nodes.push_back(getPredicateNode().release());
+    nodes.push_back(getPredicateNode().release());
+    Disjunction or_node(nodes);
+
+    ASSERT_EQUAL(2u, or_node.getSize());
+    EXPECT_EQUAL(nodes[0], or_node[0]);
+    EXPECT_EQUAL(nodes[1], or_node[1]);
+}
+
+
+}  // namespace
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/document/src/tests/primitivefieldvaluetest.cpp b/document/src/tests/primitivefieldvaluetest.cpp
new file mode 100644
index 00000000000..90e51ec071d
--- /dev/null
+++ b/document/src/tests/primitivefieldvaluetest.cpp
@@ -0,0 +1,422 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/document/fieldvalue/fieldvalues.h>
+#include <vespa/document/serialization/vespadocumentdeserializer.h>
+#include <vespa/vdstestlib/cppunit/macros.h>
+#include <vespa/vespalib/objects/nbostream.h>
+
+using vespalib::nbostream;
+
+namespace document {
+
+struct PrimitiveFieldValueTest : public CppUnit::TestFixture {
+    void setUp() {}
+    void tearDown() {}
+
+    void testLiterals();
+    void testRaw();
+    void testNumerics();
+    void testFloatDoubleCasts();
+
+    CPPUNIT_TEST_SUITE(PrimitiveFieldValueTest);
+    CPPUNIT_TEST(testLiterals);
+    CPPUNIT_TEST(testRaw);
+    CPPUNIT_TEST(testNumerics);
+    CPPUNIT_TEST(testFloatDoubleCasts);
+    CPPUNIT_TEST_SUITE_END();
+};
+
+CPPUNIT_TEST_SUITE_REGISTRATION(PrimitiveFieldValueTest);
+
+namespace {
+template <typename T>
+void deserialize(const ByteBuffer &buffer, T &value) {
+    uint16_t version = Document::getNewestSerializationVersion();
+    nbostream stream(buffer.getBufferAtPos(), buffer.getRemaining());
+    DocumentTypeRepo repo;
+    VespaDocumentDeserializer deserializer(repo, stream, version);
+    deserializer.read(value);
+}
+
+    /**
+     * Test common functionality, such as serialization, comparisons, and
+     * assignment. medium1 and medium2 should be equal, but not the same
+     * instance.
+     */
+    template<typename Type>
+    void
+    testCommon(const Type& smallest, const Type& medium1,
+               const Type& medium2, const Type& largest)
+    {
+        try{
+                // Less
+            CPPUNIT_ASSERT(!(smallest < smallest));
+            CPPUNIT_ASSERT(smallest < medium1);
+            CPPUNIT_ASSERT(smallest < medium2);
+            CPPUNIT_ASSERT(smallest < largest);
+            CPPUNIT_ASSERT(!(medium1 < smallest));
+            CPPUNIT_ASSERT(!(medium1 < medium1));
+            CPPUNIT_ASSERT(!(medium1 < medium2));
+            CPPUNIT_ASSERT(medium1 < largest);
+            CPPUNIT_ASSERT(!(medium2 < smallest));
+            CPPUNIT_ASSERT(!(medium2 < medium1));
+            CPPUNIT_ASSERT(!(medium2 < medium2));
+            CPPUNIT_ASSERT(medium2 < largest);
+            CPPUNIT_ASSERT(!(largest < smallest));
+            CPPUNIT_ASSERT(!(largest < medium1));
+            CPPUNIT_ASSERT(!(largest < medium2));
+            CPPUNIT_ASSERT(!(largest < largest));
+                // Equal
+            CPPUNIT_ASSERT(smallest == smallest);
+            CPPUNIT_ASSERT(!(smallest == medium1));
+            CPPUNIT_ASSERT(!(smallest == medium2));
+            CPPUNIT_ASSERT(!(smallest == largest));
+            CPPUNIT_ASSERT(!(medium1 == smallest));
+            CPPUNIT_ASSERT(medium1 == medium1);
+            CPPUNIT_ASSERT(medium1 == medium2);
+            CPPUNIT_ASSERT(!(medium1 == largest));
+            CPPUNIT_ASSERT(!(medium2 == smallest));
+            CPPUNIT_ASSERT(medium2 == medium1);
+            CPPUNIT_ASSERT(medium2 == medium2);
+            CPPUNIT_ASSERT(!(medium2 == largest));
+            CPPUNIT_ASSERT(!(largest == smallest));
+            CPPUNIT_ASSERT(!(largest == medium1));
+            CPPUNIT_ASSERT(!(largest == medium2));
+            CPPUNIT_ASSERT(largest == largest);
+                // Greater
+            CPPUNIT_ASSERT(!(smallest > smallest));
+            CPPUNIT_ASSERT(!(smallest > medium1));
+            CPPUNIT_ASSERT(!(smallest > medium2));
+            CPPUNIT_ASSERT(!(smallest > largest));
+            CPPUNIT_ASSERT(medium1 > smallest);
+            CPPUNIT_ASSERT(!(medium1 > medium1));
+            CPPUNIT_ASSERT(!(medium1 > medium2));
+            CPPUNIT_ASSERT(!(medium1 > largest));
+            CPPUNIT_ASSERT(medium2 > smallest);
+            CPPUNIT_ASSERT(!(medium2 > medium1));
+            CPPUNIT_ASSERT(!(medium2 > medium2));
+            CPPUNIT_ASSERT(!(medium2 > largest));
+            CPPUNIT_ASSERT(largest > smallest);
+            CPPUNIT_ASSERT(largest > medium1);
+            CPPUNIT_ASSERT(largest > medium2);
+            CPPUNIT_ASSERT(!(largest > largest));
+                // Currently >=, <= and != is deducted from the above, so not
+                // checking separately
+
+                // Serialization
+            Type t;
+            std::unique_ptr<ByteBuffer> buf(smallest.serialize());
+            buf->flip();
+            deserialize(*buf, t);
+            CPPUNIT_ASSERT_EQUAL(smallest, t);
+
+            buf = medium1.serialize();
+            buf->flip();
+            deserialize(*buf, t);
+            CPPUNIT_ASSERT_EQUAL(medium1, t);
+            CPPUNIT_ASSERT_EQUAL(medium2, t);
+
+            buf = largest.serialize();
+            buf->flip();
+            deserialize(*buf, t);
+            CPPUNIT_ASSERT_EQUAL(largest, t);
+
+                // Assignment
+            CPPUNIT_ASSERT_EQUAL(smallest, t = smallest);
+            CPPUNIT_ASSERT_EQUAL(medium1, t = medium1);
+            CPPUNIT_ASSERT_EQUAL(largest, t = largest);
+
+            Type t1(smallest);
+            Type t2(medium1);
+            Type t3(medium2);
+            Type t4(largest);
+            CPPUNIT_ASSERT_EQUAL(smallest, t1);
+            CPPUNIT_ASSERT_EQUAL(medium1, t2);
+            CPPUNIT_ASSERT_EQUAL(medium2, t3);
+            CPPUNIT_ASSERT_EQUAL(largest, t4);
+
+            t.assign(smallest);
+            CPPUNIT_ASSERT_EQUAL(smallest, t);
+            t.assign(medium2);
+            CPPUNIT_ASSERT_EQUAL(medium1, t);
+            t.assign(largest);
+            CPPUNIT_ASSERT_EQUAL(largest, t);
+
+            // Catch errors and say what type there were trouble with.
+        } catch (std::exception& e) {
+            std::cerr << "\nFailed for type " << *smallest.getDataType()
+                      << "\n";
+            throw;
+        }
+    }
+
+    template<typename Literal>
+    void
+    testLiteral()
+    {
+        testCommon(Literal(),
+                   Literal("bar"),
+                   Literal("bar"),
+                   Literal("foo"));
+        Literal value("foo");
+            // Textual output
+        CPPUNIT_ASSERT_EQUAL(std::string("foo"), value.toString(false, ""));
+        CPPUNIT_ASSERT_EQUAL(std::string("foo"), value.toString(true, "  "));
+        CPPUNIT_ASSERT_EQUAL(std::string("<value>foo</value>\n"),
+                             value.toXml("  "));
+
+            // Conversion
+        CPPUNIT_ASSERT_EQUAL(typename Literal::string(value.getAsString()), value.getValue());
+
+            // Operator =
+        value = "anotherVal";
+        CPPUNIT_ASSERT_EQUAL(typename Literal::string("anotherVal"), value.getValue());
+        value = std::string("yetAnotherVal");
+        CPPUNIT_ASSERT_EQUAL(typename Literal::string("yetAnotherVal"), value.getValue());
+
+        // Test that a just deserialized value can be serialized again
+        // (literals have lazy deserialization so behaves diff then
+        value = "foo";
+        std::unique_ptr<ByteBuffer> buf(value.serialize());
+        buf->flip();
+        Literal value2("Other");
+        deserialize(*buf, value2);
+        buf = value2.serialize();
+        buf->flip();
+        deserialize(*buf, value2);
+        CPPUNIT_ASSERT_EQUAL(value, value2);
+
+        // Verify that get value ref gives us ref within original bytebuffer
+        // (operator== use above should not modify this)
+        buf = value.serialize();
+        buf->flip();
+        deserialize(*buf, value2);
+
+        CPPUNIT_ASSERT_EQUAL(size_t(3), value2.getValueRef().size());
+        // Zero termination
+        CPPUNIT_ASSERT(*(value2.getValueRef().c_str() + value2.getValueRef().size()) == '\0');
+    }
+
+}
+
+void
+PrimitiveFieldValueTest::testLiterals()
+{
+    testLiteral<StringFieldValue>();
+}
+
+void
+PrimitiveFieldValueTest::testRaw()
+{
+    testCommon(RawFieldValue(),
+               RawFieldValue("bar\0bar", 7),
+               RawFieldValue("bar\0bar", 7),
+               RawFieldValue("bar\0other", 9));
+    RawFieldValue value("\tfoo\0\r\n", 7);
+        // Textual output
+    CPPUNIT_ASSERT_EQUAL(std::string(
+            "0: 09 66 6f 6f 00 0d 0a                            .foo..."),
+            value.toString(false, ""));
+    CPPUNIT_ASSERT_EQUAL(std::string(
+            "0: 09 66 6f 6f 00 0d 0a                            .foo..."),
+            value.toString(true, "  "));
+    CPPUNIT_ASSERT_EQUAL(std::string(
+            "<value binaryencoding=\"base64\">CWZvbwANCg==</value>\n"),
+            value.toXml("  "));
+
+    value.setValue("grmpf", 4);
+    CPPUNIT_ASSERT(strncmp("grmpf", value.getValueRef().c_str(),
+                           value.getValueRef().size()) == 0);
+}
+
+#define ASSERT_FAILED_CONV(getter, totype, floating) \
+{ \
+    totype toType; \
+    FieldValue::UP copy(value.clone()); \
+    try{ \
+        getter; \
+        std::ostringstream ost; \
+        ost << "Conversion unexpectedly worked from max value of " \
+            << *value.getDataType() << " to " << *toType.getDataType(); \
+        CPPUNIT_FAIL(ost.str().c_str()); \
+    } catch (std::exception& e) { \
+        CPPUNIT_ASSERT_EQUAL( \
+                std::string("bad numeric conversion: positive overflow"), \
+                std::string(e.what())); \
+    } \
+        /* Verify that we can convert to smaller type if value is within \
+           range. Only tests integer to integer. No floating point. */ \
+    if (!floating) { \
+        totype::Number maxV = std::numeric_limits<totype::Number>::max(); \
+        value.setValue((Number) maxV); \
+        getter; \
+    } \
+    value.assign(*copy); \
+}
+
+namespace {
+
+    template<typename Numeric>
+    void
+    testNumeric(const std::string& maxVal, bool floatingPoint)
+    {
+        typedef typename Numeric::Number Number;
+        Number maxValue(std::numeric_limits<Number>::max());
+            // Test common fieldvalue stuff
+        testCommon(Numeric(),
+                   Numeric(Number(1)),
+                   Numeric(Number(1)),
+                   Numeric(Number(maxValue)));
+        Numeric value;
+        value.setValue(maxValue);
+            // Test textual output
+        CPPUNIT_ASSERT_EQUAL(maxVal, value.toString(false, ""));
+        CPPUNIT_ASSERT_EQUAL(maxVal, value.toString(true, "  "));
+        CPPUNIT_ASSERT_EQUAL("<value>" + maxVal + "</value>\n",
+                             value.toXml("  "));
+            // Test numeric conversions
+            //
+            // Currently, all safe conversion works. For instance, a byte can be
+            // converted to a long, a long can be converted to a byte, given
+            // that it has a value in the range -128 to 127.
+            //
+            // All integers will also convert automatically to floating point
+            // numbers. No checks is done as to how precise floating point
+            // representation can keep the value.
+        if (floatingPoint || sizeof(Number) > sizeof(unsigned char)) {
+            // No longer throws. This is guarded on the perimeter by java code.
+            // ASSERT_FAILED_CONV(value.getAsByte(), ByteFieldValue, floatingPoint);
+        } else {
+            CPPUNIT_ASSERT_EQUAL((char) maxValue, value.getAsByte());
+        }
+        if (floatingPoint || sizeof(Number) > sizeof(int32_t)) {
+            // No longer throws. This is guarded on the perimeter by java code.
+            // ASSERT_FAILED_CONV(value.getAsInt(), IntFieldValue, floatingPoint);
+        } else {
+            CPPUNIT_ASSERT_EQUAL((int32_t) maxValue, value.getAsInt());
+        }
+        if (floatingPoint || sizeof(Number) > sizeof(int64_t)) {
+            // No longer throws. This is guarded on the perimeter by java code.
+            // ASSERT_FAILED_CONV(value.getAsLong(), LongFieldValue, floatingPoint);
+        } else {
+            CPPUNIT_ASSERT_EQUAL((int64_t) maxValue, value.getAsLong());
+        }
+        if (floatingPoint && sizeof(Number) > sizeof(float)) {
+            // No longer throws. This is guarded on the perimeter by java code.
+            // ASSERT_FAILED_CONV(value.getAsFloat(), FloatFieldValue, true);
+        } else {
+            CPPUNIT_ASSERT_EQUAL((float) maxValue, value.getAsFloat());
+        }
+        CPPUNIT_ASSERT_EQUAL((double) maxValue, value.getAsDouble());
+        // Test some simple conversions
+        Numeric a(0);
+        a = std::string("5");
+        CPPUNIT_ASSERT_EQUAL(5, a.getAsInt());
+    }
+
+}
+
+void
+PrimitiveFieldValueTest::testFloatDoubleCasts()
+{
+    float inf(std::numeric_limits<float>::infinity());
+    CPPUNIT_ASSERT_EQUAL(inf, static_cast<float>(static_cast<double>(inf)));
+}
+
+void
+PrimitiveFieldValueTest::testNumerics()
+{
+    testNumeric<ByteFieldValue>("127", false);
+    testNumeric<ShortFieldValue>("32767", false);
+    testNumeric<IntFieldValue>("2147483647", false);
+    testNumeric<LongFieldValue>("9223372036854775807", false);
+    testNumeric<FloatFieldValue>("3.40282e+38", true);
+    testNumeric<DoubleFieldValue>("1.79769e+308", true);
+
+        // Test range
+    ByteFieldValue b1(-128);
+    ByteFieldValue b2(-1);
+    CPPUNIT_ASSERT_EQUAL(-128, (int) b1.getValue());
+    CPPUNIT_ASSERT_EQUAL(-1, (int) b2.getValue());
+
+    ShortFieldValue s1(-32768);
+    ShortFieldValue s2(65535);
+    CPPUNIT_ASSERT_EQUAL((int16_t)-32768, s1.getValue());
+    CPPUNIT_ASSERT_EQUAL((int16_t)65535, s2.getValue());
+    CPPUNIT_ASSERT_EQUAL((int16_t)-1, s2.getValue());
+
+    IntFieldValue i1(-2147483647-1);
+    IntFieldValue i2(4294967295U);
+
+    CPPUNIT_ASSERT_EQUAL((int) -2147483647-1, i1.getValue());
+    CPPUNIT_ASSERT_EQUAL((int) -1, i2.getValue());
+
+    LongFieldValue l1(-9223372036854775807ll-1);
+    LongFieldValue l2(18446744073709551615ull);
+
+    CPPUNIT_ASSERT_EQUAL((int64_t) -9223372036854775807ll-1, l1.getValue());
+    CPPUNIT_ASSERT_EQUAL((int64_t) -1, l2.getValue());
+
+    b1 = "-128";
+    b2 = "255";
+
+    CPPUNIT_ASSERT_EQUAL(-128, (int) b1.getValue());
+    CPPUNIT_ASSERT_EQUAL(-1, (int) b2.getValue());
+    i1 = "-2147483648";
+    i2 = "4294967295";
+    CPPUNIT_ASSERT_EQUAL((int) -2147483647-1, i1.getValue());
+    CPPUNIT_ASSERT_EQUAL((int) -1, i2.getValue());
+
+    l1 = "-9223372036854775808";
+    l2 = "18446744073709551615";
+
+    int64_t bnv = -1;
+    bnv <<= 63;
+    CPPUNIT_ASSERT_EQUAL(bnv, l1.getValue());
+    CPPUNIT_ASSERT_EQUAL((int64_t) -9223372036854775807ll-1, l1.getValue());
+    CPPUNIT_ASSERT_EQUAL((int64_t) -1, l2.getValue());
+
+        // Test some special cases for bytes
+        // (as unsigned char is not always handled as a number)
+    b1 = "0xff";
+    CPPUNIT_ASSERT_EQUAL(-1, (int) b1.getValue());
+    b1 = "53";
+    CPPUNIT_ASSERT_EQUAL(53, (int) b1.getValue());
+    CPPUNIT_ASSERT_EQUAL(vespalib::string("53"), b1.getAsString());
+
+    try{
+        b1 = "-129";
+        CPPUNIT_FAIL("Expected -129 to be invalid byte");
+    } catch (std::exception& e) {}
+    try{
+        b1 = "256";
+        CPPUNIT_FAIL("Expected -129 to be invalid byte");
+    } catch (std::exception& e) {}
+    try{
+        s1 = "-32769";
+        CPPUNIT_FAIL("Expected -32769 to be invalid int");
+    } catch (std::exception& e) {}
+    try{
+        s1 = "65536";
+        CPPUNIT_FAIL("Expected 65536 to be invalid int");
+    } catch (std::exception& e) {}
+    try{
+        i1 = "-2147483649";
+        CPPUNIT_FAIL("Expected -2147483649 to be invalid int");
+    } catch (std::exception& e) {}
+    try{
+        i1 = "4294967296";
+        CPPUNIT_FAIL("Expected 4294967296 to be invalid int");
+    } catch (std::exception& e) {}
+    try{
+        l1 = "-9223372036854775809";
+        CPPUNIT_FAIL("Expected -9223372036854775809 to be invalid long");
+    } catch (std::exception& e) {}
+    try{
+        l1 = "18446744073709551616";
+        CPPUNIT_FAIL("Expected 18446744073709551616 to be invalid long");
+    } catch (std::exception& e) {}
+}
+
+} // document
diff --git a/document/src/tests/repo/.gitignore b/document/src/tests/repo/.gitignore
new file mode 100644
index 00000000000..bd4e77abb07
--- /dev/null
+++ b/document/src/tests/repo/.gitignore
@@ -0,0 +1,5 @@
+*.So
+*_test
+.depend
+Makefile
+document_documenttyperepo_test_app
diff --git a/document/src/tests/repo/CMakeLists.txt b/document/src/tests/repo/CMakeLists.txt
new file mode 100644
index 00000000000..efd74caf954
--- /dev/null
+++ b/document/src/tests/repo/CMakeLists.txt
@@ -0,0 +1,10 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(document_documenttyperepo_test_app
+    SOURCES
+    documenttyperepo_test.cpp
+    DEPENDS
+    document
+    AFTER
+    document_documentconfig
+)
+vespa_add_test(NAME document_documenttyperepo_test_app COMMAND document_documenttyperepo_test_app)
diff --git a/document/src/tests/repo/documenttyperepo_test.cpp b/document/src/tests/repo/documenttyperepo_test.cpp
new file mode 100644
index 00000000000..003bd1698af
--- /dev/null
+++ b/document/src/tests/repo/documenttyperepo_test.cpp
@@ -0,0 +1,519 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Unit tests for documenttyperepo.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("documenttyperepo_test");
+
+#include <vespa/config/print/asciiconfigwriter.h>
+#include <vespa/document/config/config-documenttypes.h>
+#include <vespa/document/datatype/annotationreferencedatatype.h>
+#include <vespa/document/datatype/arraydatatype.h>
+#include <vespa/document/datatype/documenttype.h>
+#include <vespa/document/datatype/mapdatatype.h>
+#include <vespa/document/datatype/weightedsetdatatype.h>
+#include <vespa/document/repo/configbuilder.h>
+#include <vespa/document/repo/documenttyperepo.h>
+#include <stdlib.h>
+#include <vespa/vespalib/objects/identifiable.h>
+#include <vespa/vespalib/stllike/string.h>
+#include <vespa/vespalib/testkit/testapp.h>
+#include <vespa/vespalib/util/exceptions.h>
+#include <set>
+#include <vespa/config/helper/configgetter.h>
+
+using config::AsciiConfigWriter;
+using std::set;
+using std::vector;
+using vespalib::Identifiable;
+using vespalib::IllegalArgumentException;
+using vespalib::string;
+
+using namespace document::config_builder;
+using namespace document;
+
+namespace {
+
+const string type_name = "test";
+const int32_t doc_type_id = 787121340;
+const string header_name = type_name + ".header";
+const int32_t header_id = 30;
+const string body_name = type_name + ".body";
+const int32_t body_id = 31;
+const string type_name_2 = "test_2";
+const string header_name_2 = type_name_2 + ".header";
+const string body_name_2 = type_name_2 + ".body";
+const int32_t comp_level = 10;
+const int32_t comp_minres = 80;
+const size_t comp_minsize = 120;
+const string field_name = "field_name";
+const string derived_name = "derived";
+
+TEST("requireThatDocumentTypeCanBeLookedUp") {
+    DocumenttypesConfigBuilderHelper builder;
+    builder.document(doc_type_id, type_name,
+                     Struct(header_name).setId(header_id),
+                     Struct(body_name).setId(body_id));
+    DocumentTypeRepo repo(builder.config());
+
+    const DocumentType *type = repo.getDocumentType(type_name);
+    ASSERT_TRUE(type);
+    EXPECT_EQUAL(type_name, type->getName());
+    EXPECT_EQUAL(doc_type_id, type->getId());
+    EXPECT_EQUAL(header_name, type->getFieldsType().getName());
+/*
+    TODO(vekterli): Check fields struct ID after it has been determined which ID it should get
+    EXPECT_EQUAL(header_id, type->getHeader().getId());
+    EXPECT_EQUAL(header_name, type->getHeader().getName());
+    EXPECT_EQUAL(body_id, type->getBody().getId());
+    EXPECT_EQUAL(body_name, type->getBody().getName());
+*/
+}
+
+TEST("requireThatDocumentTypeCanBeLookedUpWhenIdIsNotAHash") {
+    DocumenttypesConfigBuilderHelper builder;
+    builder.document(doc_type_id + 2, type_name,
+                     Struct(header_name).setId(header_id),
+                     Struct(body_name).setId(body_id));
+    DocumentTypeRepo repo(builder.config());
+
+    const DocumentType *type = repo.getDocumentType(type_name);
+    ASSERT_TRUE(type);
+}
+
+TEST("requireThatStructsCanConfigureCompression") {
+    DocumenttypesConfigBuilderHelper builder;
+    typedef DocumenttypesConfig::Documenttype::Datatype::Sstruct Sstruct;
+    builder.document(doc_type_id, type_name,
+                     Struct(header_name),
+                     Struct(body_name).setCompression(
+                             Sstruct::Compression::LZ4,
+                             comp_level, comp_minres, comp_minsize));
+    DocumentTypeRepo repo(builder.config());
+
+    const CompressionConfig &comp_config =
+        repo.getDocumentType(type_name)->getFieldsType()
+        .getCompressionConfig();
+    EXPECT_EQUAL(CompressionConfig::LZ4, comp_config.type);
+    EXPECT_EQUAL(comp_level, comp_config.compressionLevel);
+    EXPECT_EQUAL(comp_minres, comp_config.threshold);
+    EXPECT_EQUAL(comp_minsize, comp_config.minSize);
+}
+
+TEST("requireThatStructsCanHaveFields") {
+    DocumenttypesConfigBuilderHelper builder;
+    builder.document(doc_type_id, type_name,
+                     Struct(header_name),
+                     Struct(body_name).addField(field_name, DataType::T_INT));
+    DocumentTypeRepo repo(builder.config());
+
+    const StructDataType &s = repo.getDocumentType(type_name)->getFieldsType();
+    ASSERT_EQUAL(1u, s.getFieldCount());
+    const Field &field = s.getField(field_name);
+    EXPECT_EQUAL(DataType::T_INT, field.getDataType().getId());
+}
+
+template <typename T>
+const T &getFieldDataType(const DocumentTypeRepo &repo) {
+    const DataType &d = repo.getDocumentType(type_name)
+                        ->getFieldsType().getField(field_name).getDataType();
+    const T *t = dynamic_cast<const T *>(&d);
+    ASSERT_TRUE(t);
+    return *t;
+}
+
+TEST("requireThatArraysCanBeConfigured") {
+    DocumenttypesConfigBuilderHelper builder;
+    builder.document(doc_type_id, type_name,
+                     Struct(header_name),
+                     Struct(body_name).addField(field_name,
+                             Array(DataType::T_STRING)));
+    DocumentTypeRepo repo(builder.config());
+
+    const ArrayDataType &a = getFieldDataType<ArrayDataType>(repo);
+    EXPECT_EQUAL(DataType::T_STRING, a.getNestedType().getId());
+}
+
+TEST("requireThatWsetsCanBeConfigured") {
+    DocumenttypesConfigBuilderHelper builder;
+    builder.document(doc_type_id, type_name,
+                     Struct(header_name),
+                     Struct(body_name).addField(field_name,
+                             Wset(DataType::T_INT)
+                             .removeIfZero().createIfNonExistent()));
+    DocumentTypeRepo repo(builder.config());
+
+    const WeightedSetDataType &w = getFieldDataType<WeightedSetDataType>(repo);
+    EXPECT_EQUAL(DataType::T_INT, w.getNestedType().getId());
+    EXPECT_TRUE(w.createIfNonExistent());
+    EXPECT_TRUE(w.removeIfZero());
+}
+
+TEST("requireThatMapsCanBeConfigured") {
+    DocumenttypesConfigBuilderHelper builder;
+    builder.document(doc_type_id, type_name,
+                     Struct(header_name),
+                     Struct(body_name).addField(field_name,
+                             Map(DataType::T_INT, DataType::T_STRING)));
+    DocumentTypeRepo repo(builder.config());
+
+    const MapDataType &m = getFieldDataType<MapDataType>(repo);
+    EXPECT_EQUAL(DataType::T_INT, m.getKeyType().getId());
+    EXPECT_EQUAL(DataType::T_STRING, m.getValueType().getId());
+}
+
+TEST("requireThatAnnotationReferencesCanBeConfigured") {
+    int32_t annotation_type_id = 424;
+    DocumenttypesConfigBuilderHelper builder;
+    builder.document(doc_type_id, type_name,
+                     Struct(header_name),
+                     Struct(body_name).addField(field_name,
+                             AnnotationRef(annotation_type_id)))
+        .annotationType(annotation_type_id, "foo", -1);
+    DocumentTypeRepo repo(builder.config());
+
+    const AnnotationReferenceDataType &ar =
+        getFieldDataType<AnnotationReferenceDataType>(repo);
+    EXPECT_EQUAL(annotation_type_id, ar.getAnnotationType().getId());
+}
+
+TEST("requireThatFieldsCanNotBeHeaderAndBody") {
+    DocumenttypesConfigBuilderHelper builder;
+    builder.document(doc_type_id, type_name,
+                     Struct(header_name).addField(field_name,
+                             DataType::T_STRING),
+                     Struct(body_name).addField(field_name,
+                             DataType::T_INT));
+    EXPECT_EXCEPTION(DocumentTypeRepo repo(builder.config()),
+                     IllegalArgumentException,
+                     "Failed to add field 'field_name' to struct 'test.header': "
+                     "Name in use by field with different id");
+}
+
+TEST("requireThatDocumentStructsAreCalledHeaderAndBody") {
+    DocumenttypesConfigBuilderHelper builder;
+    builder.document(doc_type_id, type_name, Struct("foo"), Struct("bar"));
+    EXPECT_EXCEPTION(DocumentTypeRepo repo(builder.config()),
+                     IllegalArgumentException,
+                     "Previously defined as \"test.header\".");
+}
+
+TEST("requireThatDocumentsCanInheritFields") {
+    DocumenttypesConfigBuilderHelper builder;
+    builder.document(doc_type_id, type_name,
+                     Struct(header_name),
+                     Struct(body_name).addField(field_name, DataType::T_INT));
+    builder.document(doc_type_id + 1, derived_name,
+                     Struct("derived.header"),
+                     Struct("derived.body").addField("derived_field",
+                             DataType::T_STRING))
+        .inherit(doc_type_id);
+    DocumentTypeRepo repo(builder.config());
+
+    const StructDataType &s =
+        repo.getDocumentType(doc_type_id + 1)->getFieldsType();
+    ASSERT_EQUAL(2u, s.getFieldCount());
+    const Field &field = s.getField(field_name);
+    const DataType &type = field.getDataType();
+    EXPECT_EQUAL(DataType::T_INT, type.getId());
+}
+
+TEST("requireThatDocumentsCanUseInheritedTypes") {
+    const int32_t id = 64;
+    DocumenttypesConfigBuilderHelper builder;
+    builder.document(doc_type_id, type_name,
+                     Struct(header_name),
+                     Struct(body_name).addField("foo",
+                             Array(DataType::T_INT).setId(id)));
+    builder.document(doc_type_id + 1, derived_name,
+                     Struct("derived.header"),
+                     Struct("derived.body").addField(field_name, id))
+        .inherit(doc_type_id);
+    DocumentTypeRepo repo(builder.config());
+
+    const DataType &type =
+        repo.getDocumentType(doc_type_id + 1)->getFieldsType()
+        .getField(field_name).getDataType();
+    EXPECT_EQUAL(id, type.getId());
+    EXPECT_TRUE(dynamic_cast<const ArrayDataType *>(&type));
+}
+
+TEST("requireThatIllegalConfigsCausesExceptions") {
+    DocumenttypesConfigBuilderHelper builder;
+    builder.document(doc_type_id, type_name,
+                     Struct(header_name), Struct(body_name))
+        .inherit(doc_type_id + 1);
+    EXPECT_EXCEPTION(DocumentTypeRepo repo(builder.config()),
+                IllegalArgumentException, "Unable to find document");
+
+    builder = DocumenttypesConfigBuilderHelper();
+    builder.document(doc_type_id, type_name,
+                     Struct(header_name), Struct(body_name));
+    builder.config().documenttype[0].datatype[0].type =
+        DocumenttypesConfig::Documenttype::Datatype::Type(-1);
+    EXPECT_EXCEPTION(DocumentTypeRepo repo(builder.config()),
+                IllegalArgumentException, "Unknown datatype type -1");
+
+    builder = DocumenttypesConfigBuilderHelper();
+    const int id = 10000;
+    builder.document(doc_type_id, type_name,
+                     Struct(header_name),
+                     Struct(body_name).addField(field_name,
+                             Array(DataType::T_INT).setId(id)));
+    EXPECT_EQUAL(id, builder.config().documenttype[0].datatype[1].id);
+    builder.config().documenttype[0].datatype[1].array.element.id = id;
+    EXPECT_EXCEPTION(DocumentTypeRepo repo(builder.config()),
+                IllegalArgumentException, "Unknown datatype 10000");
+
+    builder = DocumenttypesConfigBuilderHelper();
+    builder.document(doc_type_id, type_name,
+                     Struct(header_name).setId(header_id),
+                     Struct(body_name).addField("foo",
+                             Struct("bar").setId(header_id)));
+    EXPECT_EXCEPTION(DocumentTypeRepo repo(builder.config()),
+                IllegalArgumentException, "Redefinition of data type");
+
+    builder = DocumenttypesConfigBuilderHelper();
+    builder.document(doc_type_id, type_name,
+                     Struct(header_name),
+                     Struct(body_name).addField(field_name,
+                             AnnotationRef(id)));
+    EXPECT_EXCEPTION(DocumentTypeRepo repo(builder.config()),
+                IllegalArgumentException, "Unknown AnnotationType");
+
+    builder = DocumenttypesConfigBuilderHelper();
+    builder.document(doc_type_id, type_name,
+                     Struct(header_name), Struct(body_name))
+        .annotationType(id, type_name, DataType::T_STRING)
+        .annotationType(id, type_name, DataType::T_INT);
+    EXPECT_EXCEPTION(DocumentTypeRepo repo(builder.config()),
+                IllegalArgumentException, "Redefinition of annotation type");
+
+    builder = DocumenttypesConfigBuilderHelper();
+    builder.document(doc_type_id, type_name,
+                     Struct(header_name), Struct(body_name))
+        .annotationType(id, type_name, DataType::T_STRING)
+        .annotationType(id, "foobar", DataType::T_STRING);
+    EXPECT_EXCEPTION(DocumentTypeRepo repo(builder.config()),
+                IllegalArgumentException, "Redefinition of annotation type");
+}
+
+TEST("requireThatDataTypesCanBeLookedUpById") {
+    DocumenttypesConfigBuilderHelper builder;
+    builder.document(doc_type_id, type_name,
+                     Struct(header_name).setId(header_id), Struct(body_name));
+    builder.document(doc_type_id + 1, type_name_2,
+                     Struct(header_name_2), Struct(body_name_2));
+    DocumentTypeRepo repo(builder.config());
+
+    const DataType *type =
+        repo.getDataType(*repo.getDocumentType(doc_type_id), header_id);
+    ASSERT_TRUE(type);
+    EXPECT_EQUAL(header_name, type->getName());
+    EXPECT_EQUAL(header_id, type->getId());
+
+    ASSERT_TRUE(!repo.getDataType(*repo.getDocumentType(doc_type_id), -1));
+    ASSERT_TRUE(!repo.getDataType(*repo.getDocumentType(doc_type_id + 1),
+                                  header_id));
+}
+
+TEST("requireThatDataTypesCanBeLookedUpByName") {
+    DocumenttypesConfigBuilderHelper builder;
+    builder.document(doc_type_id, type_name,
+                     Struct(header_name).setId(header_id),
+                     Struct(body_name));
+    builder.document(doc_type_id + 1, type_name_2,
+                     Struct(header_name_2), Struct(body_name_2));
+    DocumentTypeRepo repo(builder.config());
+
+    const DataType *type =
+        repo.getDataType(*repo.getDocumentType(doc_type_id), header_name);
+    ASSERT_TRUE(type);
+    EXPECT_EQUAL(header_name, type->getName());
+    EXPECT_EQUAL(header_id, type->getId());
+
+    EXPECT_TRUE(repo.getDataType(*repo.getDocumentType(doc_type_id),
+                                 type_name));
+    EXPECT_TRUE(!repo.getDataType(*repo.getDocumentType(doc_type_id),
+                                  field_name));
+    EXPECT_TRUE(!repo.getDataType(*repo.getDocumentType(doc_type_id + 1),
+                                  body_name));
+}
+
+TEST("requireThatInheritingDocCanRedefineIdenticalField") {
+    DocumenttypesConfigBuilderHelper builder;
+    builder.document(doc_type_id, type_name,
+                     Struct(header_name),
+                     Struct(body_name)
+                     .addField(field_name, DataType::T_STRING)
+                     .setId(body_id));
+    builder.document(doc_type_id + 1, derived_name,
+                     Struct("derived.header"),
+                     Struct("derived.body")
+                     .addField(field_name, DataType::T_STRING)
+                     .setId(body_id))
+        .inherit(doc_type_id);
+    DocumentTypeRepo repo(builder.config());
+
+    const StructDataType &s = repo.getDocumentType(doc_type_id + 1)->getFieldsType();
+    ASSERT_EQUAL(1u, s.getFieldCount());
+}
+
+TEST("requireThatAnnotationTypesCanBeConfigured") {
+    const int32_t a_id = 654;
+    const string a_name = "annotation_name";
+    DocumenttypesConfigBuilderHelper builder;
+    builder.document(doc_type_id, type_name,
+                     Struct(header_name), Struct(body_name))
+        .annotationType(a_id, a_name, DataType::T_STRING);
+    DocumentTypeRepo repo(builder.config());
+
+    const DocumentType *type = repo.getDocumentType(doc_type_id);
+    const AnnotationType *a_type = repo.getAnnotationType(*type, a_id);
+    ASSERT_TRUE(a_type);
+    EXPECT_EQUAL(a_name, a_type->getName());
+    ASSERT_TRUE(a_type->getDataType());
+    EXPECT_EQUAL(DataType::T_STRING, a_type->getDataType()->getId());
+}
+
+TEST("requireThatFieldIdV6IsDifferentFromV7") {
+    DocumenttypesConfigBuilderHelper builder;
+    builder.document(doc_type_id, type_name,
+                     Struct(header_name),
+                     Struct(body_name).addField(field_name, DataType::T_INT));
+    DocumentTypeRepo repo(builder.config());
+
+    const StructDataType &s = repo.getDocumentType(type_name)->getFieldsType();
+    ASSERT_EQUAL(1u, s.getFieldCount());
+    const Field &field = s.getField(field_name);
+    ASSERT_TRUE(field.getId(7) != field.getId(6));
+}
+
+TEST("requireThatDocumentsCanUseOtherDocumentTypes") {
+    DocumenttypesConfigBuilderHelper builder;
+    builder.document(doc_type_id + 1, type_name_2,
+                     Struct(header_name_2),
+                     Struct(body_name_2));
+    builder.document(doc_type_id, type_name,
+                     Struct(header_name),
+                     Struct(body_name).addField(type_name_2,
+                             doc_type_id + 1).setId(body_id));
+    DocumentTypeRepo repo(builder.config());
+
+    const DataType &type = repo.getDocumentType(doc_type_id)->getFieldsType()
+                           .getField(type_name_2).getDataType();
+    EXPECT_EQUAL(doc_type_id + 1, type.getId());
+    EXPECT_TRUE(dynamic_cast<const DocumentType *>(&type));
+}
+
+void storeId(set<int> *s, const DocumentType &type) {
+    s->insert(type.getId());
+}
+
+TEST("requireThatDocumentTypesCanBeIterated") {
+    DocumenttypesConfigBuilderHelper builder;
+    builder.document(doc_type_id, type_name,
+                     Struct(header_name), Struct(body_name));
+    builder.document(doc_type_id + 1, type_name_2,
+                     Struct(header_name_2), Struct(body_name_2));
+    DocumentTypeRepo repo(builder.config());
+
+    set<int> ids;
+    repo.forEachDocumentType(*makeClosure(storeId, &ids));
+
+    EXPECT_EQUAL(3u, ids.size());
+    ASSERT_TRUE(ids.count(DataType::T_DOCUMENT));
+    ASSERT_TRUE(ids.count(doc_type_id));
+    ASSERT_TRUE(ids.count(doc_type_id + 1));
+}
+
+TEST("requireThatDocumentLookupChecksName") {
+    DocumenttypesConfigBuilderHelper builder;
+    builder.document(doc_type_id, type_name_2,
+                     Struct(header_name_2), Struct(body_name_2));
+    DocumentTypeRepo repo(builder.config());
+
+    // "type_name" will generate the document type id
+    // "doc_type_id". However, this config assigns that id to a
+    // different type.
+    const DocumentType *type = repo.getDocumentType(type_name);
+    ASSERT_TRUE(!type);
+}
+
+TEST("requireThatBuildFromConfigWorks") {
+    DocumentTypeRepo repo(readDocumenttypesConfig("documenttypes.cfg"));
+    ASSERT_TRUE(repo.getDocumentType("document"));
+    ASSERT_TRUE(repo.getDocumentType("types"));
+    ASSERT_TRUE(repo.getDocumentType("types_search"));
+}
+
+TEST("requireThatStructsCanBeRecursive") {
+    DocumenttypesConfigBuilderHelper builder;
+    builder.document(doc_type_id, type_name,
+                     Struct(header_name).setId(header_id).addField(field_name,
+                             header_id),
+                     Struct(body_name));
+    DocumentTypeRepo repo(builder.config());
+
+    const StructDataType &s = repo.getDocumentType(type_name)->getFieldsType();
+    ASSERT_EQUAL(1u, s.getFieldCount());
+}
+
+}  // namespace
+
+TEST("requireThatMissingFileCausesException") {
+    EXPECT_EXCEPTION(readDocumenttypesConfig("illegal/missing_file"),
+                     IllegalArgumentException, "Unable to open file");
+}
+
+TEST("requireThatFieldsCanHaveAnyDocumentType") {
+    DocumenttypesConfigBuilderHelper builder;
+    builder.document(doc_type_id, type_name,
+                     Struct(header_name),
+                     Struct(body_name).addField(field_name, doc_type_id + 1));
+    // Circular dependency
+    builder.document(doc_type_id + 1, type_name_2,
+                     Struct(header_name_2),
+                     Struct(body_name_2).addField(field_name, doc_type_id));
+    DocumentTypeRepo repo(builder.config());
+    const DocumentType *type1 = repo.getDocumentType(doc_type_id);
+    const DocumentType *type2 = repo.getDocumentType(doc_type_id + 1);
+    ASSERT_TRUE(type1);
+    EXPECT_EQUAL(type1, repo.getDataType(*type1, doc_type_id));
+    EXPECT_EQUAL(type2, repo.getDataType(*type1, doc_type_id + 1));
+    EXPECT_TRUE(type1->getFieldsType().hasField(field_name));
+    ASSERT_TRUE(type2);
+    EXPECT_EQUAL(type1, repo.getDataType(*type2, doc_type_id));
+    EXPECT_EQUAL(type2, repo.getDataType(*type2, doc_type_id + 1));
+    EXPECT_TRUE(type2->getFieldsType().hasField(field_name));
+}
+
+TEST("requireThatBodyCanOccurBeforeHeaderInConfig") {
+    DocumenttypesConfigBuilderHelper builder;
+    // Add header and body in reverse order, then swap the ids.
+    builder.document(doc_type_id, type_name,
+                     Struct(body_name).setId(body_id).addField("bodystuff",
+                             DataType::T_STRING),
+                     Struct(header_name).setId(header_id).addField(
+                             "headerstuff", DataType::T_INT));
+    std::swap(builder.config().documenttype[0].headerstruct,
+              builder.config().documenttype[0].bodystruct);
+
+    DocumentTypeRepo repo(builder.config());
+    const StructDataType &s = repo.getDocumentType(type_name)->getFieldsType();
+    // Should have both fields in fields struct
+    EXPECT_TRUE(s.hasField("headerstuff"));
+    EXPECT_TRUE(s.hasField("bodystuff"));
+}
+
+TEST("Require that Array can have nested DocumentType") {
+    DocumenttypesConfigBuilderHelper builder;
+    builder.document(doc_type_id, type_name, Struct(header_name),
+                     Struct(body_name)
+                     .addField(field_name, Array(doc_type_id)));
+    DocumentTypeRepo repo(builder.config());
+    const DocumentType *type = repo.getDocumentType(doc_type_id);
+    ASSERT_TRUE(type);
+}
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/document/src/tests/repo/documenttypes.cfg b/document/src/tests/repo/documenttypes.cfg
new file mode 100644
index 00000000000..4641663a94b
--- /dev/null
+++ b/document/src/tests/repo/documenttypes.cfg
@@ -0,0 +1,421 @@
+documenttype[2]
+documenttype[0].bodystruct 348447225
+documenttype[0].headerstruct 1328581348
+documenttype[0].id -853072901
+documenttype[0].name "types"
+documenttype[0].version 0
+documenttype[0].datatype[27]
+documenttype[0].datatype[00].id 49942803
+documenttype[0].datatype[00].type ARRAY
+documenttype[0].datatype[00].array.element.id 16
+documenttype[0].datatype[01].id 339965458
+documenttype[0].datatype[01].type MAP
+documenttype[0].datatype[01].map.key.id 2
+documenttype[0].datatype[01].map.value.id 2
+documenttype[0].datatype[02].id 69621385
+documenttype[0].datatype[02].type ARRAY
+documenttype[0].datatype[02].array.element.id 339965458
+documenttype[0].datatype[03].id -2092985853
+documenttype[0].datatype[03].type STRUCT
+documenttype[0].datatype[03].sstruct.name "mystruct"
+documenttype[0].datatype[03].sstruct.version 0
+documenttype[0].datatype[03].sstruct.field[4]
+documenttype[0].datatype[03].sstruct.field[bytearr].datatype 49942803
+documenttype[0].datatype[03].sstruct.field[bytearr].id 1079701754
+documenttype[0].datatype[03].sstruct.field[bytearr].id_v6 1198855694
+documenttype[0].datatype[03].sstruct.field[bytearr].name "bytearr"
+documenttype[0].datatype[03].sstruct.field[mymap].datatype 339965458
+documenttype[0].datatype[03].sstruct.field[mymap].id 1954178122
+documenttype[0].datatype[03].sstruct.field[mymap].id_v6 707189723
+documenttype[0].datatype[03].sstruct.field[mymap].name "mymap"
+documenttype[0].datatype[03].sstruct.field[structfield].datatype 2
+documenttype[0].datatype[03].sstruct.field[structfield].id 1726890940
+documenttype[0].datatype[03].sstruct.field[structfield].id_v6 418303145
+documenttype[0].datatype[03].sstruct.field[structfield].name "structfield"
+documenttype[0].datatype[03].sstruct.field[title].datatype 2
+documenttype[0].datatype[03].sstruct.field[title].id 567626448
+documenttype[0].datatype[03].sstruct.field[title].id_v6 29129762
+documenttype[0].datatype[03].sstruct.field[title].name "title"
+documenttype[0].datatype[04].id 759956026
+documenttype[0].datatype[04].type ARRAY
+documenttype[0].datatype[04].array.element.id -2092985853
+documenttype[0].datatype[05].id 2125328771
+documenttype[0].datatype[05].type WSET
+documenttype[0].datatype[05].wset.createifnonexistent false
+documenttype[0].datatype[05].wset.removeifzero true
+documenttype[0].datatype[05].wset.key.id 2
+documenttype[0].datatype[06].id -1486737430
+documenttype[0].datatype[06].type ARRAY
+documenttype[0].datatype[06].array.element.id 2
+documenttype[0].datatype[07].id 1707615575
+documenttype[0].datatype[07].type ARRAY
+documenttype[0].datatype[07].array.element.id -1486737430
+documenttype[0].datatype[08].id 171503364
+documenttype[0].datatype[08].type MAP
+documenttype[0].datatype[08].map.key.id 1707615575
+documenttype[0].datatype[08].map.value.id 0
+documenttype[0].datatype[09].id 1100964733
+documenttype[0].datatype[09].type ARRAY
+documenttype[0].datatype[09].array.element.id 171503364
+documenttype[0].datatype[10].id -1715531035
+documenttype[0].datatype[10].type MAP
+documenttype[0].datatype[10].map.key.id 0
+documenttype[0].datatype[10].map.value.id 4
+documenttype[0].datatype[11].id -794985308
+documenttype[0].datatype[11].type ARRAY
+documenttype[0].datatype[11].array.element.id 1707615575
+documenttype[0].datatype[12].id 109267174
+documenttype[0].datatype[12].type STRUCT
+documenttype[0].datatype[12].sstruct.name "sct"
+documenttype[0].datatype[12].sstruct.version 0
+documenttype[0].datatype[12].sstruct.field[2]
+documenttype[0].datatype[12].sstruct.field[s1].datatype 2
+documenttype[0].datatype[12].sstruct.field[s1].id 2146820765
+documenttype[0].datatype[12].sstruct.field[s1].id_v6 142373281
+documenttype[0].datatype[12].sstruct.field[s1].name "s1"
+documenttype[0].datatype[12].sstruct.field[s2].datatype 2
+documenttype[0].datatype[12].sstruct.field[s2].id 45366795
+documenttype[0].datatype[12].sstruct.field[s2].id_v6 31106270
+documenttype[0].datatype[12].sstruct.field[s2].name "s2"
+documenttype[0].datatype[13].id -1244829667
+documenttype[0].datatype[13].type ARRAY
+documenttype[0].datatype[13].array.element.id 109267174
+documenttype[0].datatype[14].id -1865479609
+documenttype[0].datatype[14].type MAP
+documenttype[0].datatype[14].map.key.id 2
+documenttype[0].datatype[14].map.value.id 4
+documenttype[0].datatype[15].id 2138385264
+documenttype[0].datatype[15].type MAP
+documenttype[0].datatype[15].map.key.id 0
+documenttype[0].datatype[15].map.value.id 5
+documenttype[0].datatype[16].id 1328286588
+documenttype[0].datatype[16].type WSET
+documenttype[0].datatype[16].wset.createifnonexistent false
+documenttype[0].datatype[16].wset.removeifzero false
+documenttype[0].datatype[16].wset.key.id 2
+documenttype[0].datatype[17].id 294108848
+documenttype[0].datatype[17].type STRUCT
+documenttype[0].datatype[17].sstruct.name "folder"
+documenttype[0].datatype[17].sstruct.version 0
+documenttype[0].datatype[17].sstruct.field[4]
+documenttype[0].datatype[17].sstruct.field[FlagsCounter].datatype -1865479609
+documenttype[0].datatype[17].sstruct.field[FlagsCounter].id 1741227606
+documenttype[0].datatype[17].sstruct.field[FlagsCounter].id_v6 1287497652
+documenttype[0].datatype[17].sstruct.field[FlagsCounter].name "FlagsCounter"
+documenttype[0].datatype[17].sstruct.field[Name].datatype 2
+documenttype[0].datatype[17].sstruct.field[Name].id 2002760220
+documenttype[0].datatype[17].sstruct.field[Name].id_v6 62942997
+documenttype[0].datatype[17].sstruct.field[Name].name "Name"
+documenttype[0].datatype[17].sstruct.field[Version].datatype 0
+documenttype[0].datatype[17].sstruct.field[Version].id 64430502
+documenttype[0].datatype[17].sstruct.field[Version].id_v6 634243672
+documenttype[0].datatype[17].sstruct.field[Version].name "Version"
+documenttype[0].datatype[17].sstruct.field[anotherfolder].datatype 294108848
+documenttype[0].datatype[17].sstruct.field[anotherfolder].id 1582421848
+documenttype[0].datatype[17].sstruct.field[anotherfolder].id_v6 1898725199
+documenttype[0].datatype[17].sstruct.field[anotherfolder].name "anotherfolder"
+documenttype[0].datatype[18].id 2065577986
+documenttype[0].datatype[18].type WSET
+documenttype[0].datatype[18].wset.createifnonexistent true
+documenttype[0].datatype[18].wset.removeifzero false
+documenttype[0].datatype[18].wset.key.id 2
+documenttype[0].datatype[19].id 1901258752
+documenttype[0].datatype[19].type MAP
+documenttype[0].datatype[19].map.key.id 0
+documenttype[0].datatype[19].map.value.id -2092985853
+documenttype[0].datatype[20].id 348447225
+documenttype[0].datatype[20].type STRUCT
+documenttype[0].datatype[20].sstruct.name "types.body"
+documenttype[0].datatype[20].sstruct.version 0
+documenttype[0].datatype[20].sstruct.field[1]
+documenttype[0].datatype[20].sstruct.field[complexarray].datatype 1100964733
+documenttype[0].datatype[20].sstruct.field[complexarray].id 1028383787
+documenttype[0].datatype[20].sstruct.field[complexarray].id_v6 658530305
+documenttype[0].datatype[20].sstruct.field[complexarray].name "complexarray"
+documenttype[0].datatype[21].id 2125154557
+documenttype[0].datatype[21].type MAP
+documenttype[0].datatype[21].map.key.id 2
+documenttype[0].datatype[21].map.value.id 1
+documenttype[0].datatype[22].id -389833101
+documenttype[0].datatype[22].type MAP
+documenttype[0].datatype[22].map.key.id 0
+documenttype[0].datatype[22].map.value.id 294108848
+documenttype[0].datatype[23].id -1245117006
+documenttype[0].datatype[23].type ARRAY
+documenttype[0].datatype[23].array.element.id 0
+documenttype[0].datatype[24].id -1584287606
+documenttype[0].datatype[24].type MAP
+documenttype[0].datatype[24].map.key.id 2
+documenttype[0].datatype[24].map.value.id 0
+documenttype[0].datatype[25].id 435886609
+documenttype[0].datatype[25].type MAP
+documenttype[0].datatype[25].map.key.id 2
+documenttype[0].datatype[25].map.value.id -1245117006
+documenttype[0].datatype[26].id 1328581348
+documenttype[0].datatype[26].type STRUCT
+documenttype[0].datatype[26].sstruct.name "types.header"
+documenttype[0].datatype[26].sstruct.version 0
+documenttype[0].datatype[26].sstruct.field[25]
+documenttype[0].datatype[26].sstruct.field[Folders].datatype -389833101
+documenttype[0].datatype[26].sstruct.field[Folders].id 34575524
+documenttype[0].datatype[26].sstruct.field[Folders].id_v6 280569744
+documenttype[0].datatype[26].sstruct.field[Folders].name "Folders"
+documenttype[0].datatype[26].sstruct.field[abyte].datatype 16
+documenttype[0].datatype[26].sstruct.field[abyte].id 110138156
+documenttype[0].datatype[26].sstruct.field[abyte].id_v6 1369099343
+documenttype[0].datatype[26].sstruct.field[abyte].name "abyte"
+documenttype[0].datatype[26].sstruct.field[album0].datatype 18
+documenttype[0].datatype[26].sstruct.field[album0].id 764312262
+documenttype[0].datatype[26].sstruct.field[album0].id_v6 1409364160
+documenttype[0].datatype[26].sstruct.field[album0].name "album0"
+documenttype[0].datatype[26].sstruct.field[album1].datatype 18
+documenttype[0].datatype[26].sstruct.field[album1].id 1967160809
+documenttype[0].datatype[26].sstruct.field[album1].id_v6 1833811264
+documenttype[0].datatype[26].sstruct.field[album1].name "album1"
+documenttype[0].datatype[26].sstruct.field[along].datatype 4
+documenttype[0].datatype[26].sstruct.field[along].id 1206464520
+documenttype[0].datatype[26].sstruct.field[along].id_v6 871280609
+documenttype[0].datatype[26].sstruct.field[along].name "along"
+documenttype[0].datatype[26].sstruct.field[arrarr].datatype -794985308
+documenttype[0].datatype[26].sstruct.field[arrarr].id 1962567166
+documenttype[0].datatype[26].sstruct.field[arrarr].id_v6 885141301
+documenttype[0].datatype[26].sstruct.field[arrarr].name "arrarr"
+documenttype[0].datatype[26].sstruct.field[arrayfield].datatype -1245117006
+documenttype[0].datatype[26].sstruct.field[arrayfield].id 965790107
+documenttype[0].datatype[26].sstruct.field[arrayfield].id_v6 1010955705
+documenttype[0].datatype[26].sstruct.field[arrayfield].name "arrayfield"
+documenttype[0].datatype[26].sstruct.field[arraymapfield].datatype 435886609
+documenttype[0].datatype[26].sstruct.field[arraymapfield].id 1670805928
+documenttype[0].datatype[26].sstruct.field[arraymapfield].id_v6 1940354311
+documenttype[0].datatype[26].sstruct.field[arraymapfield].name "arraymapfield"
+documenttype[0].datatype[26].sstruct.field[doublemapfield].datatype 2138385264
+documenttype[0].datatype[26].sstruct.field[doublemapfield].id 877047192
+documenttype[0].datatype[26].sstruct.field[doublemapfield].id_v6 957317090
+documenttype[0].datatype[26].sstruct.field[doublemapfield].name "doublemapfield"
+documenttype[0].datatype[26].sstruct.field[floatmapfield].datatype 2125154557
+documenttype[0].datatype[26].sstruct.field[floatmapfield].id 1239120925
+documenttype[0].datatype[26].sstruct.field[floatmapfield].id_v6 1609437589
+documenttype[0].datatype[26].sstruct.field[floatmapfield].name "floatmapfield"
+documenttype[0].datatype[26].sstruct.field[intmapfield].datatype -1584287606
+documenttype[0].datatype[26].sstruct.field[intmapfield].id 121004462
+documenttype[0].datatype[26].sstruct.field[intmapfield].id_v6 1642487905
+documenttype[0].datatype[26].sstruct.field[intmapfield].name "intmapfield"
+documenttype[0].datatype[26].sstruct.field[foo].datatype 4
+documenttype[0].datatype[26].sstruct.field[foo].id 266111229
+documenttype[0].datatype[26].sstruct.field[foo].id_v6 383833303
+documenttype[0].datatype[26].sstruct.field[foo].name "foo"
+documenttype[0].datatype[26].sstruct.field[longmapfield].datatype -1715531035
+documenttype[0].datatype[26].sstruct.field[longmapfield].id 477718745
+documenttype[0].datatype[26].sstruct.field[longmapfield].id_v6 920341968
+documenttype[0].datatype[26].sstruct.field[longmapfield].name "longmapfield"
+documenttype[0].datatype[26].sstruct.field[maparr].datatype 69621385
+documenttype[0].datatype[26].sstruct.field[maparr].id 904375219
+documenttype[0].datatype[26].sstruct.field[maparr].id_v6 63700074
+documenttype[0].datatype[26].sstruct.field[maparr].name "maparr"
+documenttype[0].datatype[26].sstruct.field[mystructarr].datatype 759956026
+documenttype[0].datatype[26].sstruct.field[mystructarr].id 595856991
+documenttype[0].datatype[26].sstruct.field[mystructarr].id_v6 764861972
+documenttype[0].datatype[26].sstruct.field[mystructarr].name "mystructarr"
+documenttype[0].datatype[26].sstruct.field[mystructfield].datatype -2092985853
+documenttype[0].datatype[26].sstruct.field[mystructfield].id 1348513378
+documenttype[0].datatype[26].sstruct.field[mystructfield].id_v6 2033170300
+documenttype[0].datatype[26].sstruct.field[mystructfield].name "mystructfield"
+documenttype[0].datatype[26].sstruct.field[mystructmap].datatype 1901258752
+documenttype[0].datatype[26].sstruct.field[mystructmap].id 1511423250
+documenttype[0].datatype[26].sstruct.field[mystructmap].id_v6 449602635
+documenttype[0].datatype[26].sstruct.field[mystructmap].name "mystructmap"
+documenttype[0].datatype[26].sstruct.field[setfield].datatype 1328286588
+documenttype[0].datatype[26].sstruct.field[setfield].id 761581914
+documenttype[0].datatype[26].sstruct.field[setfield].id_v6 1762943268
+documenttype[0].datatype[26].sstruct.field[setfield].name "setfield"
+documenttype[0].datatype[26].sstruct.field[setfield2].datatype 18
+documenttype[0].datatype[26].sstruct.field[setfield2].id 1066659198
+documenttype[0].datatype[26].sstruct.field[setfield2].id_v6 813038565
+documenttype[0].datatype[26].sstruct.field[setfield2].name "setfield2"
+documenttype[0].datatype[26].sstruct.field[setfield3].datatype 2125328771
+documenttype[0].datatype[26].sstruct.field[setfield3].id 1180155772
+documenttype[0].datatype[26].sstruct.field[setfield3].id_v6 1697232199
+documenttype[0].datatype[26].sstruct.field[setfield3].name "setfield3"
+documenttype[0].datatype[26].sstruct.field[setfield4].datatype 2065577986
+documenttype[0].datatype[26].sstruct.field[setfield4].id 1254131631
+documenttype[0].datatype[26].sstruct.field[setfield4].id_v6 119755202
+documenttype[0].datatype[26].sstruct.field[setfield4].name "setfield4"
+documenttype[0].datatype[26].sstruct.field[stringmapfield].datatype 339965458
+documenttype[0].datatype[26].sstruct.field[stringmapfield].id 117465687
+documenttype[0].datatype[26].sstruct.field[stringmapfield].id_v6 1492788095
+documenttype[0].datatype[26].sstruct.field[stringmapfield].name "stringmapfield"
+documenttype[0].datatype[26].sstruct.field[structarrayfield].datatype -1244829667
+documenttype[0].datatype[26].sstruct.field[structarrayfield].id 335048518
+documenttype[0].datatype[26].sstruct.field[structarrayfield].id_v6 607034174
+documenttype[0].datatype[26].sstruct.field[structarrayfield].name "structarrayfield"
+documenttype[0].datatype[26].sstruct.field[structfield].datatype 109267174
+documenttype[0].datatype[26].sstruct.field[structfield].id 486207386
+documenttype[0].datatype[26].sstruct.field[structfield].id_v6 418303145
+documenttype[0].datatype[26].sstruct.field[structfield].name "structfield"
+documenttype[0].datatype[26].sstruct.field[tagfield].datatype 18
+documenttype[0].datatype[26].sstruct.field[tagfield].id 1653562069
+documenttype[0].datatype[26].sstruct.field[tagfield].id_v6 938523246
+documenttype[0].datatype[26].sstruct.field[tagfield].name "tagfield"
+documenttype[0].inherits[1]
+documenttype[0].inherits[document.0].id 8
+documenttype[1].bodystruct -1408707420
+documenttype[1].headerstruct 625114831
+documenttype[1].id 666563184
+documenttype[1].name "types_search"
+documenttype[1].version 0
+documenttype[1].datatype[12]
+documenttype[1].datatype[00].id 1328286588
+documenttype[1].datatype[00].type WSET
+documenttype[1].datatype[00].wset.createifnonexistent false
+documenttype[1].datatype[00].wset.removeifzero false
+documenttype[1].datatype[00].wset.key.id 2
+documenttype[1].datatype[01].id -1865479609
+documenttype[1].datatype[01].type MAP
+documenttype[1].datatype[01].map.key.id 2
+documenttype[1].datatype[01].map.value.id 4
+documenttype[1].datatype[02].id 109267174
+documenttype[1].datatype[02].type STRUCT
+documenttype[1].datatype[02].sstruct.name "sct"
+documenttype[1].datatype[02].sstruct.version 0
+documenttype[1].datatype[02].sstruct.field[2]
+documenttype[1].datatype[02].sstruct.field[s1].datatype 2
+documenttype[1].datatype[02].sstruct.field[s1].id 2146820765
+documenttype[1].datatype[02].sstruct.field[s1].id_v6 142373281
+documenttype[1].datatype[02].sstruct.field[s1].name "s1"
+documenttype[1].datatype[02].sstruct.field[s2].datatype 2
+documenttype[1].datatype[02].sstruct.field[s2].id 45366795
+documenttype[1].datatype[02].sstruct.field[s2].id_v6 31106270
+documenttype[1].datatype[02].sstruct.field[s2].name "s2"
+documenttype[1].datatype[03].id 294108848
+documenttype[1].datatype[03].type STRUCT
+documenttype[1].datatype[03].sstruct.name "folder"
+documenttype[1].datatype[03].sstruct.version 0
+documenttype[1].datatype[03].sstruct.field[4]
+documenttype[1].datatype[03].sstruct.field[FlagsCounter].datatype -1865479609
+documenttype[1].datatype[03].sstruct.field[FlagsCounter].id 1741227606
+documenttype[1].datatype[03].sstruct.field[FlagsCounter].id_v6 1287497652
+documenttype[1].datatype[03].sstruct.field[FlagsCounter].name "FlagsCounter"
+documenttype[1].datatype[03].sstruct.field[Name].datatype 2
+documenttype[1].datatype[03].sstruct.field[Name].id 2002760220
+documenttype[1].datatype[03].sstruct.field[Name].id_v6 62942997
+documenttype[1].datatype[03].sstruct.field[Name].name "Name"
+documenttype[1].datatype[03].sstruct.field[Version].datatype 0
+documenttype[1].datatype[03].sstruct.field[Version].id 64430502
+documenttype[1].datatype[03].sstruct.field[Version].id_v6 634243672
+documenttype[1].datatype[03].sstruct.field[Version].name "Version"
+documenttype[1].datatype[03].sstruct.field[anotherfolder].datatype 294108848
+documenttype[1].datatype[03].sstruct.field[anotherfolder].id 1582421848
+documenttype[1].datatype[03].sstruct.field[anotherfolder].id_v6 1898725199
+documenttype[1].datatype[03].sstruct.field[anotherfolder].name "anotherfolder"
+documenttype[1].datatype[04].id 2125328771
+documenttype[1].datatype[04].type WSET
+documenttype[1].datatype[04].wset.createifnonexistent false
+documenttype[1].datatype[04].wset.removeifzero true
+documenttype[1].datatype[04].wset.key.id 2
+documenttype[1].datatype[05].id -1245117006
+documenttype[1].datatype[05].type ARRAY
+documenttype[1].datatype[05].array.element.id 0
+documenttype[1].datatype[06].id 2065577986
+documenttype[1].datatype[06].type WSET
+documenttype[1].datatype[06].wset.createifnonexistent true
+documenttype[1].datatype[06].wset.removeifzero false
+documenttype[1].datatype[06].wset.key.id 2
+documenttype[1].datatype[07].id 339965458
+documenttype[1].datatype[07].type MAP
+documenttype[1].datatype[07].map.key.id 2
+documenttype[1].datatype[07].map.value.id 2
+documenttype[1].datatype[08].id 625114831
+documenttype[1].datatype[08].type STRUCT
+documenttype[1].datatype[08].sstruct.name "types_search.header"
+documenttype[1].datatype[08].sstruct.version 0
+documenttype[1].datatype[08].sstruct.field[15]
+documenttype[1].datatype[08].sstruct.field[abyte].datatype 16
+documenttype[1].datatype[08].sstruct.field[abyte].id 110138156
+documenttype[1].datatype[08].sstruct.field[abyte].id_v6 1369099343
+documenttype[1].datatype[08].sstruct.field[abyte].name "abyte"
+documenttype[1].datatype[08].sstruct.field[album0].datatype 18
+documenttype[1].datatype[08].sstruct.field[album0].id 764312262
+documenttype[1].datatype[08].sstruct.field[album0].id_v6 1409364160
+documenttype[1].datatype[08].sstruct.field[album0].name "album0"
+documenttype[1].datatype[08].sstruct.field[album1].datatype 18
+documenttype[1].datatype[08].sstruct.field[album1].id 1967160809
+documenttype[1].datatype[08].sstruct.field[album1].id_v6 1833811264
+documenttype[1].datatype[08].sstruct.field[album1].name "album1"
+documenttype[1].datatype[08].sstruct.field[along].datatype 4
+documenttype[1].datatype[08].sstruct.field[along].id 1206464520
+documenttype[1].datatype[08].sstruct.field[along].id_v6 871280609
+documenttype[1].datatype[08].sstruct.field[along].name "along"
+documenttype[1].datatype[08].sstruct.field[arrayfield].datatype -1245117006
+documenttype[1].datatype[08].sstruct.field[arrayfield].id 965790107
+documenttype[1].datatype[08].sstruct.field[arrayfield].id_v6 1010955705
+documenttype[1].datatype[08].sstruct.field[arrayfield].name "arrayfield"
+documenttype[1].datatype[08].sstruct.field[foo].datatype 4
+documenttype[1].datatype[08].sstruct.field[foo].id 266111229
+documenttype[1].datatype[08].sstruct.field[foo].id_v6 383833303
+documenttype[1].datatype[08].sstruct.field[foo].name "foo"
+documenttype[1].datatype[08].sstruct.field[other].datatype 4
+documenttype[1].datatype[08].sstruct.field[other].id 2443357
+documenttype[1].datatype[08].sstruct.field[other].id_v6 903806222
+documenttype[1].datatype[08].sstruct.field[other].name "other"
+documenttype[1].datatype[08].sstruct.field[rankfeatures].datatype 2
+documenttype[1].datatype[08].sstruct.field[rankfeatures].id 1883197392
+documenttype[1].datatype[08].sstruct.field[rankfeatures].id_v6 699950698
+documenttype[1].datatype[08].sstruct.field[rankfeatures].name "rankfeatures"
+documenttype[1].datatype[08].sstruct.field[setfield].datatype 1328286588
+documenttype[1].datatype[08].sstruct.field[setfield].id 761581914
+documenttype[1].datatype[08].sstruct.field[setfield].id_v6 1762943268
+documenttype[1].datatype[08].sstruct.field[setfield].name "setfield"
+documenttype[1].datatype[08].sstruct.field[setfield2].datatype 18
+documenttype[1].datatype[08].sstruct.field[setfield2].id 1066659198
+documenttype[1].datatype[08].sstruct.field[setfield2].id_v6 813038565
+documenttype[1].datatype[08].sstruct.field[setfield2].name "setfield2"
+documenttype[1].datatype[08].sstruct.field[setfield3].datatype 2125328771
+documenttype[1].datatype[08].sstruct.field[setfield3].id 1180155772
+documenttype[1].datatype[08].sstruct.field[setfield3].id_v6 1697232199
+documenttype[1].datatype[08].sstruct.field[setfield3].name "setfield3"
+documenttype[1].datatype[08].sstruct.field[setfield4].datatype 2065577986
+documenttype[1].datatype[08].sstruct.field[setfield4].id 1254131631
+documenttype[1].datatype[08].sstruct.field[setfield4].id_v6 119755202
+documenttype[1].datatype[08].sstruct.field[setfield4].name "setfield4"
+documenttype[1].datatype[08].sstruct.field[stringmapfield].datatype 339965458
+documenttype[1].datatype[08].sstruct.field[stringmapfield].id 117465687
+documenttype[1].datatype[08].sstruct.field[stringmapfield].id_v6 1492788095
+documenttype[1].datatype[08].sstruct.field[stringmapfield].name "stringmapfield"
+documenttype[1].datatype[08].sstruct.field[summaryfeatures].datatype 2
+documenttype[1].datatype[08].sstruct.field[summaryfeatures].id 1840337115
+documenttype[1].datatype[08].sstruct.field[summaryfeatures].id_v6 1981648971
+documenttype[1].datatype[08].sstruct.field[summaryfeatures].name "summaryfeatures"
+documenttype[1].datatype[08].sstruct.field[tagfield].datatype 18
+documenttype[1].datatype[08].sstruct.field[tagfield].id 1653562069
+documenttype[1].datatype[08].sstruct.field[tagfield].id_v6 938523246
+documenttype[1].datatype[08].sstruct.field[tagfield].name "tagfield"
+documenttype[1].datatype[09].id 49942803
+documenttype[1].datatype[09].type ARRAY
+documenttype[1].datatype[09].array.element.id 16
+documenttype[1].datatype[10].id -2092985853
+documenttype[1].datatype[10].type STRUCT
+documenttype[1].datatype[10].sstruct.name "mystruct"
+documenttype[1].datatype[10].sstruct.version 0
+documenttype[1].datatype[10].sstruct.field[4]
+documenttype[1].datatype[10].sstruct.field[bytearr].datatype 49942803
+documenttype[1].datatype[10].sstruct.field[bytearr].id 1079701754
+documenttype[1].datatype[10].sstruct.field[bytearr].id_v6 1198855694
+documenttype[1].datatype[10].sstruct.field[bytearr].name "bytearr"
+documenttype[1].datatype[10].sstruct.field[mymap].datatype 339965458
+documenttype[1].datatype[10].sstruct.field[mymap].id 1954178122
+documenttype[1].datatype[10].sstruct.field[mymap].id_v6 707189723
+documenttype[1].datatype[10].sstruct.field[mymap].name "mymap"
+documenttype[1].datatype[10].sstruct.field[structfield].datatype 2
+documenttype[1].datatype[10].sstruct.field[structfield].id 1726890940
+documenttype[1].datatype[10].sstruct.field[structfield].id_v6 418303145
+documenttype[1].datatype[10].sstruct.field[structfield].name "structfield"
+documenttype[1].datatype[10].sstruct.field[title].datatype 2
+documenttype[1].datatype[10].sstruct.field[title].id 567626448
+documenttype[1].datatype[10].sstruct.field[title].id_v6 29129762
+documenttype[1].datatype[10].sstruct.field[title].name "title"
+documenttype[1].datatype[11].id -1408707420
+documenttype[1].datatype[11].type STRUCT
+documenttype[1].datatype[11].sstruct.name "types_search.body"
+documenttype[1].datatype[11].sstruct.version 0
+documenttype[1].inherits[1]
+documenttype[1].inherits[document.0].id 8
diff --git a/document/src/tests/serialization/.gitignore b/document/src/tests/serialization/.gitignore
new file mode 100644
index 00000000000..9f5bc440533
--- /dev/null
+++ b/document/src/tests/serialization/.gitignore
@@ -0,0 +1,6 @@
+*_test
+.depend
+Makefile
+document_annotationserializer_test_app
+document_compression_test_app
+document_vespadocumentserializer_test_app
diff --git a/document/src/tests/serialization/CMakeLists.txt b/document/src/tests/serialization/CMakeLists.txt
new file mode 100644
index 00000000000..dcbf067e4a5
--- /dev/null
+++ b/document/src/tests/serialization/CMakeLists.txt
@@ -0,0 +1,28 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(document_vespadocumentserializer_test_app
+    SOURCES
+    vespadocumentserializer_test.cpp
+    DEPENDS
+    document
+    AFTER
+    document_documentconfig
+)
+vespa_add_test(NAME document_vespadocumentserializer_test_app COMMAND document_vespadocumentserializer_test_app)
+vespa_add_executable(document_annotationserializer_test_app
+    SOURCES
+    annotationserializer_test.cpp
+    DEPENDS
+    document
+    AFTER
+    document_documentconfig
+)
+vespa_add_test(NAME document_annotationserializer_test_app COMMAND document_annotationserializer_test_app)
+vespa_add_executable(document_compression_test_app
+    SOURCES
+    compression_test.cpp
+    DEPENDS
+    document
+    AFTER
+    document_documentconfig
+)
+vespa_add_test(NAME document_compression_test_app COMMAND document_compression_test_app)
diff --git a/document/src/tests/serialization/annotation.serialize.test.cfg b/document/src/tests/serialization/annotation.serialize.test.cfg
new file mode 100644
index 00000000000..a8a79ddc6f2
--- /dev/null
+++ b/document/src/tests/serialization/annotation.serialize.test.cfg
@@ -0,0 +1,77 @@
+enablecompression false
+datatype[5]
+datatype[0].id 10001
+datatype[0].arraytype[0]
+datatype[0].weightedsettype[0]
+datatype[0].structtype[1]
+datatype[0].structtype[0].name "empty"
+datatype[0].structtype[0].version 0
+datatype[0].structtype[0].field[0]
+datatype[0].structtype[0].inherits[0]
+datatype[0].documenttype[0]
+datatype[0].annotationreftype[0]
+datatype[1].id 10012
+datatype[1].arraytype[0]
+datatype[1].weightedsettype[0]
+datatype[1].structtype[1]
+datatype[1].structtype[0].name "myposition"
+datatype[1].structtype[0].version 0
+datatype[1].structtype[0].field[2]
+datatype[1].structtype[0].field[0].name "latitude"
+datatype[1].structtype[0].field[0].datatype 4
+datatype[1].structtype[0].field[0].id[0]
+datatype[1].structtype[0].field[1].name "longitude"
+datatype[1].structtype[0].field[1].datatype 4
+datatype[1].structtype[0].field[1].id[0]
+datatype[1].structtype[0].inherits[0]
+datatype[1].documenttype[0]
+datatype[1].annotationreftype[0]
+datatype[2].id 10022
+datatype[2].arraytype[0]
+datatype[2].weightedsettype[0]
+datatype[2].structtype[0]
+datatype[2].documenttype[0]
+datatype[2].annotationreftype[1]
+datatype[2].annotationreftype[0].annotation "text"
+datatype[3].id 10032
+datatype[3].arraytype[1]
+datatype[3].arraytype[0].datatype 10022
+datatype[3].weightedsettype[0]
+datatype[3].structtype[0]
+datatype[3].documenttype[0]
+datatype[3].annotationreftype[0]
+datatype[4].id 10002
+datatype[4].arraytype[0]
+datatype[4].weightedsettype[0]
+datatype[4].structtype[1]
+datatype[4].structtype[0].name "annotation.city"
+datatype[4].structtype[0].version 0
+datatype[4].structtype[0].field[2]
+datatype[4].structtype[0].field[0].name "position"
+datatype[4].structtype[0].field[0].datatype 10012
+datatype[4].structtype[0].field[0].id[0]
+datatype[4].structtype[0].field[1].name "references"
+datatype[4].structtype[0].field[1].datatype 10032
+datatype[4].structtype[0].field[1].id[0]
+datatype[4].structtype[0].inherits[0]
+datatype[4].documenttype[0]
+datatype[4].annotationreftype[0]
+annotationtype[6]
+annotationtype[0].name "text"
+annotationtype[0].id 20000
+annotationtype[0].datatype 10001
+annotationtype[1].name "begintag"
+annotationtype[1].id 20001
+annotationtype[1].datatype 10001
+annotationtype[2].name "endtag"
+annotationtype[2].id 20002
+annotationtype[2].datatype 10001
+annotationtype[3].name "body"
+annotationtype[3].id 20003
+annotationtype[3].datatype 10001
+annotationtype[4].name "paragraph"
+annotationtype[4].id 20004
+annotationtype[4].datatype 10001
+annotationtype[5].name "city"
+annotationtype[5].id 20005
+annotationtype[5].datatype 10002
diff --git a/document/src/tests/serialization/annotation.serialize.test.repo.cfg b/document/src/tests/serialization/annotation.serialize.test.repo.cfg
new file mode 100644
index 00000000000..d7fa447fa34
--- /dev/null
+++ b/document/src/tests/serialization/annotation.serialize.test.repo.cfg
@@ -0,0 +1,130 @@
+enablecompression false
+documenttype[1]
+documenttype[0].id 1000
+documenttype[0].name "my_document"
+documenttype[0].version 0
+documenttype[0].headerstruct -284186494
+documenttype[0].bodystruct -2039820521
+documenttype[0].inherits[0]
+documenttype[0].datatype[5]
+documenttype[0].datatype[0].id 10001
+documenttype[0].datatype[0].type STRUCT
+documenttype[0].datatype[0].array.element.id 0
+documenttype[0].datatype[0].map.key.id 0
+documenttype[0].datatype[0].map.value.id 0
+documenttype[0].datatype[0].wset.key.id 0
+documenttype[0].datatype[0].wset.createifnonexistent false
+documenttype[0].datatype[0].wset.removeifzero false
+documenttype[0].datatype[0].annotationref.annotation.id 0
+documenttype[0].datatype[0].sstruct.name "empty"
+documenttype[0].datatype[0].sstruct.version 0
+documenttype[0].datatype[0].sstruct.compression.type NONE
+documenttype[0].datatype[0].sstruct.compression.level 0
+documenttype[0].datatype[0].sstruct.compression.threshold 90
+documenttype[0].datatype[0].sstruct.compression.minsize 0
+documenttype[0].datatype[0].sstruct.field[0]
+documenttype[0].datatype[1].id 10012
+documenttype[0].datatype[1].type STRUCT
+documenttype[0].datatype[1].array.element.id 0
+documenttype[0].datatype[1].map.key.id 0
+documenttype[0].datatype[1].map.value.id 0
+documenttype[0].datatype[1].wset.key.id 0
+documenttype[0].datatype[1].wset.createifnonexistent false
+documenttype[0].datatype[1].wset.removeifzero false
+documenttype[0].datatype[1].annotationref.annotation.id 0
+documenttype[0].datatype[1].sstruct.name "myposition"
+documenttype[0].datatype[1].sstruct.version 0
+documenttype[0].datatype[1].sstruct.compression.type NONE
+documenttype[0].datatype[1].sstruct.compression.level 0
+documenttype[0].datatype[1].sstruct.compression.threshold 90
+documenttype[0].datatype[1].sstruct.compression.minsize 0
+documenttype[0].datatype[1].sstruct.field[2]
+documenttype[0].datatype[1].sstruct.field[0].name "latitude"
+documenttype[0].datatype[1].sstruct.field[0].id 945500241
+documenttype[0].datatype[1].sstruct.field[0].id_v6 1437005468
+documenttype[0].datatype[1].sstruct.field[0].datatype 4
+documenttype[0].datatype[1].sstruct.field[1].name "longitude"
+documenttype[0].datatype[1].sstruct.field[1].id 1458007305
+documenttype[0].datatype[1].sstruct.field[1].id_v6 1517975127
+documenttype[0].datatype[1].sstruct.field[1].datatype 4
+documenttype[0].datatype[2].id 10022
+documenttype[0].datatype[2].type ANNOTATIONREF
+documenttype[0].datatype[2].array.element.id 0
+documenttype[0].datatype[2].map.key.id 0
+documenttype[0].datatype[2].map.value.id 0
+documenttype[0].datatype[2].wset.key.id 0
+documenttype[0].datatype[2].wset.createifnonexistent false
+documenttype[0].datatype[2].wset.removeifzero false
+documenttype[0].datatype[2].annotationref.annotation.id 20000
+documenttype[0].datatype[2].sstruct.name ""
+documenttype[0].datatype[2].sstruct.version 0
+documenttype[0].datatype[2].sstruct.compression.type NONE
+documenttype[0].datatype[2].sstruct.compression.level 0
+documenttype[0].datatype[2].sstruct.compression.threshold 95
+documenttype[0].datatype[2].sstruct.compression.minsize 200
+documenttype[0].datatype[2].sstruct.field[0]
+documenttype[0].datatype[3].id 10032
+documenttype[0].datatype[3].type ARRAY
+documenttype[0].datatype[3].array.element.id 10022
+documenttype[0].datatype[3].map.key.id 0
+documenttype[0].datatype[3].map.value.id 0
+documenttype[0].datatype[3].wset.key.id 0
+documenttype[0].datatype[3].wset.createifnonexistent false
+documenttype[0].datatype[3].wset.removeifzero false
+documenttype[0].datatype[3].annotationref.annotation.id 0
+documenttype[0].datatype[3].sstruct.name ""
+documenttype[0].datatype[3].sstruct.version 0
+documenttype[0].datatype[3].sstruct.compression.type NONE
+documenttype[0].datatype[3].sstruct.compression.level 0
+documenttype[0].datatype[3].sstruct.compression.threshold 95
+documenttype[0].datatype[3].sstruct.compression.minsize 200
+documenttype[0].datatype[3].sstruct.field[0]
+documenttype[0].datatype[4].id 10002
+documenttype[0].datatype[4].type STRUCT
+documenttype[0].datatype[4].array.element.id 0
+documenttype[0].datatype[4].map.key.id 0
+documenttype[0].datatype[4].map.value.id 0
+documenttype[0].datatype[4].wset.key.id 0
+documenttype[0].datatype[4].wset.createifnonexistent false
+documenttype[0].datatype[4].wset.removeifzero false
+documenttype[0].datatype[4].annotationref.annotation.id 0
+documenttype[0].datatype[4].sstruct.name "annotation.city"
+documenttype[0].datatype[4].sstruct.version 0
+documenttype[0].datatype[4].sstruct.compression.type NONE
+documenttype[0].datatype[4].sstruct.compression.level 0
+documenttype[0].datatype[4].sstruct.compression.threshold 90
+documenttype[0].datatype[4].sstruct.compression.minsize 0
+documenttype[0].datatype[4].sstruct.field[2]
+documenttype[0].datatype[4].sstruct.field[0].name "position"
+documenttype[0].datatype[4].sstruct.field[0].id 818217702
+documenttype[0].datatype[4].sstruct.field[0].id_v6 632310942
+documenttype[0].datatype[4].sstruct.field[0].datatype 10012
+documenttype[0].datatype[4].sstruct.field[1].name "references"
+documenttype[0].datatype[4].sstruct.field[1].id 1581548957
+documenttype[0].datatype[4].sstruct.field[1].id_v6 839497794
+documenttype[0].datatype[4].sstruct.field[1].datatype 10032
+documenttype[0].annotationtype[6]
+documenttype[0].annotationtype[0].id 20000
+documenttype[0].annotationtype[0].name "text"
+documenttype[0].annotationtype[0].datatype 10001
+documenttype[0].annotationtype[0].inherits[0]
+documenttype[0].annotationtype[1].id 20001
+documenttype[0].annotationtype[1].name "begintag"
+documenttype[0].annotationtype[1].datatype 10001
+documenttype[0].annotationtype[1].inherits[0]
+documenttype[0].annotationtype[2].id 20002
+documenttype[0].annotationtype[2].name "endtag"
+documenttype[0].annotationtype[2].datatype 10001
+documenttype[0].annotationtype[2].inherits[0]
+documenttype[0].annotationtype[3].id 20003
+documenttype[0].annotationtype[3].name "body"
+documenttype[0].annotationtype[3].datatype 10001
+documenttype[0].annotationtype[3].inherits[0]
+documenttype[0].annotationtype[4].id 20004
+documenttype[0].annotationtype[4].name "paragraph"
+documenttype[0].annotationtype[4].datatype 10001
+documenttype[0].annotationtype[4].inherits[0]
+documenttype[0].annotationtype[5].id 20005
+documenttype[0].annotationtype[5].name "city"
+documenttype[0].annotationtype[5].datatype 10002
+documenttype[0].annotationtype[5].inherits[0]
diff --git a/document/src/tests/serialization/annotationserializer_test.cpp b/document/src/tests/serialization/annotationserializer_test.cpp
new file mode 100644
index 00000000000..1cc3fc19373
--- /dev/null
+++ b/document/src/tests/serialization/annotationserializer_test.cpp
@@ -0,0 +1,274 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Unit tests for annotation serialization.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("annotationserializer_test");
+
+#include <stdlib.h>
+#include <vespa/document/annotation/alternatespanlist.h>
+#include <vespa/document/annotation/annotation.h>
+#include <vespa/document/annotation/span.h>
+#include <vespa/document/annotation/spanlist.h>
+#include <vespa/document/annotation/spantree.h>
+#include <vespa/document/config/config-documenttypes.h>
+#include <vespa/document/fieldvalue/stringfieldvalue.h>
+#include <vespa/document/repo/fixedtyperepo.h>
+#include <vespa/document/serialization/annotationdeserializer.h>
+#include <vespa/document/serialization/annotationserializer.h>
+#include <vespa/document/serialization/vespadocumentdeserializer.h>
+#include <vespa/document/serialization/vespadocumentserializer.h>
+#include <vespa/vespalib/objects/nbostream.h>
+#include <vespa/vespalib/testkit/testapp.h>
+#include <fstream>
+#include <sstream>
+#include <vector>
+
+using document::DocumenttypesConfig;
+using std::fstream;
+using std::ostringstream;
+using std::string;
+using std::vector;
+using vespalib::nbostream;
+using namespace document;
+
+namespace {
+
+class Test : public vespalib::TestApp {
+    StringFieldValue::SpanTrees readSpanTree(const string &file_name, const FixedTypeRepo &repo);
+
+    void requireThatSimpleSpanTreeIsDeserialized();
+    void requireThatAdvancedSpanTreeIsDeserialized();
+    void requireThatSpanTreeCanBeSerialized();
+    void requireThatUnknownAnnotationIsSkipped();
+
+public:
+    int Main();
+};
+
+int
+Test::Main()
+{
+    if (getenv("TEST_SUBSET") != 0) { return 0; }
+    TEST_INIT("annotationserializer_test");
+    TEST_DO(requireThatSimpleSpanTreeIsDeserialized());
+    TEST_DO(requireThatAdvancedSpanTreeIsDeserialized());
+    TEST_DO(requireThatSpanTreeCanBeSerialized());
+    TEST_DO(requireThatUnknownAnnotationIsSkipped());
+
+    TEST_DONE();
+}
+
+template <typename T, int N> int arraysize(const T (&)[N]) { return N; }
+
+StringFieldValue::SpanTrees
+Test::readSpanTree(const string &file_name, const FixedTypeRepo &repo) {
+    FastOS_File file(file_name.c_str());
+    ASSERT_TRUE(file.OpenReadOnlyExisting());
+    char buffer[1024];
+    ssize_t size = file.Read(buffer, arraysize(buffer));
+    ASSERT_TRUE(size != -1);
+
+    nbostream stream(buffer, size);
+    VespaDocumentDeserializer deserializer(repo, stream, 8);
+    StringFieldValue value;
+    deserializer.read(value);
+
+    EXPECT_EQUAL(0u, stream.size());
+    ASSERT_TRUE(value.hasSpanTrees());
+    return value.getSpanTrees();
+}
+
+void Test::requireThatSimpleSpanTreeIsDeserialized() {
+    DocumentTypeRepo type_repo(readDocumenttypesConfig("annotation.serialize.test.repo.cfg"));
+    FixedTypeRepo repo(type_repo);
+    SpanTree::UP span_tree = std::move(readSpanTree("test_data_serialized_simple", repo).front());
+
+    EXPECT_EQUAL("html", span_tree->getName());
+    const SimpleSpanList *root = dynamic_cast<const SimpleSpanList *>(&span_tree->getRoot());
+    ASSERT_TRUE(root);
+    EXPECT_EQUAL(5u, root->size());
+    SimpleSpanList::const_iterator it = root->begin();
+    EXPECT_EQUAL(Span(0, 19), (*it++));
+    EXPECT_EQUAL(Span(19, 5), (*it++));
+    EXPECT_EQUAL(Span(24, 21), (*it++));
+    EXPECT_EQUAL(Span(45, 23), (*it++));
+    EXPECT_EQUAL(Span(68, 14), (*it++));
+    EXPECT_TRUE(it == root->end());
+}
+
+struct AnnotationComparator {
+    vector<string> expect;
+    vector<string> actual;
+    template <typename ITR>
+    AnnotationComparator &addActual(ITR pos, ITR end) {
+        for (; pos != end; ++pos) {
+            ostringstream ost;
+            pos->print(ost, true, "");
+            actual.push_back(ost.str());
+        }
+        return *this;
+    }
+    AnnotationComparator &addExpected(const string &e) {
+        expect.push_back(e);
+        return *this;
+    }
+    void compare() {
+        std::sort(expect.begin(), expect.end());
+        std::sort(actual.begin(), actual.end());
+        EXPECT_EQUAL(expect.size(), actual.size());
+        for (size_t i = 0; i < expect.size() && i < actual.size(); ++i) {
+            EXPECT_EQUAL(expect[i].size(), actual[i].size());
+            EXPECT_EQUAL(expect[i], actual[i]);
+        }
+    }
+};
+
+void Test::requireThatAdvancedSpanTreeIsDeserialized() {
+    DocumentTypeRepo type_repo(
+            readDocumenttypesConfig("annotation.serialize.test.repo.cfg"));
+    FixedTypeRepo repo(type_repo, "my_document");
+    SpanTree::UP span_tree = std::move(readSpanTree("test_data_serialized_advanced", repo).front());
+
+    EXPECT_EQUAL("html", span_tree->getName());
+    const SpanList *root = dynamic_cast<const SpanList *>(&span_tree->getRoot());
+    ASSERT_TRUE(root);
+    EXPECT_EQUAL(4u, root->size());
+    SpanList::const_iterator it = root->begin();
+    EXPECT_EQUAL(Span(0, 6), *(static_cast<Span *>(*it++)));
+    AlternateSpanList *alt_list = dynamic_cast<AlternateSpanList *>(*it++);
+    EXPECT_EQUAL(Span(27, 9), *(static_cast<Span *>(*it++)));
+    EXPECT_EQUAL(Span(36, 8), *(static_cast<Span *>(*it++)));
+    EXPECT_TRUE(it == root->end());
+
+    ASSERT_TRUE(alt_list);
+    EXPECT_EQUAL(2u, alt_list->getNumSubtrees());
+    EXPECT_EQUAL(0.9, alt_list->getProbability(0));
+    EXPECT_EQUAL(0.1, alt_list->getProbability(1));
+    EXPECT_EQUAL(4u, alt_list->getSubtree(0).size());
+    it = alt_list->getSubtree(0).begin();
+    EXPECT_EQUAL(Span(6, 3), *(static_cast<Span *>(*it++)));
+    EXPECT_EQUAL(Span(9, 10), *(static_cast<Span *>(*it++)));
+    EXPECT_EQUAL(Span(19, 4), *(static_cast<Span *>(*it++)));
+    EXPECT_EQUAL(Span(23, 4), *(static_cast<Span *>(*it++)));
+    EXPECT_TRUE(it == alt_list->getSubtree(0).end());
+    EXPECT_EQUAL(2u, alt_list->getSubtree(1).size());
+    it = alt_list->getSubtree(1).begin();
+    EXPECT_EQUAL(Span(6, 13), *(static_cast<Span *>(*it++)));
+    EXPECT_EQUAL(Span(19, 8), *(static_cast<Span *>(*it++)));
+    EXPECT_TRUE(it == alt_list->getSubtree(1).end());
+
+    EXPECT_EQUAL(12u, span_tree->numAnnotations());
+
+    AnnotationComparator comparator;
+    comparator.addActual(span_tree->begin(), span_tree->end())
+        .addExpected("Annotation(AnnotationType(20001, begintag)\n"
+                     "  Span(6, 3))")
+        .addExpected("Annotation(AnnotationType(20000, text)\n"
+                     "  Span(9, 10))")
+        .addExpected("Annotation(AnnotationType(20000, text)\n"
+                     "  Span(19, 4))")
+        .addExpected("Annotation(AnnotationType(20002, endtag)\n"
+                     "  Span(23, 4))")
+        .addExpected("Annotation(AnnotationType(20000, text)\n"
+                     "  Span(6, 13))")
+        .addExpected("Annotation(AnnotationType(20003, body)\n"
+                     "  Span(19, 8))")
+        .addExpected("Annotation(AnnotationType(20004, paragraph)\n"
+                     "  AlternateSpanList(\n"
+                     "    Probability 0.9 : SpanList(\n"
+                     "      Span(6, 3)\n"
+                     "      Span(9, 10)\n"
+                     "      Span(19, 4)\n"
+                     "      Span(23, 4)),\n"
+                     "    Probability 0.1 : SpanList(\n"
+                     "      Span(6, 13)\n"
+                     "      Span(19, 8))))")
+        .addExpected("Annotation(AnnotationType(20001, begintag)\n"
+                     "  Span(0, 6))")
+        .addExpected("Annotation(AnnotationType(20000, text)\n"
+                     "  Span(27, 9))")
+        .addExpected("Annotation(AnnotationType(20002, endtag)\n"
+                     "  Span(36, 8))")
+        .addExpected("Annotation(AnnotationType(20003, body)\n"
+                     "  SpanList(\n"
+                     "    Span(0, 6)\n"
+                     "    AlternateSpanList(\n"
+                     "      Probability 0.9 : SpanList(\n"
+                     "        Span(6, 3)\n"
+                     "        Span(9, 10)\n"
+                     "        Span(19, 4)\n"
+                     "        Span(23, 4)),\n"
+                     "      Probability 0.1 : SpanList(\n"
+                     "        Span(6, 13)\n"
+                     "        Span(19, 8)))\n"
+                     "    Span(27, 9)\n"
+                     "    Span(36, 8)))")
+        .addExpected("Annotation(AnnotationType(20005, city)\n"
+                     "  Struct annotation.city(\n"
+                     "    position - Struct myposition(\n"
+                     "      latitude - 37,\n"
+                     "      longitude - -122\n"
+                     "    ),\n"
+                     "    references - Array(size: 2,\n"
+                     "      AnnotationReferenceFieldValue(n),\n"
+                     "      AnnotationReferenceFieldValue(n)\n"
+                     "    )\n"
+                     "  ))");
+    TEST_DO(comparator.compare());
+}
+
+void Test::requireThatSpanTreeCanBeSerialized() {
+    DocumentTypeRepo type_repo(
+            readDocumenttypesConfig("annotation.serialize.test.repo.cfg"));
+    FixedTypeRepo repo(type_repo, "my_document");
+    string file_name = "test_data_serialized_advanced";
+
+    FastOS_File file(file_name.c_str());
+    ASSERT_TRUE(file.OpenReadOnlyExisting());
+    char buffer[1024];
+    ssize_t size = file.Read(buffer, arraysize(buffer));
+    ASSERT_TRUE(size != -1);
+
+    nbostream stream(buffer, size);
+    VespaDocumentDeserializer deserializer(repo, stream, 8);
+    StringFieldValue value;
+    deserializer.read(value);
+
+    SpanTree::UP span_tree = std::move(value.getSpanTrees().front());
+    EXPECT_EQUAL("html", span_tree->getName());
+    EXPECT_EQUAL(0u, stream.size());
+
+    stream.clear();
+    VespaDocumentSerializer serializer(stream);
+    serializer.write(value);
+    EXPECT_EQUAL(size, static_cast<ssize_t>(stream.size()));
+    int diff_count = 0;
+    for (size_t i = 0; i < stream.size(); ++i) {
+        if (buffer[i] != stream.peek()[i]) {
+            ++diff_count;
+        }
+        EXPECT_EQUAL((int) buffer[i], (int) stream.peek()[i]);
+    }
+    EXPECT_EQUAL(0, diff_count);
+}
+
+void Test::requireThatUnknownAnnotationIsSkipped() {
+    AnnotationType type(42, "my type");
+    Annotation annotation(type, FieldValue::UP(new StringFieldValue("foo")));
+    nbostream stream;
+    AnnotationSerializer serializer(stream);
+    serializer.write(annotation);
+
+    DocumentTypeRepo type_repo;  // Doesn't know any annotation types.
+    FixedTypeRepo repo(type_repo);
+    AnnotationDeserializer deserializer(repo, stream, 8);
+    Annotation a;
+    deserializer.readAnnotation(a);
+    EXPECT_FALSE(a.valid());
+    EXPECT_EQUAL(0u, stream.size());
+}
+
+}  // namespace
+
+TEST_APPHOOK(Test);
diff --git a/document/src/tests/serialization/compression_test.cpp b/document/src/tests/serialization/compression_test.cpp
new file mode 100644
index 00000000000..01e19ae946b
--- /dev/null
+++ b/document/src/tests/serialization/compression_test.cpp
@@ -0,0 +1,30 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Unit tests for annotation serialization.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+#include <vespa/vespalib/testkit/test_kit.h>
+#include <vespa/vespalib/stllike/string.h>
+#include <vespa/document/util/compressor.h>
+
+LOG_SETUP("compression_test");
+
+using namespace document;
+using namespace vespalib;
+
+static vespalib::string _G_compressableText("AAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEEEEE"
+                                            "AAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEEEEE"
+                                            "AAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEEEEE"
+                                            "AAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEEEEE"
+                                            "AAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEEEEE");
+
+TEST("requireThatLZ4CompressFine") {
+    CompressionConfig cfg(CompressionConfig::Type::LZ4);
+    ConstBufferRef ref(_G_compressableText.c_str(), _G_compressableText.size());
+    DataBuffer compressed;
+    EXPECT_EQUAL(CompressionConfig::Type::LZ4, compress(cfg, ref, compressed, false));
+}
+
+TEST_MAIN() {
+    TEST_RUN_ALL();
+}
diff --git a/document/src/tests/serialization/test_data_serialized_advanced b/document/src/tests/serialization/test_data_serialized_advanced
new file mode 100644
index 00000000000..d5d6ca6022a
--- /dev/null
+++ b/document/src/tests/serialization/test_data_serialized_advanced
diff --git a/document/src/tests/serialization/test_data_serialized_simple b/document/src/tests/serialization/test_data_serialized_simple
new file mode 100644
index 00000000000..ae65a2eefb4
--- /dev/null
+++ b/document/src/tests/serialization/test_data_serialized_simple
diff --git a/document/src/tests/serialization/vespadocumentserializer_test.cpp b/document/src/tests/serialization/vespadocumentserializer_test.cpp
new file mode 100644
index 00000000000..0043548a906
--- /dev/null
+++ b/document/src/tests/serialization/vespadocumentserializer_test.cpp
@@ -0,0 +1,828 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Unit tests for vespadocumentserializer.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("vespadocumentserializer_test");
+
+#include <vespa/document/annotation/annotation.h>
+#include <vespa/document/annotation/span.h>
+#include <vespa/document/annotation/spantree.h>
+#include <vespa/document/config/config-documenttypes.h>
+#include <vespa/document/datatype/annotationreferencedatatype.h>
+#include <vespa/document/datatype/arraydatatype.h>
+#include <vespa/document/datatype/documenttype.h>
+#include <vespa/document/fieldvalue/annotationreferencefieldvalue.h>
+#include <vespa/document/fieldvalue/arrayfieldvalue.h>
+#include <vespa/document/fieldvalue/bytefieldvalue.h>
+#include <vespa/document/fieldvalue/document.h>
+#include <vespa/document/fieldvalue/doublefieldvalue.h>
+#include <vespa/document/fieldvalue/floatfieldvalue.h>
+#include <vespa/document/fieldvalue/intfieldvalue.h>
+#include <vespa/document/fieldvalue/longfieldvalue.h>
+#include <vespa/document/fieldvalue/mapfieldvalue.h>
+#include <vespa/document/fieldvalue/predicatefieldvalue.h>
+#include <vespa/document/fieldvalue/rawfieldvalue.h>
+#include <vespa/document/fieldvalue/shortfieldvalue.h>
+#include <vespa/document/fieldvalue/stringfieldvalue.h>
+#include <vespa/document/fieldvalue/structfieldvalue.h>
+#include <vespa/document/fieldvalue/weightedsetfieldvalue.h>
+#include <vespa/document/fieldvalue/tensorfieldvalue.h>
+#include <vespa/document/predicate/predicate.h>
+#include <vespa/document/predicate/predicate_slime_builder.h>
+#include <vespa/document/repo/configbuilder.h>
+#include <vespa/document/repo/fixedtyperepo.h>
+#include <vespa/document/serialization/util.h>
+#include <vespa/document/serialization/vespadocumentdeserializer.h>
+#include <vespa/document/serialization/vespadocumentserializer.h>
+#include <vespa/document/serialization/annotationserializer.h>
+#include <vespa/vespalib/tensor/types.h>
+#include <vespa/vespalib/tensor/tensor_builder.h>
+#include <vespa/vespalib/tensor/tensor.h>
+#include <vespa/vespalib/tensor/default_tensor.h>
+#include <vespa/vespalib/tensor/tensor_factory.h>
+#include <stdlib.h>
+#include <vespa/vespalib/data/slime/cursor.h>
+#include <vespa/vespalib/data/slime/slime.h>
+#include <vespa/vespalib/io/fileutil.h>
+#include <vespa/vespalib/objects/nbostream.h>
+#include <vespa/vespalib/testkit/testapp.h>
+
+using document::DocumenttypesConfig;
+using vespalib::File;
+using vespalib::Slime;
+using vespalib::nbostream;
+using vespalib::slime::Cursor;
+using vespalib::tensor::Tensor;
+using vespalib::tensor::TensorBuilder;
+using vespalib::tensor::TensorCells;
+using vespalib::tensor::TensorDimensions;
+using namespace document;
+using std::string;
+using std::vector;
+using namespace document::config_builder;
+
+namespace {
+
+const int doc_type_id = 1234;
+const string doc_name = "my document";
+const int body_id = 94;
+const int inner_type_id = 95;
+const int outer_type_id = 96;
+const string type_name = "outer doc";
+const string inner_name = "inner doc";
+const int a_id = 12345;
+const string a_name = "annotation";
+const int predicate_doc_type_id = 321;
+const string predicate_field_name = "my_predicate";
+
+DocumenttypesConfig getDocTypesConfig() {
+    DocumenttypesConfigBuilderHelper builder;
+    builder.document(doc_type_id, doc_name,
+                     Struct("my document.header")
+                     .addField("header field", DataType::T_INT),
+                     Struct("my document.body")
+                     .addField("body field", DataType::T_STRING))
+        .annotationType(42, "foo_type", DataType::T_INT);
+    builder.document(inner_type_id, inner_name,
+                     Struct(inner_name + ".header"),
+                     Struct(inner_name + ".body")
+                     .addField("str", DataType::T_STRING))
+        .annotationType(a_id, a_name, DataType::T_STRING);
+    builder.document(outer_type_id, type_name,
+                     Struct(type_name + ".header"),
+                     Struct(type_name + ".body")
+                     .addField(inner_name, inner_type_id).setId(body_id));
+    builder.document(predicate_doc_type_id, "my_type",
+                     Struct("my_type.header"),
+                     Struct("my_type.body")
+                     .addField(predicate_field_name, DataType::T_PREDICATE));
+    return builder.config();
+}
+
+const DocumentTypeRepo doc_repo(getDocTypesConfig());
+const FixedTypeRepo repo(doc_repo, *doc_repo.getDocumentType(doc_type_id));
+
+template <typename T> T newFieldValue(const T&) { return T(); }
+template <> ArrayFieldValue newFieldValue(const ArrayFieldValue &value)
+{ return ArrayFieldValue(*value.getDataType()); }
+template <> MapFieldValue newFieldValue(const MapFieldValue &value)
+{ return MapFieldValue(*value.getDataType()); }
+template <> WeightedSetFieldValue newFieldValue(
+        const WeightedSetFieldValue &value)
+{ return WeightedSetFieldValue(*value.getDataType()); }
+template <> StructFieldValue newFieldValue(const StructFieldValue &value)
+{ return StructFieldValue(*value.getDataType()); }
+template <> AnnotationReferenceFieldValue newFieldValue(
+        const AnnotationReferenceFieldValue &value)
+{ return AnnotationReferenceFieldValue(*value.getDataType()); }
+
+template<typename T>
+void testDeserializeAndClone(const T& value, const nbostream &stream, bool checkEqual=true) {
+    T read_value = newFieldValue(value);
+    vespalib::MallocPtr buf(stream.size());
+    memcpy(buf.str(), stream.peek(), stream.size());
+    nbostream is(buf.c_str(), buf.size(), true);
+    VespaDocumentDeserializer deserializer(repo, is, 8);
+    deserializer.read(read_value);
+
+    EXPECT_EQUAL(0u, is.size());
+    if (checkEqual) {
+        EXPECT_EQUAL(value, read_value);
+    }
+    T clone(read_value);
+    buf.reset();
+    if (checkEqual) {
+        EXPECT_EQUAL(value, clone);
+    }
+}
+
+// Leaves the stream's read position at the start of the serialized object.
+template<typename T>
+void serializeAndDeserialize(const T& value, nbostream &stream, bool checkEqual=true) {
+    size_t start_size = stream.size();
+    VespaDocumentSerializer serializer(stream);
+    serializer.write(value);
+    size_t serialized_size = stream.size() - start_size;
+
+    testDeserializeAndClone(value, stream, checkEqual);
+    T read_value = newFieldValue(value);
+
+    VespaDocumentDeserializer deserializer(repo, stream, 8);
+    deserializer.read(read_value);
+
+    EXPECT_EQUAL(0u, stream.size());
+    if (checkEqual) {
+        EXPECT_EQUAL(value, read_value);
+    }
+    stream.adjustReadPos(-serialized_size);
+}
+
+template <typename T> struct ValueType { typedef typename T::Number Type; };
+template <> struct ValueType<IntFieldValue> { typedef uint32_t Type; };
+template <> struct ValueType<LongFieldValue> { typedef uint64_t Type; };
+
+template<typename T>
+void serializeAndDeserializeNumber(const T& value) {
+    const typename ValueType<T>::Type val = value.getValue();
+    nbostream stream;
+    serializeAndDeserialize(value, stream);
+
+    typename ValueType<T>::Type read_val;
+    stream >> read_val;
+    EXPECT_EQUAL(val, read_val);
+}
+
+TEST("requireThatPrimitiveTypeFieldValueCanBeSerialized") {
+    TEST_DO(serializeAndDeserializeNumber(ByteFieldValue(42)));
+    TEST_DO(serializeAndDeserializeNumber(ShortFieldValue(0x1234)));
+    TEST_DO(serializeAndDeserializeNumber(IntFieldValue(0x12345678)));
+    TEST_DO(serializeAndDeserializeNumber(DoubleFieldValue(34567890.123456)));
+    TEST_DO(serializeAndDeserializeNumber(FloatFieldValue(3456.1234f)));
+    TEST_DO(serializeAndDeserializeNumber(LongFieldValue(0x12345678123456LL)));
+}
+
+template <typename SizeType>
+void checkLiteralFieldValue(nbostream &stream, const string &val) {
+    uint8_t read_coding;
+    SizeType size;
+    stream >> read_coding >> size;
+    EXPECT_EQUAL(0, read_coding);
+    size &= (SizeType(-1) >> 1);  // Clear MSB.
+    vespalib::string read_val;
+    read_val.assign(stream.peek(), size);
+    stream.adjustReadPos(size);
+    EXPECT_EQUAL(val.size(), read_val.size());
+    EXPECT_EQUAL(val, read_val);
+}
+
+template <typename SizeType>
+void checkStringFieldValue(const string &val) {
+    StringFieldValue value(val);
+    nbostream stream;
+    serializeAndDeserialize(value, stream);
+
+    string val_with_null(val.c_str(), val.size() + 1);
+    checkLiteralFieldValue<SizeType>(stream, val_with_null);
+}
+
+void setSpanTree(StringFieldValue & str, const SpanTree & tree) {
+    nbostream os;
+    AnnotationSerializer serializer(os);
+    serializer.write(tree);
+    str.setSpanTrees(vespalib::ConstBufferRef(os.peek(), os.size()), repo, VespaDocumentSerializer::getCurrentVersion(), false);
+}
+
+void checkStringFieldValueWithAnnotation() {
+    StringFieldValue value("foo");
+    Span::UP root(new Span(2, 3));
+    SpanTree::UP tree(new SpanTree("test", std::move(root)));
+    AnnotationType annotation_type(42, "foo_type");
+    tree->annotate(tree->getRoot(), annotation_type);
+
+    nbostream os;
+    AnnotationSerializer serializer(os);
+    serializer.write(*tree);
+
+    value.setSpanTrees(vespalib::ConstBufferRef(os.peek(), os.size()), repo, VespaDocumentSerializer::getCurrentVersion(), true);
+    nbostream stream;
+    serializeAndDeserialize(value, stream);
+}
+
+TEST("requireThatStringFieldValueCanBeSerialized") {
+    TEST_DO(checkStringFieldValue<uint8_t>("foo bar baz"));
+    TEST_DO(checkStringFieldValue<uint32_t>(string(200, 'a')));
+    TEST_DO(checkStringFieldValueWithAnnotation());
+}
+
+template <typename SizeType>
+void checkRawFieldValue(const string &val) {
+    RawFieldValue value(val);
+    nbostream stream;
+    serializeAndDeserialize(value, stream);
+
+    uint32_t size;
+    stream >> size;
+    string read_val(stream.peek(), size);
+    stream.adjustReadPos(size);
+    EXPECT_EQUAL(val, read_val);
+}
+
+TEST("requireThatRawFieldValueCanBeSerialized") {
+    TEST_DO(checkRawFieldValue<uint8_t>("foo bar"));
+    TEST_DO(checkRawFieldValue<uint32_t>(string(200, 'b')));
+}
+
+TEST("requireThatPredicateFieldValueCanBeSerialized") {
+    PredicateSlimeBuilder builder;
+    builder.neg().feature("foo").value("bar").value("baz");
+    PredicateFieldValue value(builder.build());
+    nbostream stream;
+    serializeAndDeserialize(value, stream);
+}
+
+template <typename SizeType>
+SizeType readSize1_2_4(nbostream &stream) {
+    SizeType size;
+    stream >> size;
+    if (sizeof(SizeType) > 1) {
+        size &= (SizeType(-1) >> 2);  // Clear MSBs
+    }
+    return size;
+}
+
+template <typename SizeType>
+void checkArrayFieldValue(SizeType value_count) {
+    ArrayDataType array_type(*DataType::INT);
+    ArrayFieldValue value(array_type);
+    for (uint32_t i = 0; i < value_count; ++i) {
+        value.add(static_cast<int32_t>(i));
+    }
+
+    nbostream stream;
+    serializeAndDeserialize(value, stream);
+
+    SizeType size = readSize1_2_4<SizeType>(stream);
+    EXPECT_EQUAL(value_count, size);
+    uint32_t child;
+    for (uint32_t i = 0; i < value_count; ++i) {
+        stream >> child;
+        EXPECT_EQUAL(i, child);
+    }
+}
+
+TEST("requireThatArrayFieldValueCanBeSerialized") {
+    TEST_DO(checkArrayFieldValue<uint8_t>(2));
+    TEST_DO(checkArrayFieldValue<uint8_t>(0x7f));
+    TEST_DO(checkArrayFieldValue<uint16_t>(0x80));
+    TEST_DO(checkArrayFieldValue<uint16_t>(0x3fff));
+    TEST_DO(checkArrayFieldValue<uint32_t>(0x4000));
+}
+
+TEST("requireThatOldVersionArrayFieldValueCanBeDeserialized") {
+    uint16_t old_version = 6;
+
+    nbostream stream;
+    uint32_t type_id = 3;
+    uint32_t size = 2;
+    uint32_t element_size = 4;
+    uint32_t element1 = 21, element2 = 42;
+    stream << type_id << size
+           << element_size << element1
+           << element_size << element2;
+
+    ArrayDataType array_type(*DataType::INT);
+    ArrayFieldValue value(array_type);
+    VespaDocumentDeserializer deserializer(repo, stream, old_version);
+    deserializer.read(value);
+    ASSERT_TRUE(EXPECT_EQUAL(size, value.size()));
+    IntFieldValue *int_value = dynamic_cast<IntFieldValue *>(&value[0]);
+    ASSERT_TRUE(int_value);
+    EXPECT_EQUAL(element1, static_cast<uint32_t>(int_value->getValue()));
+    int_value = dynamic_cast<IntFieldValue *>(&value[1]);
+    ASSERT_TRUE(int_value);
+    EXPECT_EQUAL(element2, static_cast<uint32_t>(int_value->getValue()));
+}
+
+template <typename SizeType>
+void checkMapFieldValue(SizeType value_count, bool check_equal) {
+    MapDataType map_type(*DataType::LONG, *DataType::BYTE);
+    MapFieldValue value(map_type);
+    for (SizeType i = 0; i < value_count; ++i) {
+        value.push_back(LongFieldValue(i), ByteFieldValue(i));
+    }
+
+    nbostream stream;
+    serializeAndDeserialize(value, stream, check_equal);
+
+    SizeType size = readSize1_2_4<SizeType>(stream);
+    EXPECT_EQUAL(value_count, size);
+    uint64_t key;
+    uint8_t val;
+    for (SizeType i = 0; i < value_count; ++i) {
+        stream >> key >> val;
+        EXPECT_EQUAL(i, key);
+        EXPECT_EQUAL(i % 256, val);
+    }
+}
+
+TEST("requireThatMapFieldValueCanBeSerialized") {
+    TEST_DO(checkMapFieldValue<uint8_t>(2, true));
+    TEST_DO(checkMapFieldValue<uint8_t>(0x7f, true));
+    TEST_DO(checkMapFieldValue<uint16_t>(0x80, true));
+    TEST_DO(checkMapFieldValue<uint16_t>(0x3fff, false));
+    TEST_DO(checkMapFieldValue<uint32_t>(0x4000, false));
+}
+
+TEST("requireThatOldVersionMapFieldValueCanBeDeserialized") {
+    uint16_t old_version = 6;
+
+    nbostream stream;
+    uint32_t type_id = 4;
+    uint32_t size = 2;
+    uint32_t element_size = 9;
+    uint64_t key1 = 21, key2 = 42;
+    uint8_t val1 = 1, val2 = 2;
+    stream << type_id << size
+           << element_size << key1 << val1
+           << element_size << key2 << val2;
+
+    MapDataType map_type(*DataType::LONG, *DataType::BYTE);
+    MapFieldValue value(map_type);
+    VespaDocumentDeserializer deserializer(repo, stream, old_version);
+    deserializer.read(value);
+    ASSERT_TRUE(EXPECT_EQUAL(size, value.size()));
+
+    ASSERT_TRUE(value.contains(LongFieldValue(key1)));
+    ASSERT_TRUE(value.contains(LongFieldValue(key2)));
+    EXPECT_EQUAL(ByteFieldValue(val1),
+               *value.find(LongFieldValue(key1))->second);
+    EXPECT_EQUAL(ByteFieldValue(val2),
+               *value.find(LongFieldValue(key2))->second);
+}
+
+TEST("requireThatWeightedSetFieldValueCanBeSerialized") {
+    WeightedSetDataType ws_type(*DataType::DOUBLE, false, false);
+    WeightedSetFieldValue value(ws_type);
+    value.add(DoubleFieldValue(3.14), 2);
+    value.add(DoubleFieldValue(2.71), 3);
+
+    nbostream stream;
+    serializeAndDeserialize(value, stream);
+
+    uint32_t type_id;
+    uint32_t size;
+    stream >> type_id >> size;
+    EXPECT_EQUAL(2u, size);
+    double val;
+    uint32_t weight;
+    stream >> size >> val >> weight;
+    EXPECT_EQUAL(12u, size);
+    EXPECT_EQUAL(3.14, val);
+    EXPECT_EQUAL(2u, weight);
+    stream >> size >> val >> weight;
+    EXPECT_EQUAL(12u, size);
+    EXPECT_EQUAL(2.71, val);
+    EXPECT_EQUAL(3u, weight);
+}
+
+const Field field1("field1", *DataType::INT, false);
+const Field field2("field2", *DataType::STRING, false);
+
+StructDataType getStructDataType() {
+    StructDataType struct_type("struct");
+    struct_type.addField(field1);
+    struct_type.addField(field2);
+    return struct_type;
+}
+
+StructFieldValue getStructFieldValue(const StructDataType& structType) {
+    StructFieldValue value(structType);
+    value.setValue(field1, IntFieldValue(42));
+    value.setValue(field2, StringFieldValue("foooooooooooooooooooooobaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"));
+    return value;
+}
+
+void checkStructSerialization(const StructFieldValue &value,
+                              CompressionConfig::Type comp_type) {
+    nbostream stream;
+    serializeAndDeserialize(value, stream);
+    uint32_t data_size;
+    uint8_t compression_type;
+    uint16_t uncompressed_size;
+    uint8_t field_count;
+    int32_t element1_id, element2_id;
+    uint16_t element1_size, element2_size;
+    stream >> data_size >> compression_type;
+    if (CompressionConfig::isCompressed(comp_type)) {
+        stream >> uncompressed_size;
+        EXPECT_EQUAL(24u, data_size);
+        EXPECT_EQUAL(64u, uncompressed_size);
+    } else {
+        EXPECT_EQUAL(64u, data_size);
+    }
+    stream >> field_count
+           >> element1_id >> element1_size
+           >> element2_id >> element2_size;
+    EXPECT_EQUAL(comp_type, compression_type);
+    EXPECT_EQUAL(2u, field_count);
+    EXPECT_EQUAL(field1.getId(8), element1_id & 0x7fffffff);
+    EXPECT_EQUAL(4u, element1_size);
+    EXPECT_EQUAL(field2.getId(8), element2_id & 0x7fffffff);
+    EXPECT_EQUAL(60u, element2_size);
+}
+
+TEST("requireThatUncompressedStructFieldValueCanBeSerialized") {
+    StructDataType structType(getStructDataType());
+    StructFieldValue value = getStructFieldValue(structType);
+    checkStructSerialization(value, CompressionConfig::NONE);
+}
+
+TEST("requireThatCompressedStructFieldValueCanBeSerialized") {
+    StructDataType structType(getStructDataType());
+    StructFieldValue value = getStructFieldValue(structType);
+    const_cast<StructDataType &>(value.getStructType()).setCompressionConfig(CompressionConfig(CompressionConfig::LZ4, 0, 95));
+    checkStructSerialization(value, CompressionConfig::LZ4);
+}
+
+TEST("requireThatReserializationPreservesCompressionIfUnmodified") {
+    StructDataType structType(getStructDataType());
+    StructFieldValue value = getStructFieldValue(structType);
+    const_cast<StructDataType &>(value.getStructType()).
+        setCompressionConfig(CompressionConfig(CompressionConfig::LZ4, 0, 95));
+
+    checkStructSerialization(value, CompressionConfig::LZ4);
+
+    nbostream os;
+    VespaDocumentSerializer serializer(os);
+    serializer.write(value);
+
+    StructDataType struct_type(getStructDataType());
+    StructFieldValue value2(struct_type);
+    VespaDocumentDeserializer deserializer(repo, os, 8);
+    deserializer.read(value2);
+    checkStructSerialization(value, CompressionConfig::LZ4);
+    // No lazy serialization of structs anymore, only documents
+    checkStructSerialization(value2, CompressionConfig::NONE);
+    EXPECT_EQUAL(value, value2);
+}
+
+template <typename T, int N> int arraysize(const T (&)[N]) { return N; }
+
+TEST("requireThatDocumentCanBeSerialized") {
+    const uint32_t serialization_version = 8;
+    const DocumentType &type = repo.getDocumentType();
+
+    DocumentId doc_id("doc::testdoc");
+    Document value(type, doc_id);
+
+    value.setValue(type.getField("header field"), IntFieldValue(42));
+    value.setValue(type.getField("body field"), StringFieldValue("foobar"));
+
+    nbostream stream;
+    serializeAndDeserialize(value, stream);
+
+    uint16_t read_version;
+    uint32_t size;
+    stream >> read_version >> size;
+    EXPECT_EQUAL(serialization_version, read_version);
+    EXPECT_EQUAL(65u, size);
+    EXPECT_EQUAL(doc_id.getScheme().toString(), stream.peek());
+    stream.adjustReadPos(doc_id.getScheme().toString().size() + 1);
+    uint8_t content_code;
+    stream >> content_code;
+    EXPECT_EQUAL(0x07, content_code);
+    EXPECT_EQUAL(type.getName(), stream.peek());
+    stream.adjustReadPos(type.getName().size() + 1);
+    stream >> read_version;
+    EXPECT_EQUAL(0, read_version);
+}
+
+TEST("requireThatOldVersionDocumentCanBeDeserialized") {
+    uint16_t old_version = 6;
+    uint16_t data_size = 432;
+    string doc_id = "doc::testdoc";
+    uint8_t content_code = 0x01;
+    uint32_t crc = 42;
+
+    nbostream stream;
+    stream << old_version << data_size;
+    stream.write(doc_id.c_str(), doc_id.size() + 1);
+    stream << content_code;
+    stream.write(doc_name.c_str(), doc_name.size() + 1);
+    stream << static_cast<uint16_t>(0) << crc;  // version (unused)
+
+    Document value;
+    VespaDocumentDeserializer deserializer(repo, stream, old_version);
+    deserializer.read(value);
+
+    EXPECT_EQUAL(doc_id, value.getId().getScheme().toString());
+    EXPECT_EQUAL(doc_name, value.getType().getName());
+    EXPECT_TRUE(value.getFields().empty());
+}
+
+TEST("requireThatUnmodifiedDocumentRetainsUnknownFieldOnSerialization") {
+
+    DocumenttypesConfigBuilderHelper builder1, builder2;
+    builder1.document(doc_type_id, doc_name,
+                      Struct("my document.header")
+                      .addField("field2", DataType::T_STRING),
+                      Struct("my document.body"));
+    builder2.document(doc_type_id, doc_name,
+                      Struct("my document.header")
+                      .addField("field1", DataType::T_INT)
+                      .addField("field2", DataType::T_STRING),
+                      Struct("my document.body"));
+		
+    DocumentTypeRepo repo1Field(builder1.config());
+    DocumentTypeRepo repo2Fields(builder2.config());
+
+    uint32_t serial_version = 8;
+
+    DocumentId doc_id("doc::testdoc");
+    Document value(*repo2Fields.getDocumentType(doc_type_id), doc_id);
+
+    value.setValue("field1", IntFieldValue(42));
+    value.setValue("field2", StringFieldValue("megafoo"));
+
+    nbostream stream;
+    VespaDocumentSerializer serializer(stream);
+    serializer.write(value);
+
+    Document read_value;
+    // Deserialize+serialize with type where field1 is not known.
+    VespaDocumentDeserializer deserializer1(repo1Field, stream, serial_version);
+    deserializer1.read(read_value);
+    EXPECT_EQUAL(0u, stream.size());
+
+    EXPECT_EQUAL(1u, read_value.getSetFieldCount());
+
+    stream.clear();
+    serializer.write(read_value);
+
+    Document read_value_2;
+    // Field should not have vanished.
+    VespaDocumentDeserializer deserializer2(repo2Fields, stream, serial_version);
+    deserializer2.read(read_value_2);
+    EXPECT_EQUAL(value, read_value_2);
+}
+
+TEST("requireThatAnnotationReferenceFieldValueCanBeSerialized") {
+    AnnotationType annotation_type(0, "atype");
+    AnnotationReferenceDataType type(annotation_type, 0);
+    int annotation_id = 420;
+    AnnotationReferenceFieldValue value(type, annotation_id);
+    nbostream stream;
+    serializeAndDeserialize(value, stream);
+
+    int read_id = readSize1_2_4<uint16_t>(stream);
+    EXPECT_EQUAL(annotation_id, read_id);
+}
+
+TEST("requireThatDocumentWithDocumentCanBeSerialized") {
+    const DocumentTypeRepo &my_repo = repo.getDocumentTypeRepo();
+    const DocumentType *inner_type = my_repo.getDocumentType(inner_type_id);
+    ASSERT_TRUE(inner_type);
+    const AnnotationType *a_type =
+        my_repo.getAnnotationType(*inner_type, a_id);
+    StringFieldValue str("foo");
+    SpanTree::UP tree(new SpanTree("name", Span::UP(new Span(0, 3))));
+    tree->annotate(Annotation::UP(new Annotation(*a_type)));
+
+
+    setSpanTree(str, *tree);
+    const Field str_field("str", *DataType::STRING, false);
+
+    Document inner(*inner_type, DocumentId("doc::in"));
+    inner.setValue(str_field, str);
+    const DocumentType *type = my_repo.getDocumentType(outer_type_id);
+    ASSERT_TRUE(type);
+    DocumentId doc_id("doc::testdoc");
+    Document value(*type, doc_id);
+    const Field doc_field(inner_name, *inner_type, false);
+    value.setValue(doc_field, inner);
+
+    nbostream stream;
+    serializeAndDeserialize(value, stream);
+}
+
+TEST("requireThatReadDocumentTypeThrowsIfUnknownType") {
+    string my_type("my_unknown_type");
+    nbostream stream;
+    stream.write(my_type.c_str(), my_type.size() + 1);
+    stream << static_cast<uint16_t>(0);  // version (unused)
+
+    DocumentType value;
+    VespaDocumentDeserializer deserializer(repo, stream, 8);
+    EXPECT_EXCEPTION(deserializer.read(value), DocumentTypeNotFoundException,
+                "Document type " + my_type + " not found");
+}
+
+template <typename FieldValueT>
+void serializeToFile(FieldValueT &value, const string &file_name,
+                     const DocumentType *type, const string &field_name) {
+    DocumentId doc_id("id:test:" + type->getName() + "::foo");
+    Document doc(*type, doc_id);
+    doc.setValue(type->getField(field_name), value);
+
+    nbostream stream;
+    VespaDocumentSerializer serializer(stream);
+    serializer.write(doc);
+
+    File file(file_name);
+    file.open(File::CREATE);
+    file.write(stream.peek(), stream.size(), 0);
+}
+
+void serializeToFile(PredicateFieldValue &value, const string &file_name) {
+    const DocumentType *type = doc_repo.getDocumentType(predicate_doc_type_id);
+    serializeToFile(value, file_name, type, predicate_field_name);
+}
+
+template <typename FieldValueT>
+void deserializeAndCheck(const string &file_name, FieldValueT &value,
+                         const FixedTypeRepo &myrepo,
+                         const string &field_name) {
+    File file(file_name);
+    file.open(File::READONLY);
+    vector<char> content(file.stat()._size);
+    size_t r = file.read(&content[0], content.size(), 0);
+    ASSERT_EQUAL(content.size(), r);
+
+    nbostream stream(&content[0], content.size(), true);
+    Document doc;
+    VespaDocumentDeserializer deserializer(myrepo, stream, 8);
+    deserializer.read(doc);
+
+    ASSERT_EQUAL(0, value.compare(*doc.getValue(field_name)));
+}
+
+void deserializeAndCheck(const string &file_name, PredicateFieldValue &value) {
+    deserializeAndCheck(file_name, value, repo, predicate_field_name);
+}
+
+void checkDeserialization(const string &name, std::unique_ptr<Slime> slime) {
+    const string data_dir = "../../test/resources/predicates/";
+
+    PredicateFieldValue value(std::move(slime));
+    serializeToFile(value, data_dir + name + "__cpp");
+
+    deserializeAndCheck(data_dir + name + "__cpp", value);
+    deserializeAndCheck(data_dir + name + "__java", value);
+}
+
+TEST("Require that predicate deserialization matches Java") {
+    PredicateSlimeBuilder builder;
+
+    builder.feature("foo").range(6, 9);
+    checkDeserialization("foo_in_6_9", builder.build());
+
+    builder.feature("foo").greaterEqual(6);
+    checkDeserialization("foo_in_6_x", builder.build());
+
+    builder.feature("foo").lessEqual(9);
+    checkDeserialization("foo_in_x_9", builder.build());
+
+    builder.feature("foo").value("bar");
+    checkDeserialization("foo_in_bar", builder.build());
+
+    builder.feature("foo").value("bar").value("baz");
+    checkDeserialization("foo_in_bar_baz", builder.build());
+
+    builder.neg().feature("foo").value("bar");
+    checkDeserialization("not_foo_in_bar", builder.build());
+
+    std::unique_ptr<Slime> slime(new Slime);
+    Cursor *cursor = &slime->setObject();
+    cursor->setString(Predicate::KEY, "foo");
+    cursor->setLong(Predicate::NODE_TYPE, Predicate::TYPE_FEATURE_SET);
+    checkDeserialization("foo_in_x", std::move(slime));
+
+    slime.reset(new Slime);
+    cursor = &slime->setObject();
+    cursor->setString(Predicate::KEY, "foo");
+    cursor->setLong(Predicate::NODE_TYPE, Predicate::TYPE_FEATURE_RANGE);
+    checkDeserialization("foo_in_x_x", std::move(slime));
+
+    std::unique_ptr<Slime> slime1 = builder.feature("foo").value("bar").build();
+    std::unique_ptr<Slime> slime2 = builder.feature("baz").value("cox").build();
+    builder.and_node(std::move(slime1), std::move(slime2));
+    checkDeserialization("foo_in_bar_and_baz_in_cox", builder.build());
+
+    slime1 = builder.feature("foo").value("bar").build();
+    slime2 = builder.feature("baz").value("cox").build();
+    builder.or_node(std::move(slime1), std::move(slime2));
+    checkDeserialization("foo_in_bar_or_baz_in_cox", builder.build());
+
+    builder.true_predicate();
+    checkDeserialization("true", builder.build());
+
+    builder.false_predicate();
+    checkDeserialization("false", builder.build());
+}
+
+namespace
+{
+
+Tensor::UP
+createTensor(const TensorCells &cells, const TensorDimensions &dimensions) {
+    vespalib::tensor::DefaultTensor::builder builder;
+    return vespalib::tensor::TensorFactory::create(cells, dimensions, builder);
+}
+
+
+}
+
+TEST("Require that tensors can be serialized")
+{
+    TensorFieldValue noTensorValue;
+    TensorFieldValue emptyTensorValue;
+    TensorFieldValue twoCellsTwoDimsValue;
+    nbostream stream;
+    serializeAndDeserialize(noTensorValue, stream);
+    stream.clear();
+    emptyTensorValue = createTensor({}, {});
+    serializeAndDeserialize(emptyTensorValue, stream);
+    stream.clear();
+    twoCellsTwoDimsValue = createTensor({ {{{"y", "3"}}, 3},
+                                             {{{"x", "4"}, {"y", "5"}}, 7} },
+                                        {"x", "y"});
+    serializeAndDeserialize(twoCellsTwoDimsValue, stream);
+    EXPECT_NOT_EQUAL(noTensorValue, emptyTensorValue);
+    EXPECT_NOT_EQUAL(noTensorValue, twoCellsTwoDimsValue);
+    EXPECT_NOT_EQUAL(emptyTensorValue, twoCellsTwoDimsValue);
+}
+
+
+const int tensor_doc_type_id = 321;
+const string tensor_field_name = "my_tensor";
+
+DocumenttypesConfig getTensorDocTypesConfig() {
+    DocumenttypesConfigBuilderHelper builder;
+    builder.document(tensor_doc_type_id, "my_type",
+                     Struct("my_type.header"),
+                     Struct("my_type.body")
+                     .addField(tensor_field_name, DataType::T_TENSOR));
+    return builder.config();
+}
+
+const DocumentTypeRepo tensor_doc_repo(getTensorDocTypesConfig());
+const FixedTypeRepo tensor_repo(tensor_doc_repo,
+                                *tensor_doc_repo.getDocumentType(doc_type_id));
+
+void serializeToFile(TensorFieldValue &value, const string &file_name) {
+    const DocumentType *type =
+        tensor_doc_repo.getDocumentType(tensor_doc_type_id);
+    serializeToFile(value, file_name, type, tensor_field_name);
+}
+
+void deserializeAndCheck(const string &file_name, TensorFieldValue &value) {
+    deserializeAndCheck(file_name, value, tensor_repo, tensor_field_name);
+}
+
+void checkDeserialization(const string &name, std::unique_ptr<Tensor> tensor) {
+    const string data_dir = "../../test/resources/tensor/";
+
+    TensorFieldValue value;
+    if (tensor) {
+        value = tensor->clone();
+    }
+    serializeToFile(value, data_dir + name + "__cpp");
+    deserializeAndCheck(data_dir + name + "__cpp", value);
+    deserializeAndCheck(data_dir + name + "__java", value);
+}
+
+TEST("Require that tensor deserialization matches Java") {
+    checkDeserialization("non_existing_tensor", std::unique_ptr<Tensor>());
+    checkDeserialization("empty_tensor", createTensor({}, {}));
+    checkDeserialization("multi_cell_tensor",
+                         createTensor({ {{{"dimX", "a"}, {"dimY", "bb"}}, 2.0 },
+                                           {{{"dimX", "ccc"},
+                                                   {"dimY", "dddd"}}, 3.0},
+                                           {{{"dimX", "e"}}, 5.0} },
+                                      { "dimX", "dimY" }));
+}
+
+}  // namespace
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/document/src/tests/setfieldvaluetest.h b/document/src/tests/setfieldvaluetest.h
new file mode 100644
index 00000000000..a27aeedf11d
--- /dev/null
+++ b/document/src/tests/setfieldvaluetest.h
@@ -0,0 +1,21 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/* $Id$*/
+
+#pragma once
+
+#include <cppunit/extensions/HelperMacros.h>
+
+class SetFieldValue_Test : public CppUnit::TestFixture {
+    CPPUNIT_TEST_SUITE( SetFieldValue_Test);
+    CPPUNIT_TEST(testSet);
+    CPPUNIT_TEST_SUITE_END();
+
+public:
+    void setUp();
+
+    void tearDown();
+protected:
+    void testSet();
+};
+
+
diff --git a/document/src/tests/stringtokenizertest.cpp b/document/src/tests/stringtokenizertest.cpp
new file mode 100644
index 00000000000..5ac41992621
--- /dev/null
+++ b/document/src/tests/stringtokenizertest.cpp
@@ -0,0 +1,109 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <cppunit/extensions/HelperMacros.h>
+#include <iostream>
+#include <set>
+#include <sstream>
+#include <vespa/vespalib/text/stringtokenizer.h>
+
+using vespalib::StringTokenizer;
+using std::string;
+
+class StringTokenizerTest : public CppUnit::TestFixture {
+    CPPUNIT_TEST_SUITE(StringTokenizerTest);
+    CPPUNIT_TEST(testSimpleUsage);
+    CPPUNIT_TEST_SUITE_END();
+
+protected:
+    void testSimpleUsage();
+};
+
+CPPUNIT_TEST_SUITE_REGISTRATION(StringTokenizerTest);
+
+void StringTokenizerTest::testSimpleUsage()
+{
+    {
+        string s("This,is ,a,,list ,\tof,,sepa rated\n, \rtokens,");
+        StringTokenizer tokenizer(s);
+        StringTokenizer::TokenList result;
+        result.push_back("This");
+        result.push_back("is");
+        result.push_back("a");
+        result.push_back("");
+        result.push_back("list");
+        result.push_back("of");
+        result.push_back("");
+        result.push_back("sepa rated");
+        result.push_back("tokens");
+        result.push_back("");
+
+        CPPUNIT_ASSERT_EQUAL(result.size(),
+                             static_cast<size_t>(tokenizer.size()));
+        for (unsigned int i=0; i<result.size(); i++) {
+            CPPUNIT_ASSERT_EQUAL(result[i], tokenizer[i]);
+        }
+        std::set<string> sorted(tokenizer.begin(), tokenizer.end());
+        CPPUNIT_ASSERT_EQUAL(static_cast<size_t>(8u), sorted.size());
+
+        tokenizer.removeEmptyTokens();
+        CPPUNIT_ASSERT_EQUAL(7u, tokenizer.size());
+    }
+    {
+        string s("\tAnother list with some \ntokens, and stuff.");
+        StringTokenizer tokenizer(s, " \t\n", ",.");
+        StringTokenizer::TokenList result;
+        result.push_back("");
+        result.push_back("Another");
+        result.push_back("list");
+        result.push_back("with");
+        result.push_back("some");
+        result.push_back("");
+        result.push_back("tokens");
+        result.push_back("and");
+        result.push_back("stuff");
+
+        CPPUNIT_ASSERT_EQUAL(result.size(),
+                             static_cast<size_t>(tokenizer.size()));
+        for (unsigned int i=0; i<result.size(); i++) {
+            CPPUNIT_ASSERT_EQUAL(result[i], tokenizer[i]);
+        }
+        std::set<string> sorted(tokenizer.begin(), tokenizer.end());
+        CPPUNIT_ASSERT_EQUAL(static_cast<size_t>(8u), sorted.size());
+
+        tokenizer.removeEmptyTokens();
+        CPPUNIT_ASSERT_EQUAL(7u, tokenizer.size());
+    }
+    {
+        string s(" ");
+        StringTokenizer tokenizer(s);
+        CPPUNIT_ASSERT_EQUAL(0u, tokenizer.size());
+    }
+
+    {
+        string s("");
+        StringTokenizer tokenizer(s);
+        CPPUNIT_ASSERT_EQUAL(0u, tokenizer.size());
+    }
+    {
+        // Test that there aren't any problems with using signed chars.
+        string s("Here\x01\xff be\xff\xfe dragons\xff");
+        StringTokenizer tokenizer(s, "\xff", "\x01 \xfe");
+        StringTokenizer::TokenList result;
+        result.push_back("Here");
+        result.push_back("be");
+        result.push_back("dragons");
+        result.push_back("");
+
+        CPPUNIT_ASSERT_EQUAL(result.size(),
+                             static_cast<size_t>(tokenizer.size()));
+        for (unsigned int i=0; i<result.size(); i++) {
+            CPPUNIT_ASSERT_EQUAL(result[i], tokenizer[i]);
+        }
+        std::set<string> sorted(tokenizer.begin(), tokenizer.end());
+        CPPUNIT_ASSERT_EQUAL(static_cast<size_t>(4u), sorted.size());
+
+        tokenizer.removeEmptyTokens();
+        CPPUNIT_ASSERT_EQUAL(3u, tokenizer.size());
+    }
+}
diff --git a/document/src/tests/struct_anno/.gitignore b/document/src/tests/struct_anno/.gitignore
new file mode 100644
index 00000000000..7a5ecfc76aa
--- /dev/null
+++ b/document/src/tests/struct_anno/.gitignore
@@ -0,0 +1,2 @@
+/struct_anno_test
+document_struct_anno_test_app
diff --git a/document/src/tests/struct_anno/CMakeLists.txt b/document/src/tests/struct_anno/CMakeLists.txt
new file mode 100644
index 00000000000..64678282842
--- /dev/null
+++ b/document/src/tests/struct_anno/CMakeLists.txt
@@ -0,0 +1,10 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(document_struct_anno_test_app
+    SOURCES
+    struct_anno_test.cpp
+    DEPENDS
+    document
+    AFTER
+    document_documentconfig
+)
+vespa_add_test(NAME document_struct_anno_test_app COMMAND document_struct_anno_test_app)
diff --git a/document/src/tests/struct_anno/document.dat b/document/src/tests/struct_anno/document.dat
new file mode 100644
index 00000000000..243b5f05a92
--- /dev/null
+++ b/document/src/tests/struct_anno/document.dat
diff --git a/document/src/tests/struct_anno/documentmanager.cfg b/document/src/tests/struct_anno/documentmanager.cfg
new file mode 100644
index 00000000000..94020ca7b69
--- /dev/null
+++ b/document/src/tests/struct_anno/documentmanager.cfg
@@ -0,0 +1,160 @@
+enablecompression false
+annotationtype[0]
+datatype[8]
+datatype[0].id -1049517126
+datatype[0].annotationreftype[0]
+datatype[0].arraytype[0]
+datatype[0].documenttype[1]
+datatype[0].documenttype[0].bodystruct 1008689562
+datatype[0].documenttype[0].headerstruct 166307397
+datatype[0].documenttype[0].name "simple_def"
+datatype[0].documenttype[0].version 0
+datatype[0].documenttype[0].inherits[1]
+datatype[0].documenttype[0].inherits[0].name "document"
+datatype[0].documenttype[0].inherits[0].version 0
+datatype[0].maptype[0]
+datatype[0].structtype[0]
+datatype[0].weightedsettype[0]
+datatype[1].id -1267268530
+datatype[1].annotationreftype[0]
+datatype[1].arraytype[0]
+datatype[1].documenttype[0]
+datatype[1].maptype[0]
+datatype[1].structtype[1]
+datatype[1].structtype[0].compresslevel 0
+datatype[1].structtype[0].compressminsize 800
+datatype[1].structtype[0].compressthreshold 95
+datatype[1].structtype[0].compresstype NONE
+datatype[1].structtype[0].name "simple_def_search.header"
+datatype[1].structtype[0].version 0
+datatype[1].structtype[0].field[3]
+datatype[1].structtype[0].field[0].datatype 111553393
+datatype[1].structtype[0].field[0].name "my_url"
+datatype[1].structtype[0].field[0].id[0]
+datatype[1].structtype[0].field[1].datatype 2
+datatype[1].structtype[0].field[1].name "rankfeatures"
+datatype[1].structtype[0].field[1].id[0]
+datatype[1].structtype[0].field[2].datatype 2
+datatype[1].structtype[0].field[2].name "summaryfeatures"
+datatype[1].structtype[0].field[2].id[0]
+datatype[1].structtype[0].inherits[0]
+datatype[1].weightedsettype[0]
+datatype[2].id -495710767
+datatype[2].annotationreftype[0]
+datatype[2].arraytype[0]
+datatype[2].documenttype[1]
+datatype[2].documenttype[0].bodystruct 1968090595
+datatype[2].documenttype[0].headerstruct -1267268530
+datatype[2].documenttype[0].name "simple_def_search"
+datatype[2].documenttype[0].version 0
+datatype[2].documenttype[0].inherits[1]
+datatype[2].documenttype[0].inherits[0].name "document"
+datatype[2].documenttype[0].inherits[0].version 0
+datatype[2].maptype[0]
+datatype[2].structtype[0]
+datatype[2].weightedsettype[0]
+datatype[3].id 1008689562
+datatype[3].annotationreftype[0]
+datatype[3].arraytype[0]
+datatype[3].documenttype[0]
+datatype[3].maptype[0]
+datatype[3].structtype[1]
+datatype[3].structtype[0].compresslevel 0
+datatype[3].structtype[0].compressminsize 800
+datatype[3].structtype[0].compressthreshold 95
+datatype[3].structtype[0].compresstype NONE
+datatype[3].structtype[0].name "simple_def.body"
+datatype[3].structtype[0].version 0
+datatype[3].structtype[0].field[0]
+datatype[3].structtype[0].inherits[0]
+datatype[3].weightedsettype[0]
+datatype[4].id 111553393
+datatype[4].annotationreftype[0]
+datatype[4].arraytype[0]
+datatype[4].documenttype[0]
+datatype[4].maptype[0]
+datatype[4].structtype[1]
+datatype[4].structtype[0].compresslevel 0
+datatype[4].structtype[0].compressminsize 800
+datatype[4].structtype[0].compressthreshold 95
+datatype[4].structtype[0].compresstype NONE
+datatype[4].structtype[0].name "url"
+datatype[4].structtype[0].version 0
+datatype[4].structtype[0].field[7]
+datatype[4].structtype[0].field[0].datatype 2
+datatype[4].structtype[0].field[0].name "all"
+datatype[4].structtype[0].field[0].id[0]
+datatype[4].structtype[0].field[1].datatype 2
+datatype[4].structtype[0].field[1].name "fragment"
+datatype[4].structtype[0].field[1].id[0]
+datatype[4].structtype[0].field[2].datatype 2
+datatype[4].structtype[0].field[2].name "host"
+datatype[4].structtype[0].field[2].id[0]
+datatype[4].structtype[0].field[3].datatype 2
+datatype[4].structtype[0].field[3].name "path"
+datatype[4].structtype[0].field[3].id[0]
+datatype[4].structtype[0].field[4].datatype 2
+datatype[4].structtype[0].field[4].name "port"
+datatype[4].structtype[0].field[4].id[0]
+datatype[4].structtype[0].field[5].datatype 2
+datatype[4].structtype[0].field[5].name "query"
+datatype[4].structtype[0].field[5].id[0]
+datatype[4].structtype[0].field[6].datatype 2
+datatype[4].structtype[0].field[6].name "scheme"
+datatype[4].structtype[0].field[6].id[0]
+datatype[4].structtype[0].inherits[0]
+datatype[4].weightedsettype[0]
+datatype[5].id 1381038251
+datatype[5].annotationreftype[0]
+datatype[5].arraytype[0]
+datatype[5].documenttype[0]
+datatype[5].maptype[0]
+datatype[5].structtype[1]
+datatype[5].structtype[0].compresslevel 0
+datatype[5].structtype[0].compressminsize 800
+datatype[5].structtype[0].compressthreshold 95
+datatype[5].structtype[0].compresstype NONE
+datatype[5].structtype[0].name "position"
+datatype[5].structtype[0].version 0
+datatype[5].structtype[0].field[2]
+datatype[5].structtype[0].field[0].datatype 0
+datatype[5].structtype[0].field[0].name "x"
+datatype[5].structtype[0].field[0].id[0]
+datatype[5].structtype[0].field[1].datatype 0
+datatype[5].structtype[0].field[1].name "y"
+datatype[5].structtype[0].field[1].id[0]
+datatype[5].structtype[0].inherits[0]
+datatype[5].weightedsettype[0]
+datatype[6].id 166307397
+datatype[6].annotationreftype[0]
+datatype[6].arraytype[0]
+datatype[6].documenttype[0]
+datatype[6].maptype[0]
+datatype[6].structtype[1]
+datatype[6].structtype[0].compresslevel 0
+datatype[6].structtype[0].compressminsize 800
+datatype[6].structtype[0].compressthreshold 95
+datatype[6].structtype[0].compresstype NONE
+datatype[6].structtype[0].name "simple_def.header"
+datatype[6].structtype[0].version 0
+datatype[6].structtype[0].field[1]
+datatype[6].structtype[0].field[0].datatype 111553393
+datatype[6].structtype[0].field[0].name "my_url"
+datatype[6].structtype[0].field[0].id[0]
+datatype[6].structtype[0].inherits[0]
+datatype[6].weightedsettype[0]
+datatype[7].id 1968090595
+datatype[7].annotationreftype[0]
+datatype[7].arraytype[0]
+datatype[7].documenttype[0]
+datatype[7].maptype[0]
+datatype[7].structtype[1]
+datatype[7].structtype[0].compresslevel 0
+datatype[7].structtype[0].compressminsize 800
+datatype[7].structtype[0].compressthreshold 95
+datatype[7].structtype[0].compresstype NONE
+datatype[7].structtype[0].name "simple_def_search.body"
+datatype[7].structtype[0].version 0
+datatype[7].structtype[0].field[0]
+datatype[7].structtype[0].inherits[0]
+datatype[7].weightedsettype[0]
diff --git a/document/src/tests/struct_anno/documenttypes.cfg b/document/src/tests/struct_anno/documenttypes.cfg
new file mode 100644
index 00000000000..38ef6d54125
--- /dev/null
+++ b/document/src/tests/struct_anno/documenttypes.cfg
@@ -0,0 +1,98 @@
+enablecompression false
+documenttype[2]
+documenttype[0].id -495710767
+documenttype[0].name "simple_def_search"
+documenttype[0].version 0
+documenttype[0].headerstruct -1267268530
+documenttype[0].bodystruct 1968090595
+documenttype[0].inherits[0]
+documenttype[0].datatype[2]
+documenttype[0].datatype[0].id -1267268530
+documenttype[0].datatype[0].type STRUCT
+documenttype[0].datatype[0].array.element.id 0
+documenttype[0].datatype[0].map.key.id 0
+documenttype[0].datatype[0].map.value.id 0
+documenttype[0].datatype[0].wset.key.id 0
+documenttype[0].datatype[0].wset.createifnonexistent false
+documenttype[0].datatype[0].wset.removeifzero false
+documenttype[0].datatype[0].annotationref.annotation.id 0
+documenttype[0].datatype[0].sstruct.name "simple_def_search.header"
+documenttype[0].datatype[0].sstruct.version 0
+documenttype[0].datatype[0].sstruct.compression.type NONE
+documenttype[0].datatype[0].sstruct.compression.level 0
+documenttype[0].datatype[0].sstruct.compression.threshold 90
+documenttype[0].datatype[0].sstruct.compression.minsize 0
+documenttype[0].datatype[0].sstruct.field[3]
+documenttype[0].datatype[0].sstruct.field[0].name "my_url"
+documenttype[0].datatype[0].sstruct.field[0].id 1127377169
+documenttype[0].datatype[0].sstruct.field[0].id_v6 707904318
+documenttype[0].datatype[0].sstruct.field[0].datatype 111553393
+documenttype[0].datatype[0].sstruct.field[1].name "rankfeatures"
+documenttype[0].datatype[0].sstruct.field[1].id 1883197392
+documenttype[0].datatype[0].sstruct.field[1].id_v6 699950698
+documenttype[0].datatype[0].sstruct.field[1].datatype 2
+documenttype[0].datatype[0].sstruct.field[2].name "summaryfeatures"
+documenttype[0].datatype[0].sstruct.field[2].id 1840337115
+documenttype[0].datatype[0].sstruct.field[2].id_v6 1981648971
+documenttype[0].datatype[0].sstruct.field[2].datatype 2
+documenttype[0].datatype[1].id 1968090595
+documenttype[0].datatype[1].type STRUCT
+documenttype[0].datatype[1].array.element.id 0
+documenttype[0].datatype[1].map.key.id 0
+documenttype[0].datatype[1].map.value.id 0
+documenttype[0].datatype[1].wset.key.id 0
+documenttype[0].datatype[1].wset.createifnonexistent false
+documenttype[0].datatype[1].wset.removeifzero false
+documenttype[0].datatype[1].annotationref.annotation.id 0
+documenttype[0].datatype[1].sstruct.name "simple_def_search.body"
+documenttype[0].datatype[1].sstruct.version 0
+documenttype[0].datatype[1].sstruct.compression.type NONE
+documenttype[0].datatype[1].sstruct.compression.level 0
+documenttype[0].datatype[1].sstruct.compression.threshold 90
+documenttype[0].datatype[1].sstruct.compression.minsize 0
+documenttype[0].datatype[1].sstruct.field[0]
+documenttype[0].annotationtype[0]
+documenttype[1].id -1049517126
+documenttype[1].name "simple_def"
+documenttype[1].version 0
+documenttype[1].headerstruct 166307397
+documenttype[1].bodystruct 1008689562
+documenttype[1].inherits[0]
+documenttype[1].datatype[2]
+documenttype[1].datatype[0].id 166307397
+documenttype[1].datatype[0].type STRUCT
+documenttype[1].datatype[0].array.element.id 0
+documenttype[1].datatype[0].map.key.id 0
+documenttype[1].datatype[0].map.value.id 0
+documenttype[1].datatype[0].wset.key.id 0
+documenttype[1].datatype[0].wset.createifnonexistent false
+documenttype[1].datatype[0].wset.removeifzero false
+documenttype[1].datatype[0].annotationref.annotation.id 0
+documenttype[1].datatype[0].sstruct.name "simple_def.header"
+documenttype[1].datatype[0].sstruct.version 0
+documenttype[1].datatype[0].sstruct.compression.type NONE
+documenttype[1].datatype[0].sstruct.compression.level 0
+documenttype[1].datatype[0].sstruct.compression.threshold 90
+documenttype[1].datatype[0].sstruct.compression.minsize 0
+documenttype[1].datatype[0].sstruct.field[1]
+documenttype[1].datatype[0].sstruct.field[0].name "my_url"
+documenttype[1].datatype[0].sstruct.field[0].id 1127377169
+documenttype[1].datatype[0].sstruct.field[0].id_v6 707904318
+documenttype[1].datatype[0].sstruct.field[0].datatype 111553393
+documenttype[1].datatype[1].id 1008689562
+documenttype[1].datatype[1].type STRUCT
+documenttype[1].datatype[1].array.element.id 0
+documenttype[1].datatype[1].map.key.id 0
+documenttype[1].datatype[1].map.value.id 0
+documenttype[1].datatype[1].wset.key.id 0
+documenttype[1].datatype[1].wset.createifnonexistent false
+documenttype[1].datatype[1].wset.removeifzero false
+documenttype[1].datatype[1].annotationref.annotation.id 0
+documenttype[1].datatype[1].sstruct.name "simple_def.body"
+documenttype[1].datatype[1].sstruct.version 0
+documenttype[1].datatype[1].sstruct.compression.type NONE
+documenttype[1].datatype[1].sstruct.compression.level 0
+documenttype[1].datatype[1].sstruct.compression.threshold 90
+documenttype[1].datatype[1].sstruct.compression.minsize 0
+documenttype[1].datatype[1].sstruct.field[0]
+documenttype[1].annotationtype[0]
diff --git a/document/src/tests/struct_anno/struct_anno_test.cpp b/document/src/tests/struct_anno/struct_anno_test.cpp
new file mode 100644
index 00000000000..85b3b8f2d0b
--- /dev/null
+++ b/document/src/tests/struct_anno/struct_anno_test.cpp
@@ -0,0 +1,87 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("struct_anno_test");
+
+#include <stdlib.h>
+#include <vespa/document/annotation/alternatespanlist.h>
+#include <vespa/document/annotation/annotation.h>
+#include <vespa/document/annotation/span.h>
+#include <vespa/document/annotation/spanlist.h>
+#include <vespa/document/annotation/spantree.h>
+#include <vespa/document/config/config-documenttypes.h>
+#include <vespa/document/document.h>
+#include <vespa/document/fieldvalue/stringfieldvalue.h>
+#include <vespa/document/repo/fixedtyperepo.h>
+#include <vespa/document/serialization/annotationdeserializer.h>
+#include <vespa/document/serialization/annotationserializer.h>
+#include <vespa/document/serialization/vespadocumentdeserializer.h>
+#include <vespa/document/serialization/vespadocumentserializer.h>
+#include <vespa/vespalib/objects/nbostream.h>
+#include <vespa/vespalib/testkit/testapp.h>
+#include <sstream>
+#include <vector>
+
+using std::ostringstream;
+using std::string;
+using std::vector;
+using vespalib::nbostream;
+using namespace document;
+
+namespace {
+
+class Test : public vespalib::TestApp {
+    void requireThatStructFieldsCanContainAnnotations();
+
+public:
+    int Main();
+};
+
+int Test::Main() {
+    if (getenv("TEST_SUBSET") != 0) { return 0; }
+    TEST_INIT("struct_anno_test");
+    TEST_DO(requireThatStructFieldsCanContainAnnotations());
+    TEST_DONE();
+}
+
+template <typename T, int N> int arraysize(const T (&)[N]) { return N; }
+
+void Test::requireThatStructFieldsCanContainAnnotations() {
+    DocumentTypeRepo repo(readDocumenttypesConfig("documenttypes.cfg"));
+
+    FastOS_File file("document.dat");
+    ASSERT_TRUE(file.OpenReadOnlyExisting());
+    char buffer[1024];
+    ssize_t size = file.Read(buffer, arraysize(buffer));
+    ASSERT_TRUE(size != -1);
+
+    nbostream stream(buffer, size);
+    VespaDocumentDeserializer deserializer(repo, stream, 8);
+    Document doc;
+    deserializer.read(doc);
+
+    FieldValue::UP urlRef = doc.getValue("my_url");
+    ASSERT_TRUE(urlRef.get() != NULL);
+    const StructFieldValue *url = dynamic_cast<const StructFieldValue*>(urlRef.get());
+    ASSERT_TRUE(url != NULL);
+
+    FieldValue::UP strRef = url->getValue("scheme");
+    const StringFieldValue *str = dynamic_cast<const StringFieldValue*>(strRef.get());
+    ASSERT_TRUE(str != NULL);
+
+    SpanTree::UP tree = std::move(str->getSpanTrees().front());
+
+    EXPECT_EQUAL("my_tree", tree->getName());
+    const SimpleSpanList *root = dynamic_cast<const SimpleSpanList*>(&tree->getRoot());
+    ASSERT_TRUE(root != NULL);
+    EXPECT_EQUAL(1u, root->size());
+    SimpleSpanList::const_iterator it = root->begin();
+    EXPECT_EQUAL(Span(0, 6), (*it++));
+    EXPECT_TRUE(it == root->end());
+
+    EXPECT_EQUAL(1u, tree->numAnnotations());
+}
+
+}  // namespace
+
+TEST_APPHOOK(Test);
diff --git a/document/src/tests/structfieldvaluetest.cpp b/document/src/tests/structfieldvaluetest.cpp
new file mode 100644
index 00000000000..c1bf53d5322
--- /dev/null
+++ b/document/src/tests/structfieldvaluetest.cpp
@@ -0,0 +1,185 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/document/fieldvalue/fieldvalues.h>
+#include <vespa/document/repo/configbuilder.h>
+#include <vespa/document/serialization/vespadocumentdeserializer.h>
+#include <vespa/vdstestlib/cppunit/macros.h>
+#include <vespa/vespalib/objects/nbostream.h>
+
+using vespalib::nbostream;
+using document::config_builder::Struct;
+using document::config_builder::Wset;
+using document::config_builder::Array;
+using document::config_builder::Map;
+
+namespace document {
+
+struct StructFieldValueTest : public CppUnit::TestFixture {
+    void setUp() {}
+    void tearDown() {}
+
+    void testStruct();
+
+    CPPUNIT_TEST_SUITE(StructFieldValueTest);
+    CPPUNIT_TEST(testStruct);
+    CPPUNIT_TEST_SUITE_END();
+
+};
+
+CPPUNIT_TEST_SUITE_REGISTRATION(StructFieldValueTest);
+
+namespace {
+template <typename T>
+void deserialize(const ByteBuffer &buffer, T &value, const FixedTypeRepo &repo)
+{
+    uint16_t version = Document::getNewestSerializationVersion();
+    nbostream stream(buffer.getBufferAtPos(), buffer.getRemaining());
+    VespaDocumentDeserializer deserializer(repo, stream, version);
+    deserializer.read(value);
+}
+}  // namespace
+
+void StructFieldValueTest::testStruct()
+{
+    config_builder::DocumenttypesConfigBuilderHelper builder;
+    builder.document(42, "test",
+                     Struct("test.header")
+                     .addField("int", DataType::T_INT)
+                     .addField("long", DataType::T_LONG)
+                     .addField("content", DataType::T_STRING),
+                     Struct("test.body"));
+    DocumentTypeRepo doc_repo(builder.config());
+    const DocumentType *doc_type = doc_repo.getDocumentType(42);
+    CPPUNIT_ASSERT(doc_type);
+    FixedTypeRepo repo(doc_repo, *doc_type);
+    const DataType &type = *repo.getDataType("test.header");
+    StructFieldValue value(type);
+    const Field &intF = value.getField("int");
+    const Field &longF = value.getField("long");
+    const Field &strF = value.getField("content");
+
+        // Initially empty
+    CPPUNIT_ASSERT_EQUAL(size_t(0), value.getSetFieldCount());
+    CPPUNIT_ASSERT(!value.hasValue(intF));
+
+    value.setValue(intF, IntFieldValue(1));
+
+        // Not empty
+    CPPUNIT_ASSERT_EQUAL(size_t(1), value.getSetFieldCount());
+    CPPUNIT_ASSERT(value.hasValue(intF));
+
+        // Adding some more
+    value.setValue(longF, LongFieldValue(2));
+
+        // Not empty
+    CPPUNIT_ASSERT_EQUAL(size_t(2), value.getSetFieldCount());
+    CPPUNIT_ASSERT_EQUAL(1, value.getValue(intF)->getAsInt());
+    CPPUNIT_ASSERT_EQUAL(2, value.getValue(longF)->getAsInt());
+
+        // Serialize & equality
+    std::unique_ptr<ByteBuffer> buffer(value.serialize());
+    buffer->flip();
+
+    CPPUNIT_ASSERT_EQUAL(buffer->getLength(), buffer->getLimit());
+    StructFieldValue value2(type);
+    CPPUNIT_ASSERT(value != value2);
+
+    deserialize(*buffer, value2, repo);
+
+    CPPUNIT_ASSERT(value2.hasValue(intF));
+    CPPUNIT_ASSERT_EQUAL(value, value2);
+
+    // Various ways of removing
+    {
+        // By value
+        buffer->setPos(0);
+        deserialize(*buffer, value2, repo);
+        value2.remove(intF);
+        CPPUNIT_ASSERT(!value2.hasValue(intF));
+        CPPUNIT_ASSERT_EQUAL(size_t(1), value2.getSetFieldCount());
+
+        // Clearing all
+        buffer->setPos(0);
+        deserialize(*buffer, value2, repo);
+        value2.clear();
+        CPPUNIT_ASSERT(!value2.hasValue(intF));
+        CPPUNIT_ASSERT_EQUAL(size_t(0), value2.getSetFieldCount());
+    }
+
+    // Updating
+    value2 = value;
+    CPPUNIT_ASSERT_EQUAL(value, value2);
+    value2.setValue(strF, StringFieldValue("foo"));
+    CPPUNIT_ASSERT(value2.hasValue(strF));
+    CPPUNIT_ASSERT_EQUAL(vespalib::string("foo"),
+                         value2.getValue(strF)->getAsString());
+    CPPUNIT_ASSERT(value != value2);
+    value2.assign(value);
+    CPPUNIT_ASSERT_EQUAL(value, value2);
+    StructFieldValue::UP valuePtr(value2.clone());
+
+    CPPUNIT_ASSERT(valuePtr.get());
+    CPPUNIT_ASSERT_EQUAL(value, *valuePtr);
+
+        // Iterating
+    const StructFieldValue& constVal(value);
+    for(StructFieldValue::const_iterator it = constVal.begin();
+        it != constVal.end(); ++it)
+    {
+        constVal.getValue(it.field());
+    }
+
+        // Comparison
+    value2 = value;
+    CPPUNIT_ASSERT_EQUAL(0, value.compare(value2));
+    value2.remove(intF);
+    CPPUNIT_ASSERT(value.compare(value2) < 0);
+    CPPUNIT_ASSERT(value2.compare(value) > 0);
+    value2 = value;
+    value2.setValue(intF, IntFieldValue(5));
+    CPPUNIT_ASSERT(value.compare(value2) < 0);
+    CPPUNIT_ASSERT(value2.compare(value) > 0);
+
+        // Output
+    CPPUNIT_ASSERT_EQUAL(
+            std::string("Struct test.header(\n"
+                        "  int - 1,\n"
+                        "  long - 2\n"
+                        ")"),
+            value.toString(false));
+    CPPUNIT_ASSERT_EQUAL(
+            std::string("  Struct test.header(\n"
+                        "..  int - 1,\n"
+                        "..  long - 2\n"
+                        "..)"),
+            "  " + value.toString(true, ".."));
+    CPPUNIT_ASSERT_EQUAL(
+            std::string("<value>\n"
+                        "  <int>1</int>\n"
+                        "  <long>2</long>\n"
+                        "</value>"),
+            value.toXml("  "));
+
+        // Failure situations.
+
+        // Refuse to accept non-struct types
+    try{
+        StructFieldValue value6(*DataType::DOCUMENT);
+        CPPUNIT_FAIL("Didn't complain about non-struct type");
+    } catch (std::exception& e) {
+        CPPUNIT_ASSERT_CONTAIN("Cannot generate a struct value with "
+                               "non-struct type", e.what());
+    }
+
+        // Refuse to set wrong types
+    try{
+        value2.setValue(intF, StringFieldValue("bar"));
+        CPPUNIT_FAIL("Failed to check type equality in setValue");
+    } catch (std::exception& e) {
+        CPPUNIT_ASSERT_CONTAIN("Cannot assign value of type", e.what());
+    }
+}
+
+} // document
+
diff --git a/document/src/tests/tensor_fieldvalue/.gitignore b/document/src/tests/tensor_fieldvalue/.gitignore
new file mode 100644
index 00000000000..74ede436f4c
--- /dev/null
+++ b/document/src/tests/tensor_fieldvalue/.gitignore
@@ -0,0 +1 @@
+document_tensor_fieldvalue_test_app
diff --git a/document/src/tests/tensor_fieldvalue/CMakeLists.txt b/document/src/tests/tensor_fieldvalue/CMakeLists.txt
new file mode 100644
index 00000000000..d0fa1526bbc
--- /dev/null
+++ b/document/src/tests/tensor_fieldvalue/CMakeLists.txt
@@ -0,0 +1,10 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(document_tensor_fieldvalue_test_app
+    SOURCES
+    tensor_fieldvalue_test.cpp
+    DEPENDS
+    document
+    AFTER
+    document_documentconfig
+)
+vespa_add_test(NAME document_tensor_fieldvalue_test_app COMMAND document_tensor_fieldvalue_test_app)
diff --git a/document/src/tests/tensor_fieldvalue/tensor_fieldvalue_test.cpp b/document/src/tests/tensor_fieldvalue/tensor_fieldvalue_test.cpp
new file mode 100644
index 00000000000..f0dd49c472c
--- /dev/null
+++ b/document/src/tests/tensor_fieldvalue/tensor_fieldvalue_test.cpp
@@ -0,0 +1,72 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Unit tests for tensor_fieldvalue.
+
+#include <vespa/log/log.h>
+LOG_SETUP("fieldvalue_test");
+#include <vespa/fastos/fastos.h>
+
+#include <vespa/document/fieldvalue/tensorfieldvalue.h>
+#include <vespa/vespalib/tensor/tensor.h>
+#include <vespa/vespalib/tensor/types.h>
+#include <vespa/vespalib/tensor/default_tensor.h>
+#include <vespa/vespalib/tensor/tensor_factory.h>
+
+#include <vespa/vespalib/testkit/testapp.h>
+
+using namespace document;
+using namespace vespalib::tensor;
+
+namespace
+{
+
+Tensor::UP
+createTensor(const TensorCells &cells, const TensorDimensions &dimensions) {
+    vespalib::tensor::DefaultTensor::builder builder;
+    return vespalib::tensor::TensorFactory::create(cells, dimensions, builder);
+}
+
+FieldValue::UP clone(FieldValue &fv) {
+    auto ret = FieldValue::UP(fv.clone());
+    EXPECT_NOT_EQUAL(ret.get(), &fv);
+    EXPECT_EQUAL(*ret, fv);
+    EXPECT_EQUAL(fv, *ret);
+    return ret;
+}
+
+}
+
+TEST("require that TensorFieldValue can be assigned tensors and cloned") {
+    TensorFieldValue noTensorValue;
+    TensorFieldValue emptyTensorValue;
+    TensorFieldValue twoCellsTwoDimsValue;
+    emptyTensorValue = createTensor({}, {});
+    twoCellsTwoDimsValue = createTensor({ {{{"y", "3"}}, 3},
+                                             {{{"x", "4"}, {"y", "5"}}, 7} },
+                                        {"x", "y"});
+    EXPECT_NOT_EQUAL(noTensorValue, emptyTensorValue);
+    EXPECT_NOT_EQUAL(noTensorValue, twoCellsTwoDimsValue);
+    EXPECT_NOT_EQUAL(emptyTensorValue, noTensorValue);
+    EXPECT_NOT_EQUAL(emptyTensorValue, twoCellsTwoDimsValue);
+    EXPECT_NOT_EQUAL(twoCellsTwoDimsValue, noTensorValue);
+    EXPECT_NOT_EQUAL(twoCellsTwoDimsValue, emptyTensorValue);
+    FieldValue::UP noneClone = clone(noTensorValue);
+    FieldValue::UP emptyClone = clone(emptyTensorValue);
+    FieldValue::UP twoClone = clone(twoCellsTwoDimsValue);
+    EXPECT_NOT_EQUAL(*noneClone, *emptyClone);
+    EXPECT_NOT_EQUAL(*noneClone, *twoClone);
+    EXPECT_NOT_EQUAL(*emptyClone, *noneClone);
+    EXPECT_NOT_EQUAL(*emptyClone, *twoClone);
+    EXPECT_NOT_EQUAL(*twoClone, *noneClone);
+    EXPECT_NOT_EQUAL(*twoClone, *emptyClone);
+    TensorFieldValue twoCellsTwoDimsValue2;
+    twoCellsTwoDimsValue2 =
+        createTensor({ {{{"y", "3"}}, 3},
+                          {{{"x", "4"}, {"y", "5"}}, 7} },
+                     {"x", "y"});
+    EXPECT_NOT_EQUAL(*noneClone, twoCellsTwoDimsValue2);
+    EXPECT_NOT_EQUAL(*emptyClone, twoCellsTwoDimsValue2);
+    EXPECT_EQUAL(*twoClone, twoCellsTwoDimsValue2);
+}
+
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/document/src/tests/testbytebuffer.cpp b/document/src/tests/testbytebuffer.cpp
new file mode 100644
index 00000000000..b4b8eb55e95
--- /dev/null
+++ b/document/src/tests/testbytebuffer.cpp
@@ -0,0 +1,493 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+/* $Id$*/
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/document/util/stringutil.h>
+#include <vespa/document/util/bytebuffer.h>
+#include <vespa/document/fieldvalue/serializablearray.h>
+#include "heapdebugger.h"
+#include <iostream>
+#include "testbytebuffer.h"
+#include <vespa/vespalib/util/macro.h>
+
+using namespace document;
+
+CPPUNIT_TEST_SUITE_REGISTRATION( ByteBuffer_Test );
+
+
+void ByteBuffer_Test::setUp()
+{
+    enableHeapUsageMonitor();
+}
+
+
+void ByteBuffer_Test::test_constructors()
+{
+    size_t MemUsedAtEntry=getHeapUsage();
+
+
+    ByteBuffer* simple=new ByteBuffer();
+    delete simple;
+
+    ByteBuffer* less_simple=new ByteBuffer("hei",3);
+    CPPUNIT_ASSERT(strcmp(less_simple->getBufferAtPos(),"hei")==0);
+    delete less_simple;
+
+    CPPUNIT_ASSERT(getHeapUsage()-MemUsedAtEntry == 0);
+}
+
+void ByteBuffer_Test::test_assignment_operator()
+{
+    size_t MemUsedAtEntry=getHeapUsage();
+    try {
+        ByteBuffer b1;
+        ByteBuffer b2 = b1;
+
+        CPPUNIT_ASSERT_EQUAL(b1.getPos(),b2.getPos());
+        CPPUNIT_ASSERT_EQUAL(b1.getLength(),b2.getLength());
+        CPPUNIT_ASSERT_EQUAL(b1.getLimit(),b2.getLimit());
+        CPPUNIT_ASSERT_EQUAL(b1.getRemaining(),b2.getRemaining());
+
+    } catch (std::exception &e) {
+        fprintf(stderr,"Unexpected exception at %s: \"%s\"\n", VESPA_STRLOC.c_str(),e.what());
+        CPPUNIT_ASSERT(false);
+    }
+
+    try {
+        ByteBuffer b1(100);
+        b1.putInt(1);
+        b1.putInt(2);
+
+        ByteBuffer b2 = b1;
+
+        CPPUNIT_ASSERT_EQUAL(b1.getPos(),b2.getPos());
+        CPPUNIT_ASSERT_EQUAL(b1.getLength(),b2.getLength());
+        CPPUNIT_ASSERT_EQUAL(b1.getLimit(),b2.getLimit());
+        CPPUNIT_ASSERT_EQUAL(b1.getRemaining(),b2.getRemaining());
+
+        int test = 0;
+        b2.flip();
+        b2.getInt(test);
+        CPPUNIT_ASSERT_EQUAL(1,test);
+        b2.getInt(test);
+        CPPUNIT_ASSERT_EQUAL(2,test);
+
+
+        CPPUNIT_ASSERT_EQUAL(b1.getPos(),b2.getPos());
+        CPPUNIT_ASSERT_EQUAL(b1.getLength(),b2.getLength());
+        CPPUNIT_ASSERT_EQUAL((size_t) 8,b2.getLimit());
+        CPPUNIT_ASSERT_EQUAL((size_t) 0,b2.getRemaining());
+
+        // Test Selfassignment == no change
+        //
+        b2 = b2;
+
+
+        CPPUNIT_ASSERT_EQUAL(b1.getPos(),b2.getPos());
+        CPPUNIT_ASSERT_EQUAL(b1.getLength(),b2.getLength());
+        CPPUNIT_ASSERT_EQUAL((size_t) 8,b2.getLimit());
+        CPPUNIT_ASSERT_EQUAL((size_t) 0,b2.getRemaining());
+
+        ByteBuffer b3;
+        // Empty
+        b2 = b3;
+
+        CPPUNIT_ASSERT_EQUAL((size_t) 0,b2.getPos());
+        CPPUNIT_ASSERT_EQUAL((size_t) 0,b2.getLength());
+        CPPUNIT_ASSERT_EQUAL((size_t) 0,b2.getLimit());
+        CPPUNIT_ASSERT_EQUAL((size_t) 0,b2.getRemaining());
+
+    } catch (std::exception &e) {
+        fprintf(stderr,"Unexpected exception at %s: \"%s\"\n", VESPA_STRLOC.c_str(),e.what());
+        CPPUNIT_ASSERT(false);
+    }
+
+    CPPUNIT_ASSERT(getHeapUsage()-MemUsedAtEntry == 0);
+
+}
+
+void ByteBuffer_Test::test_copy_constructor()
+{
+    size_t MemUsedAtEntry=getHeapUsage();
+    try {
+        // Empty buffer first
+        ByteBuffer b1;
+        ByteBuffer b2(b1);
+
+        CPPUNIT_ASSERT_EQUAL(b1.getPos(),b2.getPos());
+        CPPUNIT_ASSERT_EQUAL(b1.getLength(),b2.getLength());
+        CPPUNIT_ASSERT_EQUAL(b1.getLimit(),b2.getLimit());
+        CPPUNIT_ASSERT_EQUAL(b1.getRemaining(),b2.getRemaining());
+
+    } catch (std::exception &e) {
+        fprintf(stderr,"Unexpected exception at %s: %s\n", VESPA_STRLOC.c_str(),e.what());
+        CPPUNIT_ASSERT(false);
+    }
+
+    try {
+        ByteBuffer b1(100);
+        b1.putInt(1);
+        b1.putInt(2);
+        ByteBuffer b2(b1);
+
+
+        CPPUNIT_ASSERT_EQUAL(b1.getPos(),b2.getPos());
+        CPPUNIT_ASSERT_EQUAL(b1.getLength(),b2.getLength());
+        CPPUNIT_ASSERT_EQUAL(b1.getLimit(),b2.getLimit());
+        CPPUNIT_ASSERT_EQUAL(b1.getRemaining(),b2.getRemaining());
+
+        int test = 0;
+        b2.flip();
+        b2.getInt(test);
+        CPPUNIT_ASSERT_EQUAL(1,test);
+        b2.getInt(test);
+        CPPUNIT_ASSERT_EQUAL(2,test);
+
+    } catch (std::exception &e) {
+        fprintf(stderr,"Unexpected exception at %s: %s\n", VESPA_STRLOC.c_str(),e.what());
+        CPPUNIT_ASSERT(false);
+    }
+
+    CPPUNIT_ASSERT(getHeapUsage()-MemUsedAtEntry == 0);
+
+}
+
+void ByteBuffer_Test::test_slice()
+{
+    ByteBuffer* newBuf=ByteBuffer::copyBuffer("hei der",8);
+
+    ByteBuffer* slice = new ByteBuffer;
+    slice->sliceFrom(*newBuf, 0,newBuf->getLength());
+    delete newBuf;
+    newBuf = NULL;
+
+    CPPUNIT_ASSERT(strcmp(slice->getBufferAtPos(),"hei der")==0);
+
+    ByteBuffer* slice2 = new ByteBuffer;
+    slice2->sliceFrom(*slice, 4, slice->getLength());
+    delete slice;
+    slice = NULL;
+
+    CPPUNIT_ASSERT(strcmp(slice2->getBufferAtPos(),"der")==0);
+    CPPUNIT_ASSERT(strcmp(slice2->getBuffer(),"hei der")==0);
+    delete slice2;
+    slice2 = NULL;
+
+    ByteBuffer* newBuf2=new ByteBuffer("hei der", 8);
+    ByteBuffer* slice3=new ByteBuffer;
+    ByteBuffer* slice4=new ByteBuffer;
+
+    slice3->sliceFrom(*newBuf2, 4, newBuf2->getLength());
+    slice4->sliceFrom(*newBuf2, 0, newBuf2->getLength());
+    delete newBuf2;
+    newBuf2 = NULL;
+
+    CPPUNIT_ASSERT(strcmp(slice3->getBufferAtPos(),"der")==0);
+    CPPUNIT_ASSERT(strcmp(slice4->getBuffer(),"hei der")==0);
+
+    delete slice3;
+    slice3 = NULL;
+
+    CPPUNIT_ASSERT(strcmp(slice4->getBuffer(),"hei der")==0);
+
+    delete slice4;
+    slice4 = NULL;
+}
+
+void ByteBuffer_Test::test_slice2()
+{
+    size_t MemUsedAtEntry=getHeapUsage();
+
+    ByteBuffer* newBuf=ByteBuffer::copyBuffer("hei der",8);
+
+    ByteBuffer slice;
+    slice.sliceFrom(*newBuf, 0, newBuf->getLength());
+
+    delete newBuf;
+    newBuf = NULL;
+
+    CPPUNIT_ASSERT(strcmp(slice.getBufferAtPos(),"hei der")==0);
+
+    ByteBuffer slice2;
+    slice2.sliceFrom(slice, 4, slice.getLength());
+
+    CPPUNIT_ASSERT(strcmp(slice2.getBufferAtPos(),"der")==0);
+    CPPUNIT_ASSERT(strcmp(slice2.getBuffer(),"hei der")==0);
+
+    ByteBuffer* newBuf2=new ByteBuffer("hei der", 8);
+
+    slice.sliceFrom(*newBuf2, 4, newBuf2->getLength());
+    slice2.sliceFrom(*newBuf2, 0, newBuf2->getLength());
+    delete newBuf2;
+    newBuf2 = NULL;
+
+    CPPUNIT_ASSERT(strcmp(slice.getBufferAtPos(),"der")==0);
+    CPPUNIT_ASSERT(strcmp(slice2.getBuffer(),"hei der")==0);
+
+    CPPUNIT_ASSERT(getHeapUsage()-MemUsedAtEntry == 0);
+}
+
+
+void ByteBuffer_Test::test_putGetFlip()
+{
+    ByteBuffer* newBuf=new ByteBuffer(100);
+
+    try {
+        newBuf->putInt(10);
+        int test;
+        newBuf->flip();
+
+        newBuf->getInt(test);
+        CPPUNIT_ASSERT(test==10);
+
+        newBuf->clear();
+        newBuf->putDouble(3.35);
+        newBuf->flip();
+        CPPUNIT_ASSERT(newBuf->getRemaining()==sizeof(double));
+        double test2;
+        newBuf->getDouble(test2);
+        CPPUNIT_ASSERT(test2==3.35);
+
+        newBuf->clear();
+        newBuf->putBytes("heisann",8);
+        newBuf->putInt(4);
+        CPPUNIT_ASSERT(newBuf->getPos()==12);
+        CPPUNIT_ASSERT(newBuf->getLength()==100);
+        newBuf->flip();
+        CPPUNIT_ASSERT(newBuf->getRemaining()==12);
+
+        char testStr[12];
+        newBuf->getBytes(testStr, 8);
+        CPPUNIT_ASSERT(strcmp(testStr,"heisann")==0);
+        newBuf->getInt(test);
+        CPPUNIT_ASSERT(test==4);
+    } catch (std::exception &e) {
+        fprintf(stderr,"Unexpected exception at %s: %s\n", VESPA_STRLOC.c_str(),e.what());
+        CPPUNIT_ASSERT(false);
+    }
+    delete newBuf;
+}
+
+
+void ByteBuffer_Test::test_NumberEncodings()
+{
+    ByteBuffer* buf=new ByteBuffer(1024);
+
+    // Check 0
+    buf->putInt1_2_4Bytes(124);
+    buf->putInt2_4_8Bytes(124);
+    buf->putInt1_4Bytes(124);
+    // Check 1
+    buf->putInt1_2_4Bytes(127);
+    buf->putInt2_4_8Bytes(127);
+    buf->putInt1_4Bytes(127);
+    // Check 2
+    buf->putInt1_2_4Bytes(128);
+    buf->putInt2_4_8Bytes(128);
+    buf->putInt1_4Bytes(128);
+    // Check 3
+    buf->putInt1_2_4Bytes(255);
+    buf->putInt2_4_8Bytes(255);
+    buf->putInt1_4Bytes(255);
+    // Check 4
+    buf->putInt1_2_4Bytes(256);
+    buf->putInt2_4_8Bytes(256);
+    buf->putInt1_4Bytes(256);
+    // Check 5
+    buf->putInt1_2_4Bytes(0);
+    buf->putInt2_4_8Bytes(0);
+    buf->putInt1_4Bytes(0);
+    // Check 6
+    buf->putInt1_2_4Bytes(1);
+    buf->putInt2_4_8Bytes(1);
+    buf->putInt1_4Bytes(1);
+
+    // Check 7
+    try {
+        buf->putInt1_2_4Bytes(0x7FFFFFFF);
+        CPPUNIT_ASSERT(false);
+    } catch (InputOutOfRangeException& e) { }
+    buf->putInt2_4_8Bytes(0x7FFFFFFFll);
+    buf->putInt1_4Bytes(0x7FFFFFFF);
+
+    try {
+        buf->putInt2_4_8Bytes(0x7FFFFFFFFFFFFFFFll);
+        CPPUNIT_ASSERT(false);
+    } catch (InputOutOfRangeException& e) { }
+
+    buf->putInt1_2_4Bytes(0x7FFF);
+    // Check 8
+    buf->putInt2_4_8Bytes(0x7FFFll);
+    buf->putInt1_4Bytes(0x7FFF);
+    buf->putInt1_2_4Bytes(0x7F);
+    // Check 9
+    buf->putInt2_4_8Bytes(0x7Fll);
+    buf->putInt1_4Bytes(0x7F);
+
+    try {
+        buf->putInt1_2_4Bytes(-1);
+        CPPUNIT_ASSERT(false);
+    } catch (InputOutOfRangeException& e) { }
+    try {
+        buf->putInt2_4_8Bytes(-1);
+        CPPUNIT_ASSERT(false);
+    } catch (InputOutOfRangeException& e) { }
+    try {
+        buf->putInt1_4Bytes(-1);
+        CPPUNIT_ASSERT(false);
+    } catch (InputOutOfRangeException& e) { }
+
+    try {
+        buf->putInt1_2_4Bytes(-0x7FFFFFFF);
+        CPPUNIT_ASSERT(false);
+    } catch (InputOutOfRangeException& e) { }
+    try {
+        buf->putInt2_4_8Bytes(-0x7FFFFFFF);
+        CPPUNIT_ASSERT(false);
+    } catch (InputOutOfRangeException& e) { }
+    try {
+        buf->putInt1_4Bytes(-0x7FFFFFFF);
+        CPPUNIT_ASSERT(false);
+    } catch (InputOutOfRangeException& e) { }
+
+    try {
+        buf->putInt2_4_8Bytes(-0x7FFFFFFFFFFFFFFFll);
+        CPPUNIT_ASSERT(false);
+    } catch (InputOutOfRangeException& e) { }
+
+    uint32_t endWritePos = buf->getPos();
+    buf->setPos(0);
+
+    int32_t tmp32;
+    int64_t tmp64;
+
+    // Check 0
+    buf->getInt1_2_4Bytes(tmp32);
+    CPPUNIT_ASSERT_EQUAL(124, tmp32);
+    buf->getInt2_4_8Bytes(tmp64);
+    CPPUNIT_ASSERT_EQUAL((int64_t)124, tmp64);
+    buf->getInt1_4Bytes(tmp32);
+    CPPUNIT_ASSERT_EQUAL(124, tmp32);
+    // Check 1
+    buf->getInt1_2_4Bytes(tmp32);
+    CPPUNIT_ASSERT_EQUAL(127, tmp32);
+    buf->getInt2_4_8Bytes(tmp64);
+    CPPUNIT_ASSERT_EQUAL((int64_t)127, tmp64);
+    buf->getInt1_4Bytes(tmp32);
+    CPPUNIT_ASSERT_EQUAL(127, tmp32);
+    // Check 2
+    buf->getInt1_2_4Bytes(tmp32);
+    CPPUNIT_ASSERT_EQUAL(128, tmp32);
+    buf->getInt2_4_8Bytes(tmp64);
+    CPPUNIT_ASSERT_EQUAL((int64_t)128, tmp64);
+    buf->getInt1_4Bytes(tmp32);
+    CPPUNIT_ASSERT_EQUAL(128, tmp32);
+    // Check 3
+    buf->getInt1_2_4Bytes(tmp32);
+    CPPUNIT_ASSERT_EQUAL(255, tmp32);
+    buf->getInt2_4_8Bytes(tmp64);
+    CPPUNIT_ASSERT_EQUAL((int64_t)255, tmp64);
+    buf->getInt1_4Bytes(tmp32);
+    CPPUNIT_ASSERT_EQUAL(255, tmp32);
+    // Check 4
+    buf->getInt1_2_4Bytes(tmp32);
+    CPPUNIT_ASSERT_EQUAL(256, tmp32);
+    buf->getInt2_4_8Bytes(tmp64);
+    CPPUNIT_ASSERT_EQUAL((int64_t)256, tmp64);
+    buf->getInt1_4Bytes(tmp32);
+    CPPUNIT_ASSERT_EQUAL(256, tmp32);
+    // Check 5
+    buf->getInt1_2_4Bytes(tmp32);
+    CPPUNIT_ASSERT_EQUAL(0, tmp32);
+    buf->getInt2_4_8Bytes(tmp64);
+    CPPUNIT_ASSERT_EQUAL((int64_t)0, tmp64);
+    buf->getInt1_4Bytes(tmp32);
+    CPPUNIT_ASSERT_EQUAL(0, tmp32);
+    // Check 6
+    buf->getInt1_2_4Bytes(tmp32);
+    CPPUNIT_ASSERT_EQUAL(1, tmp32);
+    buf->getInt2_4_8Bytes(tmp64);
+    CPPUNIT_ASSERT_EQUAL((int64_t)1, tmp64);
+    buf->getInt1_4Bytes(tmp32);
+    CPPUNIT_ASSERT_EQUAL(1, tmp32);
+    // Check 7
+    buf->getInt2_4_8Bytes(tmp64);
+    CPPUNIT_ASSERT_EQUAL((int64_t)0x7FFFFFFF, tmp64);
+    buf->getInt1_4Bytes(tmp32);
+    CPPUNIT_ASSERT_EQUAL(0x7FFFFFFF, tmp32);
+    buf->getInt1_2_4Bytes(tmp32);
+    CPPUNIT_ASSERT_EQUAL(0x7FFF, tmp32);
+    // Check 8
+    buf->getInt2_4_8Bytes(tmp64);
+    CPPUNIT_ASSERT_EQUAL((int64_t)0x7FFF, tmp64);
+    buf->getInt1_4Bytes(tmp32);
+    CPPUNIT_ASSERT_EQUAL(0x7FFF, tmp32);
+    buf->getInt1_2_4Bytes(tmp32);
+    CPPUNIT_ASSERT_EQUAL(0x7F, tmp32);
+    // Check 9
+    buf->getInt2_4_8Bytes(tmp64);
+    CPPUNIT_ASSERT_EQUAL((int64_t)0x7F, tmp64);
+    buf->getInt1_4Bytes(tmp32);
+    CPPUNIT_ASSERT_EQUAL(0x7F, tmp32);
+
+    uint32_t endReadPos = buf->getPos();
+    CPPUNIT_ASSERT_EQUAL(endWritePos, endReadPos);
+
+    delete buf;
+}
+
+void ByteBuffer_Test::test_NumberLengths()
+{
+    ByteBuffer b;
+    CPPUNIT_ASSERT_EQUAL((size_t) 1, b.getSerializedSize1_4Bytes(0));
+    CPPUNIT_ASSERT_EQUAL((size_t) 1, b.getSerializedSize1_4Bytes(1));
+    CPPUNIT_ASSERT_EQUAL((size_t) 1, b.getSerializedSize1_4Bytes(4));
+    CPPUNIT_ASSERT_EQUAL((size_t) 1, b.getSerializedSize1_4Bytes(31));
+    CPPUNIT_ASSERT_EQUAL((size_t) 1, b.getSerializedSize1_4Bytes(126));
+    CPPUNIT_ASSERT_EQUAL((size_t) 1, b.getSerializedSize1_4Bytes(127));
+    CPPUNIT_ASSERT_EQUAL((size_t) 4, b.getSerializedSize1_4Bytes(128));
+    CPPUNIT_ASSERT_EQUAL((size_t) 4, b.getSerializedSize1_4Bytes(129));
+    CPPUNIT_ASSERT_EQUAL((size_t) 4, b.getSerializedSize1_4Bytes(255));
+    CPPUNIT_ASSERT_EQUAL((size_t) 4, b.getSerializedSize1_4Bytes(256));
+    CPPUNIT_ASSERT_EQUAL((size_t) 4, b.getSerializedSize1_4Bytes(0x7FFFFFFF));
+
+    CPPUNIT_ASSERT_EQUAL((size_t) 2, b.getSerializedSize2_4_8Bytes(0));
+    CPPUNIT_ASSERT_EQUAL((size_t) 2, b.getSerializedSize2_4_8Bytes(1));
+    CPPUNIT_ASSERT_EQUAL((size_t) 2, b.getSerializedSize2_4_8Bytes(4));
+    CPPUNIT_ASSERT_EQUAL((size_t) 2, b.getSerializedSize2_4_8Bytes(31));
+    CPPUNIT_ASSERT_EQUAL((size_t) 2, b.getSerializedSize2_4_8Bytes(126));
+    CPPUNIT_ASSERT_EQUAL((size_t) 2, b.getSerializedSize2_4_8Bytes(127));
+    CPPUNIT_ASSERT_EQUAL((size_t) 2, b.getSerializedSize2_4_8Bytes(128));
+    CPPUNIT_ASSERT_EQUAL((size_t) 2, b.getSerializedSize2_4_8Bytes(32767));
+    CPPUNIT_ASSERT_EQUAL((size_t) 4, b.getSerializedSize2_4_8Bytes(32768));
+    CPPUNIT_ASSERT_EQUAL((size_t) 4, b.getSerializedSize2_4_8Bytes(32769));
+    CPPUNIT_ASSERT_EQUAL((size_t) 4, b.getSerializedSize2_4_8Bytes(1030493));
+    CPPUNIT_ASSERT_EQUAL((size_t) 4, b.getSerializedSize2_4_8Bytes(0x3FFFFFFF));
+    CPPUNIT_ASSERT_EQUAL((size_t) 8, b.getSerializedSize2_4_8Bytes(0x40000000));
+    CPPUNIT_ASSERT_EQUAL((size_t) 8, b.getSerializedSize2_4_8Bytes(0x40000001));
+
+    CPPUNIT_ASSERT_EQUAL((size_t) 1, b.getSerializedSize1_2_4Bytes(0));
+    CPPUNIT_ASSERT_EQUAL((size_t) 1, b.getSerializedSize1_2_4Bytes(1));
+    CPPUNIT_ASSERT_EQUAL((size_t) 1, b.getSerializedSize1_2_4Bytes(4));
+    CPPUNIT_ASSERT_EQUAL((size_t) 1, b.getSerializedSize1_2_4Bytes(31));
+    CPPUNIT_ASSERT_EQUAL((size_t) 1, b.getSerializedSize1_2_4Bytes(126));
+    CPPUNIT_ASSERT_EQUAL((size_t) 1, b.getSerializedSize1_2_4Bytes(127));
+    CPPUNIT_ASSERT_EQUAL((size_t) 2, b.getSerializedSize1_2_4Bytes(128));
+    CPPUNIT_ASSERT_EQUAL((size_t) 2, b.getSerializedSize1_2_4Bytes(16383));
+    CPPUNIT_ASSERT_EQUAL((size_t) 4, b.getSerializedSize1_2_4Bytes(16384));
+    CPPUNIT_ASSERT_EQUAL((size_t) 4, b.getSerializedSize1_2_4Bytes(16385));
+}
+
+void ByteBuffer_Test::test_SerializableArray()
+{
+    SerializableArray array;
+    array.set(0,"http",4);
+    CPPUNIT_ASSERT_EQUAL(4ul, array.get(0).size());
+    SerializableArray copy(array);
+    CPPUNIT_ASSERT_EQUAL(4ul, array.get(0).size());
+    CPPUNIT_ASSERT_EQUAL(copy.get(0).size(), array.get(0).size());
+    CPPUNIT_ASSERT(copy.get(0).c_str() != array.get(0).c_str());
+    CPPUNIT_ASSERT_EQUAL(0, strcmp(copy.get(0).c_str(), array.get(0).c_str()));
+    CPPUNIT_ASSERT_EQUAL(16ul, sizeof(SerializableArray::Entry));
+}
diff --git a/document/src/tests/testbytebuffer.h b/document/src/tests/testbytebuffer.h
new file mode 100644
index 00000000000..58962d73449
--- /dev/null
+++ b/document/src/tests/testbytebuffer.h
@@ -0,0 +1,73 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/* $Id$*/
+
+#pragma once
+
+#include <cppunit/extensions/HelperMacros.h>
+
+/**
+   CPPUnit test case for ByteBuffer class.
+*/
+class ByteBuffer_Test : public CppUnit::TestFixture {
+    CPPUNIT_TEST_SUITE( ByteBuffer_Test);
+    CPPUNIT_TEST(test_constructors);
+    CPPUNIT_TEST(test_copy_constructor);
+    CPPUNIT_TEST(test_assignment_operator);
+    CPPUNIT_TEST(test_slice);
+    CPPUNIT_TEST(test_slice2);
+    CPPUNIT_TEST(test_putGetFlip);
+    CPPUNIT_TEST(test_NumberEncodings);
+    CPPUNIT_TEST(test_NumberLengths);
+    CPPUNIT_TEST(test_SerializableArray);
+    CPPUNIT_TEST_SUITE_END();
+
+public:
+  /**
+     Initialization.
+  */
+  void setUp();
+
+protected:
+    /**
+       Test construction and deletion.
+    */
+    void test_constructors();
+    void test_SerializableArray();
+
+    /**
+       Test copy constructor
+    */
+    void test_copy_constructor();
+    /**
+       Test construction and deletion.
+    */
+    void test_assignment_operator();
+
+    /**
+       Test the slice() method
+    */
+    void test_slice();
+
+    /**
+       Test the slice2() method
+    */
+    void test_slice2();
+
+    /**
+       Test put(), get() and flip() methods.
+    */
+    void test_putGetFlip();
+
+    /**
+       Test writing integers with funny encodings.
+    */
+    void test_NumberEncodings();
+
+    /**
+       Tests lengths of those encodings.
+    */
+    void test_NumberLengths();
+
+};
+
+
diff --git a/document/src/tests/testcase.h b/document/src/tests/testcase.h
new file mode 100644
index 00000000000..68ac5066c2d
--- /dev/null
+++ b/document/src/tests/testcase.h
@@ -0,0 +1,17 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/* $Id$*/
+
+#pragma once
+
+#include <cppunit/extensions/HelperMacros.h>
+
+class Document_Test : public CppUnit::TestFixture {
+  CPPUNIT_TEST_SUITE( Document_Test);
+  CPPUNIT_TEST(testShit);
+  CPPUNIT_TEST_SUITE_END();
+
+protected:
+  void testShit();
+};
+
+
diff --git a/document/src/tests/testdocmantest.cpp b/document/src/tests/testdocmantest.cpp
new file mode 100644
index 00000000000..14140b4b508
--- /dev/null
+++ b/document/src/tests/testdocmantest.cpp
@@ -0,0 +1,62 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+/* $Id$*/
+
+#include <vespa/fastos/fastos.h>
+#include <iostream>
+#include <set>
+#include <sstream>
+#include <vespa/document/base/testdocman.h>
+#include <vespa/vdstestlib/cppunit/macros.h>
+
+namespace document {
+
+class TestDocManTest : public CppUnit::TestFixture {
+    CPPUNIT_TEST_SUITE(TestDocManTest);
+    CPPUNIT_TEST(testSimpleUsage);
+    CPPUNIT_TEST_SUITE_END();
+
+protected:
+    void testSimpleUsage();
+};
+
+CPPUNIT_TEST_SUITE_REGISTRATION(TestDocManTest);
+
+void TestDocManTest::testSimpleUsage()
+{
+    TestDocMan testdm;
+    Document::UP doc1(testdm.createRandomDocument());
+    Document::UP doc2(testdm.createRandomDocument());
+    Document::UP doc3(testdm.createRandomDocument(1));
+    {
+        FieldValue::UP v(doc1->getValue(doc1->getField("content")));
+        StringFieldValue& sval(dynamic_cast<StringFieldValue&>(*v));
+        CPPUNIT_ASSERT_EQUAL(std::string("To be, or "),
+                             std::string(sval.getValue().c_str()));
+
+        FieldValue::UP v2(doc2->getValue(doc2->getField("content")));
+        StringFieldValue& sval2(dynamic_cast<StringFieldValue&>(*v));
+        CPPUNIT_ASSERT_EQUAL(std::string(sval.getValue().c_str()),
+                             std::string(sval2.getValue().c_str()));
+    }
+    {
+        FieldValue::UP v(doc3->getValue(doc3->getField("content")));
+        StringFieldValue& sval(dynamic_cast<StringFieldValue&>(*v));
+        CPPUNIT_ASSERT_EQUAL(
+                std::string("To be, or not to be: that is the question:\n"
+                            "Whether 'tis nobler in the mind to suffer\n"
+                            "The slings and a"),
+                std::string(sval.getValue().c_str()));
+    }
+    CPPUNIT_ASSERT_EQUAL(
+            vespalib::string("id:mail:testdoctype1:n=51019:192.html"),
+            doc1->getId().toString());
+    CPPUNIT_ASSERT_EQUAL(
+            vespalib::string("id:mail:testdoctype1:n=51019:192.html"),
+            doc2->getId().toString());
+    CPPUNIT_ASSERT_EQUAL(
+            vespalib::string("id:mail:testdoctype1:n=10744:245.html"),
+            doc3->getId().toString());
+}
+
+} // document
diff --git a/document/src/tests/testrunner.cpp b/document/src/tests/testrunner.cpp
new file mode 100644
index 00000000000..aabdbb3f605
--- /dev/null
+++ b/document/src/tests/testrunner.cpp
@@ -0,0 +1,15 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+#include <vespa/vdstestlib/cppunit/cppunittestrunner.h>
+
+LOG_SETUP("documentcppunittests");
+
+int
+main(int argc, char **argv)
+{
+    vdstestlib::CppUnitTestRunner testRunner;
+    return testRunner.run(argc, argv);
+}
+
diff --git a/document/src/tests/teststringutil.cpp b/document/src/tests/teststringutil.cpp
new file mode 100644
index 00000000000..9d04372a19b
--- /dev/null
+++ b/document/src/tests/teststringutil.cpp
@@ -0,0 +1,112 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ *  Test program for StringUtil methods
+ *
+ *  @version  $Id$
+ */
+
+#include <vespa/fastos/fastos.h>
+#include <stdio.h>
+#include <string.h>
+#include <stdint.h>
+#include <vespa/document/util/stringutil.h>
+#include "heapdebugger.h"
+#include "teststringutil.h"
+#include <cppunit/TestFixture.h>
+#include <cppunit/extensions/HelperMacros.h>
+
+CPPUNIT_TEST_SUITE_REGISTRATION( StringUtil_Test );
+
+using namespace document;
+using vespalib::string;
+
+void StringUtil_Test::setUp()
+{
+    enableHeapUsageMonitor();
+}
+
+void StringUtil_Test::test_escape()
+{
+    CPPUNIT_ASSERT_EQUAL(string("abz019ABZ"), StringUtil::escape("abz019ABZ"));
+    CPPUNIT_ASSERT_EQUAL(string("\\t"), StringUtil::escape("\t"));
+    CPPUNIT_ASSERT_EQUAL(string("\\n"), StringUtil::escape("\n"));
+    CPPUNIT_ASSERT_EQUAL(string("\\r"), StringUtil::escape("\r"));
+    CPPUNIT_ASSERT_EQUAL(string("\\\""), StringUtil::escape("\""));
+    CPPUNIT_ASSERT_EQUAL(string("\\f"), StringUtil::escape("\f"));
+    CPPUNIT_ASSERT_EQUAL(string("\\\\"), StringUtil::escape("\\"));
+    CPPUNIT_ASSERT_EQUAL(string("\\x05"), StringUtil::escape("\x05"));
+    CPPUNIT_ASSERT_EQUAL(string("\\tA\\ncombined\\r\\x055test"),
+                         StringUtil::escape("\tA\ncombined\r\x05""5test"));
+    CPPUNIT_ASSERT_EQUAL(string("A\\x20space\\x20separated\\x20string"),
+                         StringUtil::escape("A space separated string", ' '));
+}
+
+void StringUtil_Test::test_unescape()
+{
+    CPPUNIT_ASSERT_EQUAL(string("abz019ABZ"),
+                         StringUtil::unescape("abz019ABZ"));
+    CPPUNIT_ASSERT_EQUAL(string("\t"), StringUtil::unescape("\\t"));
+    CPPUNIT_ASSERT_EQUAL(string("\n"), StringUtil::unescape("\\n"));
+    CPPUNIT_ASSERT_EQUAL(string("\r"), StringUtil::unescape("\\r"));
+    CPPUNIT_ASSERT_EQUAL(string("\""), StringUtil::unescape("\\\""));
+    CPPUNIT_ASSERT_EQUAL(string("\f"), StringUtil::unescape("\\f"));
+    CPPUNIT_ASSERT_EQUAL(string("\\"), StringUtil::unescape("\\\\"));
+    CPPUNIT_ASSERT_EQUAL(string("\x05"), StringUtil::unescape("\\x05"));
+    CPPUNIT_ASSERT_EQUAL(string("\tA\ncombined\r\x05""5test"),
+                         StringUtil::unescape("\\tA\\ncombined\\r\\x055test"));
+    CPPUNIT_ASSERT_EQUAL(string("A space separated string"),
+                         StringUtil::unescape("A\\x20space\\x20separated\\x20string"));
+}
+
+void StringUtil_Test::test_printAsHex()
+{
+    std::vector<char> asciitable(256);
+    for (uint32_t i=0; i<256; ++i) asciitable[i] = i;
+    std::ostringstream ost;
+    ost << "\n  ";
+    StringUtil::printAsHex(ost, &asciitable[0], asciitable.size(),
+                           16, true, "  ");
+    std::string expected("\n"
+                         "    0: 00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f\n"
+                         "   16: 10 11 12 13 14 15 16 17 18 19 1a 1b 1c 1d 1e 1f\n"
+                         "   32: 20  !  \"  #  $  %  &  '  (  )  *  +  ,  -  .  /\n"
+                         "   48:  0  1  2  3  4  5  6  7  8  9  :  ;  <  =  >  ?\n"
+                         "   64:  @  A  B  C  D  E  F  G  H  I  J  K  L  M  N  O\n"
+                         "   80:  P  Q  R  S  T  U  V  W  X  Y  Z  [  \\  ]  ^  _\n"
+                         "   96:  `  a  b  c  d  e  f  g  h  i  j  k  l  m  n  o\n"
+                         "  112:  p  q  r  s  t  u  v  w  x  y  z  {  |  }  ~ 7f\n"
+                         "  128: 80 81 82 83 84 85 86 87 88 89 8a 8b 8c 8d 8e 8f\n"
+                         "  144: 90 91 92 93 94 95 96 97 98 99 9a 9b 9c 9d 9e 9f\n"
+                         "  160: a0 a1 a2 a3 a4 a5 a6 a7 a8 a9 aa ab ac ad ae af\n"
+                         "  176: b0 b1 b2 b3 b4 b5 b6 b7 b8 b9 ba bb bc bd be bf\n"
+                         "  192: c0 c1 c2 c3 c4 c5 c6 c7 c8 c9 ca cb cc cd ce cf\n"
+                         "  208: d0 d1 d2 d3 d4 d5 d6 d7 d8 d9 da db dc dd de df\n"
+                         "  224: e0 e1 e2 e3 e4 e5 e6 e7 e8 e9 ea eb ec ed ee ef\n"
+                         "  240: f0 f1 f2 f3 f4 f5 f6 f7 f8 f9 fa fb fc fd fe ff");
+    CPPUNIT_ASSERT_EQUAL(expected, ost.str());
+
+    ost.str("");
+    ost << "\n";
+    StringUtil::printAsHex(ost, &asciitable[0], asciitable.size(),
+                           15, false);
+    expected = "\n"
+               "  0: 00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e ...............\n"
+               " 15: 0f 10 11 12 13 14 15 16 17 18 19 1a 1b 1c 1d ...............\n"
+               " 30: 1e 1f 20 21 22 23 24 25 26 27 28 29 2a 2b 2c ...!\"#$%&'()*+,\n"
+               " 45: 2d 2e 2f 30 31 32 33 34 35 36 37 38 39 3a 3b -./0123456789:;\n"
+               " 60: 3c 3d 3e 3f 40 41 42 43 44 45 46 47 48 49 4a <=>?@ABCDEFGHIJ\n"
+               " 75: 4b 4c 4d 4e 4f 50 51 52 53 54 55 56 57 58 59 KLMNOPQRSTUVWXY\n"
+               " 90: 5a 5b 5c 5d 5e 5f 60 61 62 63 64 65 66 67 68 Z[\\]^_`abcdefgh\n"
+               "105: 69 6a 6b 6c 6d 6e 6f 70 71 72 73 74 75 76 77 ijklmnopqrstuvw\n"
+               "120: 78 79 7a 7b 7c 7d 7e 7f 80 81 82 83 84 85 86 xyz{|}~........\n"
+               "135: 87 88 89 8a 8b 8c 8d 8e 8f 90 91 92 93 94 95 ...............\n"
+               "150: 96 97 98 99 9a 9b 9c 9d 9e 9f a0 a1 a2 a3 a4 ...............\n"
+               "165: a5 a6 a7 a8 a9 aa ab ac ad ae af b0 b1 b2 b3 ...............\n"
+               "180: b4 b5 b6 b7 b8 b9 ba bb bc bd be bf c0 c1 c2 ...............\n"
+               "195: c3 c4 c5 c6 c7 c8 c9 ca cb cc cd ce cf d0 d1 ...............\n"
+               "210: d2 d3 d4 d5 d6 d7 d8 d9 da db dc dd de df e0 ...............\n"
+               "225: e1 e2 e3 e4 e5 e6 e7 e8 e9 ea eb ec ed ee ef ...............\n"
+               "240: f0 f1 f2 f3 f4 f5 f6 f7 f8 f9 fa fb fc fd fe ...............\n"
+               "255: ff                                           .";
+    CPPUNIT_ASSERT_EQUAL(expected, ost.str());
+}
diff --git a/document/src/tests/teststringutil.h b/document/src/tests/teststringutil.h
new file mode 100644
index 00000000000..be25c9667d3
--- /dev/null
+++ b/document/src/tests/teststringutil.h
@@ -0,0 +1,26 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+/* $Id$*/
+
+#pragma once
+
+#include <cppunit/extensions/HelperMacros.h>
+
+class StringUtil_Test : public CppUnit::TestFixture {
+  CPPUNIT_TEST_SUITE( StringUtil_Test);
+  CPPUNIT_TEST(test_escape);
+  CPPUNIT_TEST(test_unescape);
+  CPPUNIT_TEST(test_printAsHex);
+  CPPUNIT_TEST_SUITE_END();
+
+public:
+  void setUp();
+
+protected:
+  void test_escape();
+  void test_unescape();
+  void test_printAsHex();
+
+};
+
+
diff --git a/document/src/tests/testxml.cpp b/document/src/tests/testxml.cpp
new file mode 100644
index 00000000000..7729c748b3f
--- /dev/null
+++ b/document/src/tests/testxml.cpp
@@ -0,0 +1,153 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+/* $Id$*/
+
+#include <vespa/fastos/fastos.h>
+#include <cppunit/extensions/HelperMacros.h>
+#include <vespa/document/config/config-documenttypes.h>
+#include <vespa/document/repo/documenttyperepo.h>
+#include <iostream>
+#include <set>
+#include <sstream>
+#include <vespa/document/update/documentupdate.h>
+#include <vespa/document/update/fieldupdate.h>
+#include <vespa/document/update/addvalueupdate.h>
+#include <vespa/document/update/assignvalueupdate.h>
+#include <vespa/document/update/removevalueupdate.h>
+#include <vespa/document/fieldvalue/fieldvalues.h>
+#include <vespa/vespalib/text/stringtokenizer.h>
+
+using vespalib::StringTokenizer;
+
+namespace document {
+
+class TestXml : public CppUnit::TestFixture {
+    CPPUNIT_TEST_SUITE(TestXml);
+    CPPUNIT_TEST(testSimpleUsage);
+    CPPUNIT_TEST(testDocumentUpdate);
+    CPPUNIT_TEST_SUITE_END();
+
+    void testSimpleUsage();
+    void testDocumentUpdate();
+};
+
+CPPUNIT_TEST_SUITE_REGISTRATION(TestXml);
+
+namespace {
+
+Document::UP createTestDocument(const DocumentTypeRepo& repo)
+{
+    const DocumentType* type(repo.getDocumentType("testdoc"));
+    Document::UP
+        doc(new Document(*type,
+                         DocumentId("doc:crawler/http://www.ntnu.no/")));
+    doc->setRepo(repo);
+    std::string s("humlepungens buffer");
+    ByteBuffer bb(s.c_str(), s.size());
+
+    doc->setValue(doc->getField("intattr"), IntFieldValue(50));
+    doc->setValue(doc->getField("rawattr"), RawFieldValue("readable hei der", 7));
+    doc->setValue(doc->getField("floatattr"), FloatFieldValue(3.56));
+    doc->setValue(doc->getField("stringattr"), StringFieldValue("tjo hei"));
+
+    doc->setValue(doc->getField("doubleattr"), DoubleFieldValue(17.78623142376453));
+    doc->setValue(doc->getField("longattr"), LongFieldValue(346234765345239657LL));
+    doc->setValue(doc->getField("byteattr"), ByteFieldValue('J'));
+
+    ArrayFieldValue val(doc->getField("rawarrayattr").getDataType());
+    RawFieldValue rawVal("readable hei", 3);
+    val.add(rawVal);
+    RawFieldValue rawVal2("readable hallo", 5);
+    val.add(rawVal2);
+    RawFieldValue rawVal3("readable hei der", 7);
+    val.add(rawVal3);
+    doc->setValue(doc->getField("rawarrayattr"), val);
+
+    Document::UP doc2(new Document(*type, DocumentId(
+                            "doc:crawler/http://www.ntnu.no/2")));
+    doc2->setValue(doc2->getField("stringattr"), StringFieldValue("tjo hei paa du"));
+    doc->setValue(doc->getField("docfield"), *doc2);
+
+    return doc;
+}
+
+DocumentUpdate::UP
+createTestDocumentUpdate(const DocumentTypeRepo& repo)
+{
+    const DocumentType* type(repo.getDocumentType("testdoc"));
+    DocumentId id("doc:crawler/http://www.ntnu.no/");
+
+    DocumentUpdate::UP up(new DocumentUpdate(*type, id));
+    up->addUpdate(FieldUpdate(type->getField("intattr"))
+		  .addUpdate(AssignValueUpdate(IntFieldValue(7))));
+    up->addUpdate(FieldUpdate(type->getField("stringattr"))
+		  .addUpdate(AssignValueUpdate(
+                                  StringFieldValue("New value"))));
+    up->addUpdate(FieldUpdate(type->getField("arrayattr"))
+		  .addUpdate(AddValueUpdate(IntFieldValue(123)))
+		  .addUpdate(AddValueUpdate(IntFieldValue(456))));
+    up->addUpdate(FieldUpdate(type->getField("arrayattr"))
+		  .addUpdate(RemoveValueUpdate(IntFieldValue(123)))
+		  .addUpdate(RemoveValueUpdate(IntFieldValue(456)))
+		  .addUpdate(RemoveValueUpdate(IntFieldValue(789))));
+    return up;
+}
+
+} // anonymous ns
+
+void TestXml::testSimpleUsage()
+{
+    DocumentTypeRepo repo(readDocumenttypesConfig("data/defaultdoctypes.cfg"));
+    Document::UP doc1(createTestDocument(repo));
+    doc1->setValue(doc1->getField("stringattr"), StringFieldValue("tjohei���"));
+
+    std::string expected =
+        "<document documenttype=\"testdoc\" documentid=\"doc:crawler/http://www.ntnu.no/\">\n"
+        "  <doubleattr>17.7862</doubleattr>\n"
+        "  <intattr>50</intattr>\n"
+        "  <floatattr>3.56</floatattr>\n"
+        "  <longattr>346234765345239657</longattr>\n"
+        "  <byteattr>74</byteattr>\n"
+        "  <rawarrayattr>\n"
+        "    <item binaryencoding=\"base64\">cmVh</item>\n"
+        "    <item binaryencoding=\"base64\">cmVhZGE=</item>\n"
+        "    <item binaryencoding=\"base64\">cmVhZGFibA==</item>\n"
+        "  </rawarrayattr>\n"
+        "  <rawattr binaryencoding=\"base64\">cmVhZGFibA==</rawattr>\n"
+        "  <stringattr>tjohei���</stringattr>\n"
+        "  <docfield>\n"
+        "    <document documenttype=\"testdoc\" documentid=\"doc:crawler/http://www.ntnu.no/2\">\n"
+        "      <stringattr>tjo hei paa du</stringattr>\n"
+        "    </document>\n"
+        "  </docfield>\n"
+        "  <content type=\"contenttype\" encoding=\"encoding\" language=\"language\">humlepungens buffer</content>\n"
+        "</document>";
+}
+
+void TestXml::testDocumentUpdate()
+{
+    DocumentTypeRepo repo(readDocumenttypesConfig("data/defaultdoctypes.cfg"));
+    DocumentUpdate::UP up1(createTestDocumentUpdate(repo));
+
+    std::string expected =
+        "<document type=\"testdoc\" id=\"doc:crawler/http://www.ntnu.no/\">\n"
+        "  <alter field=\"intattr\">\n"
+        "    <assign>7</assign>\n"
+        "  </alter>\n"
+        "  <alter field=\"stringattr\">\n"
+        "    <assign>New value</assign>\n"
+        "  </alter>\n"
+        "  <alter field=\"arrayattr\">\n"
+        "    <add weight=\"1\">123</add>\n"
+        "    <add weight=\"1\">456</add>\n"
+        "  </alter>\n"
+        "  <alter field=\"arrayattr\">\n"
+        "    <remove>123</remove>\n"
+        "    <remove>456</remove>\n"
+        "    <remove>789</remove>\n"
+        "  </alter>\n"
+        "</document>";
+    CPPUNIT_ASSERT_EQUAL(expected, up1->toXml("  "));
+}
+
+} // document
diff --git a/document/src/tests/urltypetest.cpp b/document/src/tests/urltypetest.cpp
new file mode 100644
index 00000000000..0aa447b9e40
--- /dev/null
+++ b/document/src/tests/urltypetest.cpp
@@ -0,0 +1,54 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/document/datatype/urldatatype.h>
+#include <cppunit/extensions/HelperMacros.h>
+
+namespace document {
+
+class UrlTypeTest : public CppUnit::TestFixture {
+public:
+    void requireThatNameIsCorrect();
+    void requireThatExpectedFieldsAreThere();
+
+    CPPUNIT_TEST_SUITE(UrlTypeTest);
+    CPPUNIT_TEST(requireThatNameIsCorrect);
+    CPPUNIT_TEST(requireThatExpectedFieldsAreThere);
+    CPPUNIT_TEST_SUITE_END();
+};
+
+CPPUNIT_TEST_SUITE_REGISTRATION(UrlTypeTest);
+
+void
+UrlTypeTest::requireThatNameIsCorrect()
+{
+    const StructDataType &type = UrlDataType::getInstance();
+    CPPUNIT_ASSERT_EQUAL(vespalib::string("url"), type.getName());
+}
+
+void
+UrlTypeTest::requireThatExpectedFieldsAreThere()
+{
+    const StructDataType &type = UrlDataType::getInstance();
+    Field field = type.getField("all");
+    CPPUNIT_ASSERT_EQUAL(*DataType::STRING, field.getDataType());
+
+    field = type.getField("scheme");
+    CPPUNIT_ASSERT_EQUAL(*DataType::STRING, field.getDataType());
+
+    field = type.getField("host");
+    CPPUNIT_ASSERT_EQUAL(*DataType::STRING, field.getDataType());
+
+    field = type.getField("port");
+    CPPUNIT_ASSERT_EQUAL(*DataType::STRING, field.getDataType());
+
+    field = type.getField("path");
+    CPPUNIT_ASSERT_EQUAL(*DataType::STRING, field.getDataType());
+
+    field = type.getField("query");
+    CPPUNIT_ASSERT_EQUAL(*DataType::STRING, field.getDataType());
+
+    field = type.getField("fragment");
+    CPPUNIT_ASSERT_EQUAL(*DataType::STRING, field.getDataType());
+}
+
+} // document
diff --git a/document/src/tests/vespaxml/fieldpathupdates.xml b/document/src/tests/vespaxml/fieldpathupdates.xml
new file mode 100755
index 00000000000..c775509f070
--- /dev/null
+++ b/document/src/tests/vespaxml/fieldpathupdates.xml
@@ -0,0 +1,54 @@
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<vespafeed>
+    <update documenttype="news" documentid="doc:test:http://www.ntnu.no/">
+        <assign fieldpath="url">assignUrl</assign>
+        <assign fieldpath="title">assignTitle</assign>
+        <assign fieldpath="last_downloaded">1</assign>
+        <assign fieldpath="value_long">2</assign>
+        <assign fieldpath="value_content">assignContent</assign>
+        <assign fieldpath="stringarr">
+            <item>assignString1</item>
+            <item>assignString2</item>
+        </assign>
+        <assign fieldpath="intarr">
+            <item>3</item>
+            <item>4</item>
+        </assign>
+        <assign fieldpath="longarr">
+            <item>5</item>
+            <item>6</item>
+        </assign>
+        <assign fieldpath="bytearr">
+            <item>7</item>
+            <item>8</item>
+        </assign>
+        <assign fieldpath="floatarr">
+            <item>9</item>
+            <item>10</item>
+        </assign>
+        <assign fieldpath="weightedsetint">
+            <item weight="11">11</item>
+            <item weight="12">12</item>
+        </assign>
+        <assign fieldpath="weightedsetstring">
+            <item weight="13">assign13</item>
+            <item weight="14">assign14</item>
+        </assign>
+
+        <add fieldpath="stringarr">
+            <item>addString1</item>
+            <item>addString2</item>
+        </add>
+        <add fieldpath="longarr">
+            <item>5</item>
+        </add>
+
+        <assign fieldpath="weightedsetint{13}">13</assign>
+        <assign fieldpath="weightedsetint{14}">14</assign>
+        <assign fieldpath="weightedsetstring{add13}">1</assign>
+        <assign fieldpath="weightedsetstring{assign13}">130</assign>
+
+        <remove fieldpath="weightedsetstring{assign14}"/>
+        <remove fieldpath="bytearr"/>
+    </update>
+</vespafeed>
diff --git a/document/src/tests/vespaxml/test1.expected.xml b/document/src/tests/vespaxml/test1.expected.xml
new file mode 100644
index 00000000000..a369cfff4a6
--- /dev/null
+++ b/document/src/tests/vespaxml/test1.expected.xml
@@ -0,0 +1,5 @@
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<document documenttype="news" documentid="doc:crawler:http://www.ntnu.no/">
+  <title>TestTitle</title>
+  <last_downloaded>100</last_downloaded>
+</document>
diff --git a/document/src/tests/vespaxml/test1.xml b/document/src/tests/vespaxml/test1.xml
new file mode 100644
index 00000000000..1c155182b14
--- /dev/null
+++ b/document/src/tests/vespaxml/test1.xml
@@ -0,0 +1,9 @@
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<vespafeed>
+  <vespaadd idprefix="doc:crawler:">
+    <document documenttype="news" documentid="http://www.ntnu.no/">
+       <title>TestTitle</title>
+       <last_downloaded>100</last_downloaded>
+    </document>
+  </vespaadd>
+</vespafeed>
diff --git a/document/src/tests/vespaxml/test10.xml b/document/src/tests/vespaxml/test10.xml
new file mode 100644
index 00000000000..b310f14da1c
--- /dev/null
+++ b/document/src/tests/vespaxml/test10.xml
@@ -0,0 +1,9 @@
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<vespafeed>
+  <vespaadd>
+    <document type="news" id="doc:crawler/http://www.ntnu.no/">
+       <title>Test<Title</title>
+       <last_downloaded>hundred</last_downloaded>
+    </document>
+  </vespaadd>
+</vespafeed>
diff --git a/document/src/tests/vespaxml/test11.xml b/document/src/tests/vespaxml/test11.xml
new file mode 100644
index 00000000000..d8cdb32697e
--- /dev/null
+++ b/document/src/tests/vespaxml/test11.xml
@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="iso-8859-1"?>
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<vespafeed>
+  <vesparemove>
+    <document type="news" id="doc:crawler/http://www.ntnu���.no/">
+    </document>
+  </vesparemove>
+</vespafeed>
diff --git a/document/src/tests/vespaxml/test12.xml b/document/src/tests/vespaxml/test12.xml
new file mode 100644
index 00000000000..1f2f3375ade
--- /dev/null
+++ b/document/src/tests/vespaxml/test12.xml
@@ -0,0 +1,9 @@
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<vespafeed>
+  <vespaadd>
+    <document type="news" id="doc:crawler/http://www.ntnu.no/">
+       <title></title>
+       <last_downloaded></last_downloaded>
+    </document>
+  </vespaadd>
+</vespafeed>
diff --git a/document/src/tests/vespaxml/test13.xml b/document/src/tests/vespaxml/test13.xml
new file mode 100644
index 00000000000..8f12ea42c93
--- /dev/null
+++ b/document/src/tests/vespaxml/test13.xml
@@ -0,0 +1,7 @@
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<vespafeed>
+    <document type="news" id="doc:crawler:http://www.ntnu.no/">
+       <title>TestTitle</title>
+       <last_downloaded>100</last_downloaded>
+    </document>
+</vespa>
diff --git a/document/src/tests/vespaxml/test14.xml b/document/src/tests/vespaxml/test14.xml
new file mode 100644
index 00000000000..7e0eaa270ae
--- /dev/null
+++ b/document/src/tests/vespaxml/test14.xml
@@ -0,0 +1,8 @@
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<vespafeed>
+  <vespaadd>
+    <document type="news" id="doc:crawler/http://www.ntnu.no/">
+       <last_downloaded>123</last_downloaded>
+    </doc>
+  </vespaadd>
+</vespafeed>
diff --git a/document/src/tests/vespaxml/test15.xml b/document/src/tests/vespaxml/test15.xml
new file mode 100644
index 00000000000..f4b25cd0aea
--- /dev/null
+++ b/document/src/tests/vespaxml/test15.xml
@@ -0,0 +1,8 @@
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<vespafeed>
+  <vespaadd>
+    <document type="news" id="doc:crawler/http://www.ntnu.no/">
+       <title>TestTitle</tit>
+    </document>
+  </vespaadd>
+</vespafeed>
diff --git a/document/src/tests/vespaxml/test16.xml b/document/src/tests/vespaxml/test16.xml
new file mode 100644
index 00000000000..727e1dc834b
--- /dev/null
+++ b/document/src/tests/vespaxml/test16.xml
@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="iso-8859-1"?>
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<vespafeed>
+  <vespaadd>
+    <documentid type="news" id="doc:crawler:http://www.ntnu.no/">
+    </documentid>
+  </vespaadd>
+</vespafeed>
diff --git a/document/src/tests/vespaxml/test17.xml b/document/src/tests/vespaxml/test17.xml
new file mode 100644
index 00000000000..c0046253c79
--- /dev/null
+++ b/document/src/tests/vespaxml/test17.xml
@@ -0,0 +1,9 @@
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+  <vespaadd>
+    <document type="news" id="doc:crawler/http://www.ntnu.no/1">
+    </document>
+  </vespaadd>
+  <vespaadd>
+    <document type="news" id="doc:crawler/http://www.ntnu.no/2">
+    </document>
+  </vespaadd>
diff --git a/document/src/tests/vespaxml/test18.xml b/document/src/tests/vespaxml/test18.xml
new file mode 100644
index 00000000000..c8798f8fc43
--- /dev/null
+++ b/document/src/tests/vespaxml/test18.xml
@@ -0,0 +1,9 @@
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<vespafeed>
+  <vespaadd>
+    <document type="article" id="doc:crawler/http://www.ntnu.no/">
+       <title>TestTitle</title>
+       <last_downloaded>100</last_downloaded>
+    </document>
+  </vespaadd>
+</vespafeed>
diff --git a/document/src/tests/vespaxml/test2.expected.xml b/document/src/tests/vespaxml/test2.expected.xml
new file mode 100644
index 00000000000..a369cfff4a6
--- /dev/null
+++ b/document/src/tests/vespaxml/test2.expected.xml
@@ -0,0 +1,5 @@
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<document documenttype="news" documentid="doc:crawler:http://www.ntnu.no/">
+  <title>TestTitle</title>
+  <last_downloaded>100</last_downloaded>
+</document>
diff --git a/document/src/tests/vespaxml/test2.xml b/document/src/tests/vespaxml/test2.xml
new file mode 100644
index 00000000000..9dae17e830f
--- /dev/null
+++ b/document/src/tests/vespaxml/test2.xml
@@ -0,0 +1,7 @@
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<vespafeed>
+    <document documenttype="news" documentid="doc:crawler:http://www.ntnu.no/">
+       <title>TestTitle</title>
+       <last_downloaded>100</last_downloaded>
+    </document>
+</vespafeed>
diff --git a/document/src/tests/vespaxml/test20.xml b/document/src/tests/vespaxml/test20.xml
new file mode 100644
index 00000000000..867c4829c36
--- /dev/null
+++ b/document/src/tests/vespaxml/test20.xml
@@ -0,0 +1,8 @@
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<vespafeed>
+  <endoffeed>
+     <name>default</name>
+     <generations>10</generations>
+     <increment>11</increment>
+  </endoffeed>
+</vespafeed>
diff --git a/document/src/tests/vespaxml/test21.xml b/document/src/tests/vespaxml/test21.xml
new file mode 100644
index 00000000000..daad994b84b
--- /dev/null
+++ b/document/src/tests/vespaxml/test21.xml
@@ -0,0 +1,9 @@
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<vespafeed>
+  <vespaadd>
+    <document type="news" id="doc:crawler/http://www.ntnu.no/">
+       <title>TestTitle</title>
+       <last_downloaded>21474836480</last_downloaded>
+    </document>
+  </vespaadd>
+</vespafeed>
diff --git a/document/src/tests/vespaxml/test22.xml b/document/src/tests/vespaxml/test22.xml
new file mode 100644
index 00000000000..ec20024693d
--- /dev/null
+++ b/document/src/tests/vespaxml/test22.xml
@@ -0,0 +1,9 @@
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<vespafeed>
+  <vespaadd>
+    <document type="news" id="doc:crawler:http://www.ntnu.no/">
+       <title>TestTitle</title>
+       <value_long>9223372036854775807</value_long>
+    </document>
+  </vespaadd>
+</vespafeed>
diff --git a/document/src/tests/vespaxml/test23.xml b/document/src/tests/vespaxml/test23.xml
new file mode 100644
index 00000000000..68afd6ec565
--- /dev/null
+++ b/document/src/tests/vespaxml/test23.xml
@@ -0,0 +1,9 @@
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<vespafeed>
+  <vespaadd>
+    <document type="news" id="doc:crawler:http://www.ntnu.no/">
+       <title>TestTitle</title>
+       <value_long>-9223372036854775807</value_long>
+    </document>
+  </vespaadd>
+</vespafeed>
diff --git a/document/src/tests/vespaxml/test24.xml b/document/src/tests/vespaxml/test24.xml
new file mode 100644
index 00000000000..b386bb2c092
--- /dev/null
+++ b/document/src/tests/vespaxml/test24.xml
@@ -0,0 +1,9 @@
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<vespafeed>
+  <vespaadd>
+    <document type="news" id="doc:crawler:http://www.ntnu.no/">
+       <title>TestTitle</title>
+       <value_long>18446744073709551615</value_long>
+    </document>
+  </vespaadd>
+</vespafeed>
diff --git a/document/src/tests/vespaxml/test25.xml b/document/src/tests/vespaxml/test25.xml
new file mode 100644
index 00000000000..72a19e8da60
--- /dev/null
+++ b/document/src/tests/vespaxml/test25.xml
@@ -0,0 +1,9 @@
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<vespafeed>
+  <vespaadd>
+    <document type="news" id="doc:crawler/http://www.ntnu.no/">
+       <title>TestTitle</title>
+       <value_long>18446744073709551616</value_long>
+    </document>
+  </vespaadd>
+</vespafeed>
diff --git a/document/src/tests/vespaxml/test26.xml b/document/src/tests/vespaxml/test26.xml
new file mode 100644
index 00000000000..698e615c274
--- /dev/null
+++ b/document/src/tests/vespaxml/test26.xml
@@ -0,0 +1,9 @@
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<vespafeed>
+  <vespaadd>
+    <document type="news" id="doc:crawler:http://www.ntnu.no/">
+       <title>TestTitle</title>
+       <last_downloaded>0x123</last_downloaded>
+    </document>
+  </vespaadd>
+</vespafeed>
diff --git a/document/src/tests/vespaxml/test27.xml b/document/src/tests/vespaxml/test27.xml
new file mode 100644
index 00000000000..94c5f3c1d32
--- /dev/null
+++ b/document/src/tests/vespaxml/test27.xml
@@ -0,0 +1,9 @@
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<vespafeed>
+  <vespaadd>
+    <document type="news" id="doc:crawler/http://www.ntnu.no/">
+       <title>TestTitle</title>
+       <last_downloaded>0x123</last_downloaded>
+    </document>
+  </vespaadd>
+</vespafeed>
diff --git a/document/src/tests/vespaxml/test28.xml b/document/src/tests/vespaxml/test28.xml
new file mode 100644
index 00000000000..f1995d295bb
--- /dev/null
+++ b/document/src/tests/vespaxml/test28.xml
@@ -0,0 +1,10 @@
+<!-- Test content -->
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<vespafeed>
+  <vespaadd>
+    <document type="news" id="doc:crawler:http://www.ntnu.no/">
+       <title>TestTitleContent</title>
+        <value_content contenttype="text/html" encoding="utf-8" language="en"><![CDATA[<html><body><h1>This is the title</h1></body></html>]]></value_content>
+    </document>
+  </vespaadd>
+</vespafeed>
diff --git a/document/src/tests/vespaxml/test29.xml b/document/src/tests/vespaxml/test29.xml
new file mode 100644
index 00000000000..1f33b9f7555
--- /dev/null
+++ b/document/src/tests/vespaxml/test29.xml
@@ -0,0 +1,18 @@
+<!-- Test field attribute -->
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<vespafeed>
+  <vespaadd>
+    <document type="news">
+       <field name="uri">doc:crawler:http://www.ntnu.no/</field>
+       <field name="title">TestTitleContent</field>
+       <field name="value_content" contenttype="text/html" encoding="UTF-8" language="NO">This is content</field>
+       <last_downloaded>345</last_downloaded>
+    </document>
+    <document type="news">
+       <field name="uri">doc:crawler:http://www.ntnu.no/2</field>
+       <field name="title">TestTitleContent2</field>
+       <field name="value_content">This is content2</field>
+       <last_downloaded>345</last_downloaded>
+    </document>
+  </vespaadd>
+</vespafeed>
diff --git a/document/src/tests/vespaxml/test3.xml b/document/src/tests/vespaxml/test3.xml
new file mode 100644
index 00000000000..81d87679211
--- /dev/null
+++ b/document/src/tests/vespaxml/test3.xml
@@ -0,0 +1,9 @@
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<vespafeed>
+  <vesparemove idprefix="doc:crawler:">
+    <documentid type="news">
+       <uri>http://www.ntnu.no/</uri>
+       <last_downloaded></last_downloaded>
+    </documentid>
+  </vesparemove>
+</vespafeed>
diff --git a/document/src/tests/vespaxml/test30.xml b/document/src/tests/vespaxml/test30.xml
new file mode 100644
index 00000000000..6fbb6fae8ba
--- /dev/null
+++ b/document/src/tests/vespaxml/test30.xml
@@ -0,0 +1,15 @@
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<vespafeed xmlns:xsd="http://www.w3.org/2001/XMLSchema-datatypes"
+	   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+  <vespaadd>
+    <document type="news" id="doc:crawler:http://www.ntnu.no/">
+       <title xsi:type="xsd:hexBinary">E5AEB6E59BADE3808220044BE680AC</title>
+       <last_downloaded>100</last_downloaded>
+    </document>
+    <document type="news" id="doc:crawler:http://www.ntnu.org/">
+       <title xsi:type="xsd:base64Binary">5a625Zut44CCIARL5oCs</title>
+       <last_downloaded>100</last_downloaded>
+    </document>
+  </vespaadd>
+</vespafeed>
+
diff --git a/document/src/tests/vespaxml/test32.xml b/document/src/tests/vespaxml/test32.xml
new file mode 100644
index 00000000000..b2f7976c124
--- /dev/null
+++ b/document/src/tests/vespaxml/test32.xml
@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="iso-8859-1"?>
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<vespafeed>
+  <vesparemove>
+    <documentid id="doc:crawler:http://www.ntnu���.no/"/>
+  </vesparemove>
+</vespafeed>
diff --git a/document/src/tests/vespaxml/test33.xml b/document/src/tests/vespaxml/test33.xml
new file mode 100644
index 00000000000..264be7f3e02
--- /dev/null
+++ b/document/src/tests/vespaxml/test33.xml
@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="iso-8859-1"?>
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<vespafeed>
+  <vesparemove>
+    <documentid>
+      <uri>http://www.ntnu.no/</uri>
+    </documentid>
+  </vesparemove>
+</vespafeed>
diff --git a/document/src/tests/vespaxml/test34.xml b/document/src/tests/vespaxml/test34.xml
new file mode 100644
index 00000000000..a3c80415899
--- /dev/null
+++ b/document/src/tests/vespaxml/test34.xml
@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="iso-8859-1"?>
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<vespafeed>
+  <vesparemove>
+    <documentid id="http://www.ntnu.no/"/>
+  </vesparemove>
+</vespafeed>
diff --git a/document/src/tests/vespaxml/test35.xml b/document/src/tests/vespaxml/test35.xml
new file mode 100644
index 00000000000..e02a755cc99
--- /dev/null
+++ b/document/src/tests/vespaxml/test35.xml
@@ -0,0 +1,9 @@
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<vespafeed>
+  <vespaadd>
+    <document type="news" id="doc:crawler:http://www.ntnu.no/">
+       <title binaryencoding="base64">VmVzcGEgcnVsZXM=</title>
+       <last_downloaded>100</last_downloaded>
+    </document>
+  </vespaadd>
+</vespafeed>
diff --git a/document/src/tests/vespaxml/test36.xml b/document/src/tests/vespaxml/test36.xml
new file mode 100644
index 00000000000..3eed0c4cb2a
--- /dev/null
+++ b/document/src/tests/vespaxml/test36.xml
@@ -0,0 +1,43 @@
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<vespafeed>
+  <vespaadd idprefix="doc:crawler:">
+    <document type="news" id="http://www.ntnu.no/1">
+       <title>TestTitle1</title>
+       <last_downloaded>100</last_downloaded>
+       <stringarr>
+         <item>one</item>
+         <item>two</item>
+         <item>three</item>
+         <item>four</item>
+         <item>five</item>
+       </stringarr>
+       <intarr>
+         <item>1</item>
+         <item>2</item>
+         <item>3</item>
+         <item>4</item>
+         <item>5</item>
+       </intarr>
+    </document>
+    <document type="news" id="http://www.ntnu.no/2">
+       <title>TestTitle2</title>
+       <intarr>
+         <item>o1</item>
+       </intarr>
+    </document>
+    <document type="news" id="http://www.ntnu.no/3">
+       <title>TestTitle3</title>
+       <intarr>
+         <tem>1</tem>
+       </intarr>
+    </document>
+    <document type="news" id="http://www.ntnu.no/4">
+       <title>TestTitle4</title>
+       <intarr></intarr>
+    </document>
+    <document type="news" id="http://www.ntnu.no/5">
+       <title>TestTitle5</title>
+       <intarr>1</intarr>
+    </document>
+  </vespaadd>
+</vespafeed>
diff --git a/document/src/tests/vespaxml/test37.xml b/document/src/tests/vespaxml/test37.xml
new file mode 100644
index 00000000000..1c2f6e01006
--- /dev/null
+++ b/document/src/tests/vespaxml/test37.xml
@@ -0,0 +1,23 @@
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<vespafeed>
+  <vespaadd idprefix="doc:crawler:">
+    <document type="news" id="http://www.ntnu.no/1">
+       <title>TestTitle1</title>
+       <last_downloaded>100</last_downloaded>
+       <weightedsetstring>
+         <item>one</item>
+         <item>two</item>
+         <item>three</item>
+         <item>four</item>
+         <item>five</item>
+       </weightedsetstring>
+       <weightedsetint>
+         <item>1</item>
+         <item>2</item>
+         <item>3</item>
+         <item>4</item>
+         <item>5</item>
+       </weightedsetint>
+    </document>
+  </vespaadd>
+</vespafeed>
diff --git a/document/src/tests/vespaxml/test4.xml b/document/src/tests/vespaxml/test4.xml
new file mode 100644
index 00000000000..7c0c3682cd3
--- /dev/null
+++ b/document/src/tests/vespaxml/test4.xml
@@ -0,0 +1,23 @@
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<vespafeed>
+
+  <vespaadd idprefix="doc:crawler:">
+    <document type="news" id="http://www.uio.no/">
+       <title>TestTitle</title>
+       <last_downloaded>100</last_downloaded>
+    </document>
+
+    <document type="news" id="http://www.dagbladet.no/">
+       <title>Title2</title>
+       <last_downloaded>100</last_downloaded>
+    </document>
+  </vespaadd>
+
+  <vespaadd>
+    <document type="news" id="doc:crawler2:http://www.vg.no/">
+       <title>Title2</title>
+       <last_downloaded>100</last_downloaded>
+    </document>
+  </vespaadd>
+
+</vespafeed>
diff --git a/document/src/tests/vespaxml/test40.xml b/document/src/tests/vespaxml/test40.xml
new file mode 100644
index 00000000000..ecfb745415a
--- /dev/null
+++ b/document/src/tests/vespaxml/test40.xml
@@ -0,0 +1,6 @@
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<vespafeed>
+    <update documenttype="news" documentid="doc:crawler:http://www.ntnu.no/">
+       <assign field="title">TestTitle</assign>
+    </update>
+</vespafeed>
diff --git a/document/src/tests/vespaxml/test41.xml b/document/src/tests/vespaxml/test41.xml
new file mode 100644
index 00000000000..052f4076025
--- /dev/null
+++ b/document/src/tests/vespaxml/test41.xml
@@ -0,0 +1,22 @@
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<vespafeed>
+  <update documenttype="news" documentid="doc:crawler:http://www.ntnu.no/">
+    <assign field="stringarr">
+      <item>First</item>
+      <item>Second</item>
+    </assign>
+    <add field="stringarr">
+      <item>Third</item>
+      <item>Fourth</item>
+    </add>
+    <remove field="stringarr">
+      <item>Fifth</item>
+      <item>Sixth</item>
+    </remove>
+    <assign field="intarr">
+      <item>100</item>
+      <item>200</item>
+    </assign>
+    <assign field="intarr"></assign>
+  </update>
+</vespafeed>
diff --git a/document/src/tests/vespaxml/test42.xml b/document/src/tests/vespaxml/test42.xml
new file mode 100644
index 00000000000..da3e94410df
--- /dev/null
+++ b/document/src/tests/vespaxml/test42.xml
@@ -0,0 +1,22 @@
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<vespafeed>
+    <update documenttype="news" documentid="doc:crawler:http://www.ntnu.no/">
+        <assign field="weightedsetstring">
+	    <item weight="1">First</item>
+            <item weight="2">Second</item>
+        </assign>
+        <add field="weightedsetstring">
+            <item weight="3">Third</item>
+            <item weight="4">Fourth</item>
+        </add>
+        <remove field="weightedsetstring">
+            <item>Fifth</item>
+	    <item>Sixth</item>
+	</remove>
+        <assign field="weightedsetint">
+	    <item weight="1">100</item>
+            <item weight="2">200</item>
+        </assign>
+        <assign field="weightedsetint"></assign>
+    </update>
+</vespafeed>
diff --git a/document/src/tests/vespaxml/test43.xml b/document/src/tests/vespaxml/test43.xml
new file mode 100644
index 00000000000..16944dde9fa
--- /dev/null
+++ b/document/src/tests/vespaxml/test43.xml
@@ -0,0 +1,32 @@
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<vespafeed>
+    <document documenttype="news" documentid="doc:crawler:http://www.ntnu.no/">
+        <value_long>1008</value_long>
+	<weightedsetstring>
+	    <item weight="1">First</item>
+            <item weight="2">Second</item>
+        </weightedsetstring>
+    </document>
+    <update documenttype="news" documentid="doc:crawler:http://www.ntnu.no/">
+        <increment field="value_long" by="1" />
+        <decrement field="value_long" by="2" />
+	<divide field="value_long" by="3" />
+	<multiply field="value_long" by="4" />
+    </update>	
+    <update documenttype="news" documentid="doc:crawler:http://www.ntnu.no/">
+	<alter field="value_long">
+	    <increment by="5" />
+	    <decrement by="6" />
+	</alter>
+	<alter field="value_long">
+	    <divide by="7" />
+	    <multiply by="8" />
+	</alter>
+    </update>	
+    <update documenttype="news" documentid="doc:crawler:http://www.ntnu.no/">
+	<increment field="weightedsetstring" by="9">
+	    <key>First</key>
+	    <key>Second</key>
+	</increment>
+    </update>
+</vespafeed>
diff --git a/document/src/tests/vespaxml/test45.xml b/document/src/tests/vespaxml/test45.xml
new file mode 100644
index 00000000000..eb13e59a0da
--- /dev/null
+++ b/document/src/tests/vespaxml/test45.xml
@@ -0,0 +1,6 @@
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<vespafeed>
+    <update documenttype="news" documentid="doc:crawler:http://www.ntnu.no/">
+        <remove field="floatarr" />
+    </update>
+</vespafeed>
diff --git a/document/src/tests/vespaxml/test46.xml b/document/src/tests/vespaxml/test46.xml
new file mode 100644
index 00000000000..75d69f3a8b5
--- /dev/null
+++ b/document/src/tests/vespaxml/test46.xml
@@ -0,0 +1,10 @@
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<vespafeed>
+    <document documenttype="news" documentid="doc:crawler:http://www.blargh.example.com/" lastmodified="98798787">
+        <value_long>1008</value_long>
+	<weightedsetstring>
+	    <item weight="1">First</item>
+            <item weight="2">Second</item>
+        </weightedsetstring>
+    </document>
+</vespafeed>
diff --git a/document/src/tests/vespaxml/test47.xml b/document/src/tests/vespaxml/test47.xml
new file mode 100644
index 00000000000..5944158c7c6
--- /dev/null
+++ b/document/src/tests/vespaxml/test47.xml
@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<vespafeed>
+    <!-- VERY BAD: -->
+    <vespaadd>
+        <documentid type="news">
+            <uri>doc:this:is:very:bad</uri>
+        </documentid>
+    </vespaadd>
+</vespafeed>
diff --git a/document/src/tests/vespaxml/test48.xml b/document/src/tests/vespaxml/test48.xml
new file mode 100644
index 00000000000..a9bdd94d098
--- /dev/null
+++ b/document/src/tests/vespaxml/test48.xml
@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<vespafeed>
+    <!-- VERY BAD: -->
+    <documentid type="news" id="doc:this:is:even:worse" />
+</vespafeed>
diff --git a/document/src/tests/vespaxml/test49.xml b/document/src/tests/vespaxml/test49.xml
new file mode 100644
index 00000000000..dc8a67c64bb
--- /dev/null
+++ b/document/src/tests/vespaxml/test49.xml
@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<vespafeed>
+    <!-- VERY BAD: -->
+    <documentid type="news">
+        <uri>doc:this:is:very:very:ugly</uri>
+    </documentid>
+</vespafeed>
diff --git a/document/src/tests/vespaxml/test5.expected.xml b/document/src/tests/vespaxml/test5.expected.xml
new file mode 100644
index 00000000000..164799749c8
--- /dev/null
+++ b/document/src/tests/vespaxml/test5.expected.xml
@@ -0,0 +1,3 @@
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<remove documentid="doc:crawler:http://www.ntnu.no/"/>
+
diff --git a/document/src/tests/vespaxml/test5.xml b/document/src/tests/vespaxml/test5.xml
new file mode 100644
index 00000000000..a6a6f8f0f9f
--- /dev/null
+++ b/document/src/tests/vespaxml/test5.xml
@@ -0,0 +1,6 @@
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<vespafeed>
+  <vesparemove>
+    <documentid documentid="doc:crawler:http://www.ntnu.no/"/>
+  </vesparemove>
+</vespafeed>
diff --git a/document/src/tests/vespaxml/test50.xml b/document/src/tests/vespaxml/test50.xml
new file mode 100644
index 00000000000..0909bcf2795
--- /dev/null
+++ b/document/src/tests/vespaxml/test50.xml
@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<vespafeed>
+    <!-- OK: -->
+    <vesparemove>
+        <documentid type="news" id="doc:this:an:ok:removal"/>
+    </vesparemove>
+</vespafeed>
diff --git a/document/src/tests/vespaxml/test51.xml b/document/src/tests/vespaxml/test51.xml
new file mode 100644
index 00000000000..698a305d33c
--- /dev/null
+++ b/document/src/tests/vespaxml/test51.xml
@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<vespafeed>
+    <!-- OK: -->
+    <remove documentid="doc:this:is:also:an:ok:removal"/>
+</vespafeed>
diff --git a/document/src/tests/vespaxml/test52.xml b/document/src/tests/vespaxml/test52.xml
new file mode 100644
index 00000000000..d569f5ada1b
--- /dev/null
+++ b/document/src/tests/vespaxml/test52.xml
@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<vespafeed>
+    <!-- OK: -->
+    <vespaadd>
+        <document documentid="doc:blah:blah:blah" documenttype="news">
+            <value_long>2345</value_long>
+        </document>
+    </vespaadd>
+</vespafeed>
diff --git a/document/src/tests/vespaxml/test53.xml b/document/src/tests/vespaxml/test53.xml
new file mode 100644
index 00000000000..28c2161c956
--- /dev/null
+++ b/document/src/tests/vespaxml/test53.xml
@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<vespafeed>
+    <!-- HALF BAD, BUT NOT ALLOWED: -->
+    <vespaadd>
+        <document documentid="doc:blah:blah:blah" documenttype="news">
+            <value_long>2345</value_long>
+        </document>
+        <documentid type="news" id="doc:half:bad:add"/>
+    </vespaadd>
+</vespafeed>
diff --git a/document/src/tests/vespaxml/test54.xml b/document/src/tests/vespaxml/test54.xml
new file mode 100644
index 00000000000..90f41b6d32d
--- /dev/null
+++ b/document/src/tests/vespaxml/test54.xml
@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<vespafeed>
+    <!-- VERY BAD: -->
+    <vesparemove>
+        <document documentid="doc:bluh:bluh:bluh" documenttype="news">
+            <value_long>45</value_long>
+        </document>
+    </vesparemove>
+</vespafeed>
diff --git a/document/src/tests/vespaxml/test55.xml b/document/src/tests/vespaxml/test55.xml
new file mode 100644
index 00000000000..8fcd4a31d1f
--- /dev/null
+++ b/document/src/tests/vespaxml/test55.xml
@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<vespafeed>
+    <!-- HALF BAD, BUT NOT ALLOWED: -->
+    <vesparemove>
+        <documentid type="news" id="doc:this:remove:is:half:bad"/>
+        <document documentid="doc:bluh:bluh:bluh" documenttype="news">
+            <value_long>45</value_long>
+        </document>
+    </vesparemove>
+</vespafeed>
diff --git a/document/src/tests/vespaxml/test56.xml b/document/src/tests/vespaxml/test56.xml
new file mode 100644
index 00000000000..489ff494fc0
--- /dev/null
+++ b/document/src/tests/vespaxml/test56.xml
@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<vespafeed>
+    <!-- VERY BAD: -->
+    <vesparemove>
+        <document documentid="doc:bluh:bluh:bluh" documenttype="news" />
+    </vesparemove>
+</vespafeed>
diff --git a/document/src/tests/vespaxml/test57.xml b/document/src/tests/vespaxml/test57.xml
new file mode 100644
index 00000000000..6653b0260af
--- /dev/null
+++ b/document/src/tests/vespaxml/test57.xml
@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<vespafeed>
+    <!-- OK: -->
+    <document documentid="doc:blih:blih:blih" documenttype="news">
+        <value_long>235</value_long>
+    </document>
+</vespafeed>
diff --git a/document/src/tests/vespaxml/test58.xml b/document/src/tests/vespaxml/test58.xml
new file mode 100644
index 00000000000..2577f3a0e26
--- /dev/null
+++ b/document/src/tests/vespaxml/test58.xml
@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<vespafeed>
+
+  <document type="news" id="doc:music:http://music.yahoo.com">
+    <url>http://music.yahoo.com</url>
+    <stringarr>
+   <item>yahoo<item>
+   <item>hello</item>
+   </stringarr>
+      </document>
+
+
+
+</vespafeed>
diff --git a/document/src/tests/vespaxml/test59.xml b/document/src/tests/vespaxml/test59.xml
new file mode 100644
index 00000000000..970f5232e7b
--- /dev/null
+++ b/document/src/tests/vespaxml/test59.xml
@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<vespafeed>
+    <update documentid="doc:blih:blih:blih" documenttype="news">
+        <increment field="title" by="3549" />
+    </update>
+</vespafeed>
diff --git a/document/src/tests/vespaxml/test6.xml b/document/src/tests/vespaxml/test6.xml
new file mode 100644
index 00000000000..c67352bbb22
--- /dev/null
+++ b/document/src/tests/vespaxml/test6.xml
@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="iso-8859-1"?>
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<vespafeed>
+  <vesparemove>
+    <documentid type="news" id="doc:crawler:http://www.ntnu���.no/"/>
+  </vesparemove>
+</vespafeed>
diff --git a/document/src/tests/vespaxml/test7.xml b/document/src/tests/vespaxml/test7.xml
new file mode 100644
index 00000000000..2c346a5b5cf
--- /dev/null
+++ b/document/src/tests/vespaxml/test7.xml
@@ -0,0 +1,9 @@
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<vespafeed idprefix="doc:crawler:">
+  <vespaadd>
+    <document type="news" id="http://www.ntnu.no/">
+       <title>TestTitle</title>
+       <last_downloaded>100.5</last_downloaded>
+    </document>
+  </vespaadd>
+</vespafeed>
diff --git a/document/src/tests/vespaxml/test8.xml b/document/src/tests/vespaxml/test8.xml
new file mode 100644
index 00000000000..cb9617fefd8
--- /dev/null
+++ b/document/src/tests/vespaxml/test8.xml
@@ -0,0 +1,9 @@
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<vespafeed>
+  <vespaadd>
+    <document type="news" id="doc:crawler:http://www.ntnu.no/">
+       <title>TestTitle</title>
+       <last_downloaded>hundred</last_downloaded>
+    </document>
+  </vespaadd>
+</vespafeed>
diff --git a/document/src/tests/vespaxml/test9.xml b/document/src/tests/vespaxml/test9.xml
new file mode 100644
index 00000000000..8b86da414b8
--- /dev/null
+++ b/document/src/tests/vespaxml/test9.xml
@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<vespafeed>
+  <vesparemove>
+    <documentid id="doc:crawler/http://www.ntnu���.no/"/>
+  </vesparemove>
+</vespafeed>
diff --git a/document/src/tests/vespaxml/test_arraystruct.xml b/document/src/tests/vespaxml/test_arraystruct.xml
new file mode 100644
index 00000000000..5eeae4aab2d
--- /dev/null
+++ b/document/src/tests/vespaxml/test_arraystruct.xml
@@ -0,0 +1,19 @@
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<vespafeed>
+  <document documenttype="news" documentid="doc:test:struct">
+    <mystructarr>
+      <item>
+        <intval>36</intval>
+        <stringval>test</stringval>
+      </item>
+      <item>
+        <intval>39</intval>
+        <stringval>test2</stringval>
+      </item>
+      <item>
+        <intval>100</intval>
+        <stringval>cooool</stringval>
+      </item>
+    </mystructarr>
+  </document>
+</vespafeed>
diff --git a/document/src/tests/vespaxml/test_doc5.xml b/document/src/tests/vespaxml/test_doc5.xml
new file mode 100644
index 00000000000..fa6d9c4726e
--- /dev/null
+++ b/document/src/tests/vespaxml/test_doc5.xml
@@ -0,0 +1,8 @@
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<vespafeed>
+  <endoffeed>
+     <name>default</name>
+     <generation>10</generation>
+     <increment>20</increment>
+  </endoffeed>
+</vespafeed>
diff --git a/document/src/tests/vespaxml/test_doc6.xml b/document/src/tests/vespaxml/test_doc6.xml
new file mode 100644
index 00000000000..2f32701d88e
--- /dev/null
+++ b/document/src/tests/vespaxml/test_doc6.xml
@@ -0,0 +1,6 @@
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<vespafeed>
+  <endoffeed>
+     <name>default</name>
+  </endoffeed>
+</vespafeed>
diff --git a/document/src/tests/vespaxml/test_doc8.xml b/document/src/tests/vespaxml/test_doc8.xml
new file mode 100644
index 00000000000..d636e7baa96
--- /dev/null
+++ b/document/src/tests/vespaxml/test_doc8.xml
@@ -0,0 +1,9 @@
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<vespafeed>
+  <vespaadd>
+    <document type="news" version="13" id="http://www.ntnu.no/">
+       <title>TestTitle</title>
+       <last_downloaded>100</last_downloaded>
+    </document>
+  </vespaadd>
+</vespafeed>
diff --git a/document/src/tests/vespaxml/test_externalentity.xml b/document/src/tests/vespaxml/test_externalentity.xml
new file mode 100644
index 00000000000..1d87319b106
--- /dev/null
+++ b/document/src/tests/vespaxml/test_externalentity.xml
@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<!DOCTYPE vespafeed [<!ENTITY xxe SYSTEM "xxe.txt">]>
+<vespafeed>
+  <vespaadd idprefix="doc:crawler:">
+    <document documenttype="news" documentid="http://www.ntnu.no/">
+       <title>&xxe;</title>
+       <last_downloaded>100</last_downloaded>
+    </document>
+  </vespaadd>
+</vespafeed>
diff --git a/document/src/tests/vespaxml/test_idprefix.xml b/document/src/tests/vespaxml/test_idprefix.xml
new file mode 100644
index 00000000000..e3a177d6606
--- /dev/null
+++ b/document/src/tests/vespaxml/test_idprefix.xml
@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<vespafeed>
+
+  <document type="news" id="http://music.yahoo.com/bobdylan/BestOf">
+    <title>Best of Bob Dylan</title>
+  </document>
+
+  <document type="news" id="http://music.yahoo.com/metallica/BestOf">
+    <title>Best of Metallica</title>
+  </document>
+
+</vespafeed>
diff --git a/document/src/tests/vespaxml/test_struct.xml b/document/src/tests/vespaxml/test_struct.xml
new file mode 100644
index 00000000000..c18c9263b4a
--- /dev/null
+++ b/document/src/tests/vespaxml/test_struct.xml
@@ -0,0 +1,9 @@
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<vespafeed>
+  <document documenttype="news" documentid="doc:test:struct">
+    <mystruct>
+      <intval>36</intval>
+      <stringval>test</stringval>
+    </mystruct>
+  </document>
+</vespafeed>
diff --git a/document/src/tests/vespaxml/test_update1.xml b/document/src/tests/vespaxml/test_update1.xml
new file mode 100644
index 00000000000..27c7d6214d3
--- /dev/null
+++ b/document/src/tests/vespaxml/test_update1.xml
@@ -0,0 +1,12 @@
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<vespafeed>
+  <vespaadd idprefix="doc:crawler:">
+    <document type="news" id="http://www.ntnu.no/">
+       <title>TestTitle</title>
+       <last_downloaded>100</last_downloaded>
+    </document>
+    <update documenttype="news" documentid="http://www.ntnu.no/">
+          <divide field="last_downloaded" by="0" />
+    </update>
+  </vespaadd>
+</vespafeed>
diff --git a/document/src/tests/vespaxml/vespaxmldoctype.cfg b/document/src/tests/vespaxml/vespaxmldoctype.cfg
new file mode 100644
index 00000000000..3a4aa280e0d
--- /dev/null
+++ b/document/src/tests/vespaxml/vespaxmldoctype.cfg
@@ -0,0 +1,133 @@
+enablecompression false
+datatype[12]
+datatype[0].id 1002
+datatype[0].arraytype[1]
+datatype[0].arraytype[0].datatype 2
+datatype[0].weightedsettype[0]
+datatype[0].structtype[0]
+datatype[0].documenttype[0]
+datatype[1].id 1000
+datatype[1].arraytype[1]
+datatype[1].arraytype[0].datatype 0
+datatype[1].weightedsettype[0]
+datatype[1].structtype[0]
+datatype[1].documenttype[0]
+datatype[2].id 1004
+datatype[2].arraytype[1]
+datatype[2].arraytype[0].datatype 4
+datatype[2].weightedsettype[0]
+datatype[2].structtype[0]
+datatype[2].documenttype[0]
+datatype[3].id 1016
+datatype[3].arraytype[1]
+datatype[3].arraytype[0].datatype 16
+datatype[3].weightedsettype[0]
+datatype[3].structtype[0]
+datatype[3].documenttype[0]
+datatype[4].id 1001
+datatype[4].arraytype[1]
+datatype[4].arraytype[0].datatype 1
+datatype[4].weightedsettype[0]
+datatype[4].structtype[0]
+datatype[4].documenttype[0]
+datatype[5].id 2001
+datatype[5].arraytype[0]
+datatype[5].weightedsettype[1]
+datatype[5].weightedsettype[0].datatype 0
+datatype[5].weightedsettype[0].createifnonexistant false
+datatype[5].weightedsettype[0].removeifzero false
+datatype[5].structtype[0]
+datatype[5].documenttype[0]
+datatype[6].id 2002
+datatype[6].arraytype[0]
+datatype[6].weightedsettype[1]
+datatype[6].weightedsettype[0].datatype 2
+datatype[6].weightedsettype[0].createifnonexistant false
+datatype[6].weightedsettype[0].removeifzero false
+datatype[6].structtype[0]
+datatype[6].documenttype[0]
+datatype[7].id 3000
+datatype[7].arraytype[0]
+datatype[7].weightedsettype[0]
+datatype[7].structtype[1]
+datatype[7].structtype[0].name news.mystruct
+datatype[7].structtype[0].version 0
+datatype[7].structtype[0].field[2]
+datatype[7].structtype[0].field[0].name intval
+datatype[7].structtype[0].field[0].id[0]
+datatype[7].structtype[0].field[0].datatype 0
+datatype[7].structtype[0].field[1].name stringval
+datatype[7].structtype[0].field[1].id[0]
+datatype[7].structtype[0].field[1].datatype 2
+datatype[7].documenttype[0]
+datatype[8].id 103000
+datatype[8].arraytype[1]
+datatype[8].arraytype[0].datatype 3000
+datatype[8].weightedsettype[0]
+datatype[8].structtype[0]
+datatype[8].documenttype[0]
+datatype[9].id 5000
+datatype[9].arraytype[0]
+datatype[9].weightedsettype[0]
+datatype[9].structtype[1]
+datatype[9].structtype[0].name news.header
+datatype[9].structtype[0].version 0
+datatype[9].structtype[0].field[13]
+datatype[9].structtype[0].field[0].name url
+datatype[9].structtype[0].field[0].id[0]
+datatype[9].structtype[0].field[0].datatype 10
+datatype[9].structtype[0].field[1].name title
+datatype[9].structtype[0].field[1].id[0]
+datatype[9].structtype[0].field[1].datatype 2
+datatype[9].structtype[0].field[2].name last_downloaded
+datatype[9].structtype[0].field[2].id[0]
+datatype[9].structtype[0].field[2].datatype 0
+datatype[9].structtype[0].field[3].name value_long
+datatype[9].structtype[0].field[3].id[0]
+datatype[9].structtype[0].field[3].datatype 4
+datatype[9].structtype[0].field[4].name stringarr
+datatype[9].structtype[0].field[4].id[0]
+datatype[9].structtype[0].field[4].datatype 1002
+datatype[9].structtype[0].field[5].name intarr
+datatype[9].structtype[0].field[5].id[0]
+datatype[9].structtype[0].field[5].datatype 1000
+datatype[9].structtype[0].field[6].name longarr
+datatype[9].structtype[0].field[6].id[0]
+datatype[9].structtype[0].field[6].datatype 1004
+datatype[9].structtype[0].field[7].name bytearr
+datatype[9].structtype[0].field[7].id[0]
+datatype[9].structtype[0].field[7].datatype 1016
+datatype[9].structtype[0].field[8].name floatarr
+datatype[9].structtype[0].field[8].id[0]
+datatype[9].structtype[0].field[8].datatype 1001
+datatype[9].structtype[0].field[9].name weightedsetint
+datatype[9].structtype[0].field[9].id[0]
+datatype[9].structtype[0].field[9].datatype 2001
+datatype[9].structtype[0].field[10].name weightedsetstring
+datatype[9].structtype[0].field[10].id[0]
+datatype[9].structtype[0].field[10].datatype 2002
+datatype[9].structtype[0].field[11].name mystruct
+datatype[9].structtype[0].field[11].id[0]
+datatype[9].structtype[0].field[11].datatype 3000
+datatype[9].structtype[0].field[12].name mystructarr
+datatype[9].structtype[0].field[12].id[0]
+datatype[9].structtype[0].field[12].datatype 103000
+datatype[9].documenttype[0]
+datatype[10].id 5001
+datatype[10].arraytype[0]
+datatype[10].weightedsettype[0]
+datatype[10].structtype[1]
+datatype[10].structtype[0].name news.body
+datatype[10].structtype[0].version 0
+datatype[10].structtype[0].field[0]
+datatype[10].documenttype[0]
+datatype[11].id 5002
+datatype[11].arraytype[0]
+datatype[11].weightedsettype[0]
+datatype[11].structtype[0]
+datatype[11].documenttype[1]
+datatype[11].documenttype[0].name news
+datatype[11].documenttype[0].version 0
+datatype[11].documenttype[0].inherits[0]
+datatype[11].documenttype[0].headerstruct 5000
+datatype[11].documenttype[0].bodystruct 5001
diff --git a/document/src/tests/vespaxml/xxe.txt b/document/src/tests/vespaxml/xxe.txt
new file mode 100644
index 00000000000..c8c1b8233cc
--- /dev/null
+++ b/document/src/tests/vespaxml/xxe.txt
@@ -0,0 +1 @@
+scary xxe exploit
+\ No newline at end of file
diff --git a/document/src/tests/weightedsetfieldvaluetest.cpp b/document/src/tests/weightedsetfieldvaluetest.cpp
new file mode 100644
index 00000000000..bc1bdca6028
--- /dev/null
+++ b/document/src/tests/weightedsetfieldvaluetest.cpp
@@ -0,0 +1,324 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/document/fieldvalue/fieldvalues.h>
+#include <vespa/document/serialization/vespadocumentdeserializer.h>
+#include <vespa/vdstestlib/cppunit/macros.h>
+#include <vespa/vespalib/objects/nbostream.h>
+
+using vespalib::nbostream;
+
+namespace document {
+
+struct WeightedSetFieldValueTest : public CppUnit::TestFixture {
+    void setUp() {}
+    void tearDown() {}
+
+    void testWeightedSet();
+    void testAddIgnoreZeroWeight();
+
+    CPPUNIT_TEST_SUITE(WeightedSetFieldValueTest);
+    CPPUNIT_TEST(testWeightedSet);
+    CPPUNIT_TEST(testAddIgnoreZeroWeight);
+    CPPUNIT_TEST_SUITE_END();
+
+};
+
+CPPUNIT_TEST_SUITE_REGISTRATION(WeightedSetFieldValueTest);
+
+namespace {
+template <typename T>
+void deserialize(const ByteBuffer &buffer, T &value) {
+    uint16_t version = Document::getNewestSerializationVersion();
+    nbostream stream(buffer.getBufferAtPos(), buffer.getRemaining());
+    DocumentTypeRepo repo;
+    VespaDocumentDeserializer deserializer(repo, stream, version);
+    deserializer.read(value);
+}
+
+template<typename Type1, typename Type2>
+void verifyFailedAssignment(Type1& lval, const Type2& rval)
+{
+    try{
+        lval = rval;
+        CPPUNIT_FAIL("Failed to check type equality in operator=");
+    } catch (std::exception& e) {
+        CPPUNIT_ASSERT_CONTAIN("Cannot assign value of type", e.what());
+    }
+    try{
+        lval.assign(rval);
+        CPPUNIT_FAIL("Failed to check type equality in assign()");
+    } catch (std::exception& e) {
+        CPPUNIT_ASSERT_CONTAIN("Cannot assign value of type", e.what());
+    }
+}
+
+template<typename Type>
+void verifyFailedUpdate(Type& lval, const FieldValue& rval)
+{
+    try{
+        lval.add(rval);
+        CPPUNIT_FAIL("Failed to check type equality in add()");
+    } catch (std::exception& e) {
+        CPPUNIT_ASSERT_CONTAIN("These types are not compatible", e.what());
+    }
+    try{
+        lval.contains(rval);
+        CPPUNIT_FAIL("Failed to check type equality in contains()");
+    } catch (std::exception& e) {
+        CPPUNIT_ASSERT_CONTAIN("These types are not compatible", e.what());
+    }
+    try{
+        lval.remove(rval);
+        CPPUNIT_FAIL("Failed to check type equality in remove()");
+    } catch (std::exception& e) {
+        CPPUNIT_ASSERT_CONTAIN("These types are not compatible", e.what());
+    }
+}
+}  // namespace
+
+void WeightedSetFieldValueTest::testWeightedSet()
+{
+    WeightedSetDataType type(*DataType::INT, false, false);
+    WeightedSetFieldValue value(type);
+
+        // Initially empty
+    CPPUNIT_ASSERT_EQUAL(size_t(0), value.size());
+    CPPUNIT_ASSERT(value.isEmpty());
+    CPPUNIT_ASSERT(!value.contains(IntFieldValue(1)));
+
+    CPPUNIT_ASSERT(value.add(IntFieldValue(1)));
+
+        // Not empty
+    CPPUNIT_ASSERT_EQUAL(size_t(1), value.size());
+    CPPUNIT_ASSERT(!value.isEmpty());
+    CPPUNIT_ASSERT(value.contains(IntFieldValue(1)));
+
+        // Adding some more
+    CPPUNIT_ASSERT(value.add(IntFieldValue(2), 5));
+    CPPUNIT_ASSERT(value.add(IntFieldValue(3), 6));
+
+        // Not empty
+    CPPUNIT_ASSERT_EQUAL(size_t(3), value.size());
+    CPPUNIT_ASSERT(!value.isEmpty());
+    CPPUNIT_ASSERT_EQUAL(1, value.get(IntFieldValue(1)));
+    CPPUNIT_ASSERT_EQUAL(5, value.get(IntFieldValue(2)));
+    CPPUNIT_ASSERT_EQUAL(6, value.get(IntFieldValue(3)));
+
+        // Serialize & equality
+    std::unique_ptr<ByteBuffer> buffer(value.serialize());
+    buffer->flip();
+    WeightedSetFieldValue value2(type);
+    CPPUNIT_ASSERT(value != value2);
+    deserialize(*buffer, value2);
+    CPPUNIT_ASSERT_EQUAL(value, value2);
+
+        // Various ways of removing
+    {
+            // By value
+        buffer->setPos(0);
+        deserialize(*buffer, value2);
+        CPPUNIT_ASSERT(value2.remove(IntFieldValue(1)));
+        CPPUNIT_ASSERT(!value2.contains(IntFieldValue(1)));
+        CPPUNIT_ASSERT_EQUAL(size_t(2), value2.size());
+
+            // Clearing all
+        buffer->setPos(0);
+        deserialize(*buffer, value2);
+        value2.clear();
+        CPPUNIT_ASSERT(!value2.contains(IntFieldValue(1)));
+        CPPUNIT_ASSERT_EQUAL(size_t(0), value2.size());
+        CPPUNIT_ASSERT(value2.isEmpty());
+    }
+
+        // Updating
+    value2 = value;
+    CPPUNIT_ASSERT_EQUAL(value, value2);
+    CPPUNIT_ASSERT(!value2.add(IntFieldValue(2), 10)); // false = overwritten
+    CPPUNIT_ASSERT(value2.add(IntFieldValue(17), 9)); // true = added new
+    CPPUNIT_ASSERT_EQUAL(10, value2.get(IntFieldValue(2)));
+    CPPUNIT_ASSERT(value != value2);
+    value2.assign(value);
+    CPPUNIT_ASSERT_EQUAL(value, value2);
+    WeightedSetFieldValue::UP valuePtr(value2.clone());
+    CPPUNIT_ASSERT_EQUAL(value, *valuePtr);
+
+        // Iterating
+    const WeightedSetFieldValue& constVal(value);
+    for(WeightedSetFieldValue::const_iterator it = constVal.begin();
+        it != constVal.end(); ++it)
+    {
+        const FieldValue& fval1(*it->first);
+        (void) fval1;
+        CPPUNIT_ASSERT_EQUAL((uint32_t) IntFieldValue::classId,
+                             it->first->getClass().id());
+        const IntFieldValue& val = dynamic_cast<const IntFieldValue&>(*it->second);
+        (void) val;
+    }
+    value2 = value;
+    for(WeightedSetFieldValue::iterator it = value2.begin();
+        it != value2.end(); ++it)
+    {
+        IntFieldValue& val = dynamic_cast<IntFieldValue&>(*it->second);
+        val.setValue(7);
+    }
+    CPPUNIT_ASSERT(value != value2);
+    CPPUNIT_ASSERT_EQUAL(7, value2.get(IntFieldValue(2)));
+
+        // Comparison
+    value2 = value;
+    CPPUNIT_ASSERT_EQUAL(0, value.compare(value2));
+    value2.remove(IntFieldValue(1));
+    CPPUNIT_ASSERT(value.compare(value2) > 0);
+    CPPUNIT_ASSERT(value2.compare(value) < 0);
+    value2 = value;
+    value2.add(IntFieldValue(7));
+    CPPUNIT_ASSERT(value.compare(value2) < 0);
+    CPPUNIT_ASSERT(value2.compare(value) > 0);
+
+        // Output
+    CPPUNIT_ASSERT_EQUAL(
+            std::string(
+                "WeightedSet<Int>(\n"
+                "  1 - weight 1,\n"
+                "  2 - weight 5,\n"
+                "  3 - weight 6\n"
+                ")"),
+            value.toString(false));
+    CPPUNIT_ASSERT_EQUAL(
+            std::string(
+                "  WeightedSet<Int>(\n"
+                "..  1 - weight 1,\n"
+                "..  2 - weight 5,\n"
+                "..  3 - weight 6\n"
+                "..)"),
+            "  " + value.toString(true, ".."));
+    CPPUNIT_ASSERT_EQUAL(
+            std::string(
+                "<value>\n"
+                "  <item weight=\"1\">1</item>\n"
+                "  <item weight=\"5\">2</item>\n"
+                "  <item weight=\"6\">3</item>\n"
+                "</value>"),
+            value.toXml("  "));
+
+        // Failure situations.
+
+        // Refuse to accept non-weightedset types
+    try{
+        ArrayDataType arrayType(*DataType::STRING);
+        WeightedSetFieldValue value6(arrayType);
+        CPPUNIT_FAIL("Didn't complain about non-weightedset type");
+    } catch (std::exception& e) {
+        CPPUNIT_ASSERT_CONTAIN("Cannot generate a weighted set value with "
+                               "non-weighted set type", e.what());
+    }
+
+        // Verify that datatypes are verified
+        // Created almost equal types to try to get it to fail
+    WeightedSetDataType type1(*DataType::INT, false, false);
+    WeightedSetDataType type2(*DataType::LONG, false, false);
+    WeightedSetDataType type3(type1, false, false);
+    WeightedSetDataType type4(type2, false, false);
+    WeightedSetDataType type5(type2, false, true);
+    WeightedSetDataType type6(type2, true, false);
+
+        // Type differs in nested of nested type (verify recursivity)
+    {
+        WeightedSetFieldValue value3(type3);
+        WeightedSetFieldValue value4(type4);
+        verifyFailedAssignment(value3, value4);
+    }
+        // Type arguments differ
+    {
+        WeightedSetFieldValue value4(type4);
+        WeightedSetFieldValue value5(type5);
+        WeightedSetFieldValue value6(type6);
+        verifyFailedAssignment(value4, value5);
+        verifyFailedAssignment(value4, value6);
+        verifyFailedAssignment(value5, value4);
+        verifyFailedAssignment(value5, value6);
+        verifyFailedAssignment(value6, value4);
+        verifyFailedAssignment(value6, value5);
+    }
+        // Updates are checked too
+    {
+        WeightedSetFieldValue value3(type3);
+        WeightedSetFieldValue subValue(type2);
+        subValue.add(LongFieldValue(4));
+        verifyFailedUpdate(value3, subValue);
+    }
+
+        // Compare see difference even of close types.
+    {
+        WeightedSetFieldValue subValue2(type2);
+        subValue2.add(LongFieldValue(3));
+        WeightedSetFieldValue value3(type3);
+        WeightedSetFieldValue value4(type4);
+        value4.add(subValue2);
+        CPPUNIT_ASSERT(value3.compare(value4) != 0);
+    }
+
+        // Test createIfNonExisting and removeIfZero
+    {
+        WeightedSetDataType mytype1(*DataType::STRING, false, false);
+        WeightedSetDataType mytype2(*DataType::STRING, true, true);
+        CPPUNIT_ASSERT_EQUAL(*DataType::TAG, static_cast<DataType &>(mytype2));
+
+        WeightedSetFieldValue val1(mytype1);
+        val1.add("foo", 4);
+        try{
+            val1.increment("bar", 2);
+            CPPUNIT_FAIL("Expected exception incrementing with "
+                         "createIfNonExistent set false");
+        } catch (std::exception& e) {}
+        try{
+            val1.decrement("bar", 2);
+            CPPUNIT_FAIL("Expected exception incrementing with "
+                         "createIfNonExistent set false");
+        } catch (std::exception& e) {}
+        val1.increment("foo", 6);
+        CPPUNIT_ASSERT_EQUAL(10, val1.get("foo"));
+        val1.decrement("foo", 3);
+        CPPUNIT_ASSERT_EQUAL(7, val1.get("foo"));
+        val1.decrement("foo", 7);
+        CPPUNIT_ASSERT(val1.contains("foo"));
+
+        WeightedSetFieldValue val2(mytype2);
+        val2.add("foo", 4);
+        val2.increment("bar", 2);
+        CPPUNIT_ASSERT_EQUAL(2, val2.get("bar"));
+        val2.decrement("bar", 4);
+        CPPUNIT_ASSERT_EQUAL(-2, val2.get("bar"));
+        val2.increment("bar", 2);
+        CPPUNIT_ASSERT(!val2.contains("bar"));
+
+        val2.decrement("foo", 4);
+        CPPUNIT_ASSERT(!val2.contains("foo"));
+
+        val2.decrement("foo", 4);
+        CPPUNIT_ASSERT_EQUAL(-4, val2.get("foo"));
+
+        val2.add("foo", 0);
+        CPPUNIT_ASSERT(!val2.contains("foo"));
+    }
+}
+
+void
+WeightedSetFieldValueTest::testAddIgnoreZeroWeight()
+{
+    // Data type with auto-create and remove-if-zero set.
+    WeightedSetDataType wsetType(*DataType::STRING, true, true);
+    WeightedSetFieldValue ws(wsetType);
+
+    ws.addIgnoreZeroWeight(StringFieldValue("yarn"), 0);
+    CPPUNIT_ASSERT(ws.contains("yarn"));
+    CPPUNIT_ASSERT_EQUAL(0, ws.get("yarn"));
+
+    ws.addIgnoreZeroWeight(StringFieldValue("flarn"), 1);
+    CPPUNIT_ASSERT(ws.contains("flarn"));
+    CPPUNIT_ASSERT_EQUAL(1, ws.get("flarn"));
+}
+
+} // document
+
diff --git a/document/src/tests/weightedsetfieldvaluetest.h b/document/src/tests/weightedsetfieldvaluetest.h
new file mode 100644
index 00000000000..8cf52d9ea0e
--- /dev/null
+++ b/document/src/tests/weightedsetfieldvaluetest.h
@@ -0,0 +1,6 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/* $Id$*/
+
+#pragma once
+
+
diff --git a/document/src/vespa/document/.gitignore b/document/src/vespa/document/.gitignore
new file mode 100644
index 00000000000..2262596df9f
--- /dev/null
+++ b/document/src/vespa/document/.gitignore
@@ -0,0 +1,7 @@
+*.exe
+*.ilk
+*.pdb
+*.so.*
+.depend
+Makefile
+/files.txt
diff --git a/document/src/vespa/document/CMakeLists.txt b/document/src/vespa/document/CMakeLists.txt
new file mode 100644
index 00000000000..1e5ed700d0b
--- /dev/null
+++ b/document/src/vespa/document/CMakeLists.txt
@@ -0,0 +1,19 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_library(document
+    SOURCES
+    $<TARGET_OBJECTS:document_util>
+    $<TARGET_OBJECTS:document_base>
+    $<TARGET_OBJECTS:document_bucket>
+    $<TARGET_OBJECTS:document_datatypes>
+    $<TARGET_OBJECTS:document_fieldvalues>
+    $<TARGET_OBJECTS:document_select>
+    $<TARGET_OBJECTS:document_updates>
+    $<TARGET_OBJECTS:document_fieldset>
+    $<TARGET_OBJECTS:document_documentconfig>
+    $<TARGET_OBJECTS:document_annotation>
+    $<TARGET_OBJECTS:document_repo>
+    $<TARGET_OBJECTS:document_serialization>
+    $<TARGET_OBJECTS:document_predicate>
+    INSTALL lib64
+    DEPENDS
+)
diff --git a/document/src/vespa/document/annotation/.gitignore b/document/src/vespa/document/annotation/.gitignore
new file mode 100644
index 00000000000..583460ae288
--- /dev/null
+++ b/document/src/vespa/document/annotation/.gitignore
@@ -0,0 +1,3 @@
+*.So
+.depend
+Makefile
diff --git a/document/src/vespa/document/annotation/CMakeLists.txt b/document/src/vespa/document/annotation/CMakeLists.txt
new file mode 100644
index 00000000000..9b1c6bc393f
--- /dev/null
+++ b/document/src/vespa/document/annotation/CMakeLists.txt
@@ -0,0 +1,12 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_library(document_annotation OBJECT
+    SOURCES
+    alternatespanlist.cpp
+    annotation.cpp
+    span.cpp
+    spanlist.cpp
+    spantree.cpp
+    DEPENDS
+    AFTER
+    document_documentconfig
+)
diff --git a/document/src/vespa/document/annotation/alternatespanlist.cpp b/document/src/vespa/document/annotation/alternatespanlist.cpp
new file mode 100644
index 00000000000..f99053dc06d
--- /dev/null
+++ b/document/src/vespa/document/annotation/alternatespanlist.cpp
@@ -0,0 +1,74 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".alternatespanlist");
+
+#include "alternatespanlist.h"
+#include "spanlist.h"
+
+using std::unique_ptr;
+using std::ostream;
+using std::string;
+
+namespace document {
+namespace {
+template <typename T>
+void ensureSize(size_t size, T &t) {
+    if (size > t.size()) {
+        t.resize(size);
+    }
+}
+}  // namespace
+
+void AlternateSpanList::add(size_t index, unique_ptr<SpanNode> node) {
+    ensureSize(index + 1, _subtrees);
+    Subtree &subtree = _subtrees[index];
+    if (!subtree.span_list) {
+        subtree.span_list = new SpanList;
+    }
+    subtree.span_list->add(std::move(node));
+}
+
+AlternateSpanList::~AlternateSpanList() {
+    for (size_t i = 0; i < _subtrees.size(); ++i) {
+        delete _subtrees[i].span_list;
+    }
+}
+
+void AlternateSpanList::setSubtree(size_t index,
+                                   std::unique_ptr<SpanList> subtree) {
+    ensureSize(index + 1, _subtrees);
+    _subtrees[index].span_list = subtree.release();
+}
+
+void AlternateSpanList::setProbability(size_t index, double probability) {
+    ensureSize(index + 1, _subtrees);
+    _subtrees[index].probability = probability;
+}
+
+SpanList &AlternateSpanList::getSubtree(size_t index) const {
+    assert(index < _subtrees.size());
+    assert(_subtrees[index].span_list);
+    return *_subtrees[index].span_list;
+}
+
+double AlternateSpanList::getProbability(size_t index) const {
+    assert(index < _subtrees.size());
+    return _subtrees[index].probability;
+}
+
+void AlternateSpanList::print(ostream& out, bool verbose,
+                              const string& indent) const {
+    out << "AlternateSpanList(\n" << indent << "  ";
+    for (size_t i = 0; i < _subtrees.size(); ++i) {
+        out << "Probability " << _subtrees[i].probability << " : ";
+        _subtrees[i].span_list->print(out, verbose, indent + "  ");
+        if (i < _subtrees.size() - 1) {
+            out << ",\n" << indent << "  ";
+        }
+    }
+    out << ")";
+}
+
+}  // namespace document
diff --git a/document/src/vespa/document/annotation/alternatespanlist.h b/document/src/vespa/document/annotation/alternatespanlist.h
new file mode 100644
index 00000000000..4b4410e8058
--- /dev/null
+++ b/document/src/vespa/document/annotation/alternatespanlist.h
@@ -0,0 +1,47 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "spanlist.h"
+#include "spannode.h"
+#include "spantreevisitor.h"
+#include <memory>
+#include <vector>
+
+namespace document {
+class AlternateSpanList : public SpanNode {
+    struct Subtree {
+        SpanList *span_list;
+        double probability;
+        Subtree() : span_list(0), probability(0.0) {}
+    };
+    std::vector<Subtree> _subtrees;
+
+    void add(size_t index, std::unique_ptr<SpanNode> node);
+
+public:
+    typedef std::unique_ptr<AlternateSpanList> UP;
+
+    ~AlternateSpanList();
+
+    template <typename T>
+    T &add(size_t index, std::unique_ptr<T> node) {
+        T *n = node.get();
+        add(index, std::unique_ptr<SpanNode>(std::move(node)));
+        return *n;
+    }
+
+    void setSubtree(size_t index, std::unique_ptr<SpanList> subtree);
+    void setProbability(size_t index, double probability);
+
+    size_t getNumSubtrees() const { return _subtrees.size(); }
+    SpanList &getSubtree(size_t index) const;
+    double getProbability(size_t index) const;
+
+    virtual void accept(SpanTreeVisitor &visitor) const { visitor.visit(*this); }
+    virtual void print(
+            std::ostream& out, bool verbose, const std::string& indent) const;
+};
+
+}  // namespace document
+
diff --git a/document/src/vespa/document/annotation/annotation.cpp b/document/src/vespa/document/annotation/annotation.cpp
new file mode 100644
index 00000000000..f5367e4340f
--- /dev/null
+++ b/document/src/vespa/document/annotation/annotation.cpp
@@ -0,0 +1,34 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".annotation");
+
+#include "annotation.h"
+#include "spannode.h"
+#include <vespa/document/fieldvalue/fieldvalue.h>
+#include <vespa/document/util/bytebuffer.h>
+
+using std::ostream;
+using std::string;
+
+namespace document {
+
+Annotation::~Annotation() {
+}
+
+void
+Annotation::print(ostream& out, bool verbose, const string& indent) const {
+    out << "Annotation(" << *_type;
+    if (_value.get()) {
+        out << "\n" << indent << "  ";
+        _value->print(out, verbose, indent + "  ");
+    }
+    if (_node) {
+        out << "\n" << indent << "  ";
+        _node->print(out, verbose, indent + "  ");
+    }
+    out << ")";
+}
+
+}  // namespace document
diff --git a/document/src/vespa/document/annotation/annotation.h b/document/src/vespa/document/annotation/annotation.h
new file mode 100644
index 00000000000..16115122d39
--- /dev/null
+++ b/document/src/vespa/document/annotation/annotation.h
@@ -0,0 +1,52 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/document/datatype/annotationtype.h>
+#include <vespa/document/fieldvalue/fieldvalue.h>
+#include <vespa/vespalib/util/printable.h>
+#include <memory>
+
+namespace document {
+class SpanNode;
+
+class Annotation : public Printable {
+    const AnnotationType * _type;
+    const SpanNode *_node;
+    FieldValue::CP _value;
+
+public:
+    typedef std::unique_ptr<Annotation> UP;
+
+    template <typename T>
+    Annotation(const AnnotationType & type, std::unique_ptr<T> value)
+        : _type(&type), _node(nullptr), _value(value.release()) {}
+
+    Annotation(const AnnotationType &annotation) : _type(&annotation), _node(nullptr), _value(nullptr) { }
+    Annotation() : _type(nullptr), _node(nullptr), _value(nullptr) { }
+    ~Annotation();
+
+    void setType(const AnnotationType * v) { _type = v; }
+    void setSpanNode(const SpanNode &node) { _node = &node; }
+    template <typename T>
+    void setFieldValue(std::unique_ptr<T> value) { _value.reset(value.release()); }
+
+    const SpanNode *getSpanNode() const { return _node; }
+    const AnnotationType &getType() const { return *_type; }
+    bool valid() const { return _type != nullptr; }
+    int32_t getTypeId() const { return _type->getId(); }
+    const FieldValue *getFieldValue() const { return _value.get(); }
+
+    void print(std::ostream& out, bool verbose, const std::string& indent) const override;
+};
+
+inline bool operator==(const Annotation &a1, const Annotation &a2) {
+    return (a1.getType() == a2.getType() &&
+            !(!!a1.getFieldValue() ^ !!a2.getFieldValue()) &&
+            (!a1.getFieldValue() ||
+             (*a1.getFieldValue() == *a2.getFieldValue()))
+            );
+}
+
+}  // namespace document
+
diff --git a/document/src/vespa/document/annotation/span.cpp b/document/src/vespa/document/annotation/span.cpp
new file mode 100644
index 00000000000..32d3184c0e3
--- /dev/null
+++ b/document/src/vespa/document/annotation/span.cpp
@@ -0,0 +1,18 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".span");
+
+#include "span.h"
+
+using std::ostream;
+using std::string;
+
+namespace document {
+
+void Span::print(ostream& out, bool, const string&) const {
+    out << "Span(" << _from << ", " << _length << ")";
+}
+
+}  // namespace document
diff --git a/document/src/vespa/document/annotation/span.h b/document/src/vespa/document/annotation/span.h
new file mode 100644
index 00000000000..f9694ddcb5c
--- /dev/null
+++ b/document/src/vespa/document/annotation/span.h
@@ -0,0 +1,47 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "spannode.h"
+#include "spantreevisitor.h"
+#include <memory>
+#include <ostream>
+
+namespace document {
+
+class Span : public SpanNode {
+    int32_t _from;
+    int32_t _length;
+
+public:
+    typedef std::unique_ptr<Span> UP;
+
+    Span(int32_t from_pos=0, int32_t len=0) : _from(from_pos), _length(len) {}
+
+    int32_t from() const { return _from; }
+    int32_t length() const { return _length; }
+    Span & from(int32_t from_pos) { _from = from_pos; return *this; }
+    Span & length(int32_t length_pos) { _length = length_pos; return *this; }
+
+    void accept(SpanTreeVisitor &visitor) const override { visitor.visit(*this); }
+    void print(std::ostream& out, bool verbose, const std::string& indent) const override;
+};
+
+inline bool operator==(const Span &span1, const Span &span2) {
+    return span1.from() == span2.from() && span1.length() == span2.length();
+}
+
+inline bool operator<(const Span &span1, const Span &span2) {
+    if (span1.from() != span2.from()) {
+        return span1.from() < span2.from();
+    } else {
+        return span1.length() < span2.length();
+    }
+}
+
+inline bool operator>(const Span &span1, const Span &span2) {
+    return span2 < span1;
+}
+
+}  // namespace document
+
diff --git a/document/src/vespa/document/annotation/spanlist.cpp b/document/src/vespa/document/annotation/spanlist.cpp
new file mode 100644
index 00000000000..37ff8ca5f13
--- /dev/null
+++ b/document/src/vespa/document/annotation/spanlist.cpp
@@ -0,0 +1,57 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".spanlist");
+
+#include "spanlist.h"
+
+using std::ostream;
+using std::string;
+
+namespace document {
+
+SpanList::~SpanList() {
+    for (size_t i = 0; i < _span_vector.size(); ++i) {
+        delete _span_vector[i];
+    }
+}
+
+void SpanList::print(ostream& out, bool verbose, const string& indent) const {
+    out << "SpanList(";
+    if (_span_vector.size() > 1) {
+        out << "\n" << indent << "  ";
+    }
+    for (size_t i = 0; i < _span_vector.size(); ++i) {
+        _span_vector[i]->print(out, verbose, indent + "  ");
+        if (i < _span_vector.size() - 1) {
+            out << "\n" << indent << "  ";
+        }
+    }
+    out << ")";
+}
+
+SimpleSpanList::SimpleSpanList(size_t sz) :
+    _span_vector(sz)
+{
+}
+
+SimpleSpanList::~SimpleSpanList()
+{
+}
+
+void SimpleSpanList::print(ostream& out, bool verbose, const string& indent) const {
+    out << "SimpleSpanList(";
+    if (_span_vector.size() > 1) {
+        out << "\n" << indent << "  ";
+    }
+    for (size_t i = 0; i < _span_vector.size(); ++i) {
+        _span_vector[i].print(out, verbose, indent + "  ");
+        if (i < _span_vector.size() - 1) {
+            out << "\n" << indent << "  ";
+        }
+    }
+    out << ")";
+}
+
+}  // namespace document
diff --git a/document/src/vespa/document/annotation/spanlist.h b/document/src/vespa/document/annotation/spanlist.h
new file mode 100644
index 00000000000..76aac073ee7
--- /dev/null
+++ b/document/src/vespa/document/annotation/spanlist.h
@@ -0,0 +1,61 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "span.h"
+#include "spantreevisitor.h"
+#include <memory>
+#include <vector>
+
+namespace document {
+
+class SpanList : public SpanNode {
+    std::vector<SpanNode *> _span_vector;
+
+public:
+    typedef std::unique_ptr<SpanList> UP;
+    typedef std::vector<SpanNode *>::const_iterator const_iterator;
+
+    ~SpanList();
+
+    template <typename T>
+    T &add(std::unique_ptr<T> node) {
+        T *n = node.get();
+        _span_vector.push_back(node.release());
+        return *n;
+    }
+
+    size_t size() const { return _span_vector.size(); }
+    void reserve(size_t sz) { _span_vector.reserve(sz); }
+
+    const_iterator begin() const { return _span_vector.begin(); }
+    const_iterator end() const { return _span_vector.end(); }
+
+    void accept(SpanTreeVisitor &visitor) const override { visitor.visit(*this); }
+    void print(std::ostream& out, bool verbose, const std::string& indent) const override;
+};
+
+class SimpleSpanList : public SpanNode {
+    typedef std::vector<Span> SpanVector;
+    SpanVector _span_vector;
+
+public:
+    typedef std::unique_ptr<SimpleSpanList> UP;
+    typedef SpanVector::const_iterator const_iterator;
+
+    SimpleSpanList(size_t sz);
+    ~SimpleSpanList();
+
+    size_t size() const { return _span_vector.size(); }
+    Span & operator [] (size_t index) { return _span_vector[index]; }
+    const Span & operator [] (size_t index) const { return _span_vector[index]; }
+
+    const_iterator begin() const { return _span_vector.begin(); }
+    const_iterator end() const { return _span_vector.end(); }
+
+    void accept(SpanTreeVisitor &visitor) const override { visitor.visit(*this); }
+    void print(std::ostream& out, bool verbose, const std::string& indent) const override;
+};
+
+}  // namespace document
+
diff --git a/document/src/vespa/document/annotation/spannode.h b/document/src/vespa/document/annotation/spannode.h
new file mode 100644
index 00000000000..843613c209b
--- /dev/null
+++ b/document/src/vespa/document/annotation/spannode.h
@@ -0,0 +1,20 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/document/util/printable.h>
+#include <memory>
+
+namespace document {
+class SpanTreeVisitor;
+
+struct SpanNode : Printable {
+    typedef std::unique_ptr<SpanNode> UP;
+
+    virtual ~SpanNode() {}
+
+    virtual void accept(SpanTreeVisitor &visitor) const = 0;
+};
+
+}  // namespace document
+
diff --git a/document/src/vespa/document/annotation/spantree.cpp b/document/src/vespa/document/annotation/spantree.cpp
new file mode 100644
index 00000000000..6ec3ac7c6af
--- /dev/null
+++ b/document/src/vespa/document/annotation/spantree.cpp
@@ -0,0 +1,62 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".spantree");
+
+#include "spantree.h"
+
+#include "annotation.h"
+#include "spannode.h"
+#include <vespa/vespalib/stllike/string.h>
+
+using std::unique_ptr;
+using std::ostream;
+using std::ostringstream;
+using std::string;
+using vespalib::stringref;
+
+namespace document {
+
+SpanTree::~SpanTree() {
+}
+
+size_t SpanTree::annotate(Annotation::UP annotation_) {
+    _annotations.push_back(*annotation_);
+    return _annotations.size() - 1;
+}
+
+size_t SpanTree::annotate(const SpanNode &node, Annotation::UP annotation_) {
+    annotation_->setSpanNode(node);
+    return annotate(std::move(annotation_));
+}
+
+size_t SpanTree::annotate(const SpanNode &node, const AnnotationType &type) {
+    return annotate(node, Annotation::UP(new Annotation(type)));
+}
+
+void SpanTree::accept(SpanTreeVisitor &visitor) const {
+    _root->accept(visitor);
+}
+
+void SpanTree::print(ostream& out, bool verbose, const string& indent) const {
+    out << "SpanTree(\"" << _name << "\""
+        << "\n" << indent << "  ";
+    _root->print(out, verbose, indent + "  ");
+    for (const Annotation & a : _annotations) {
+        if (a.valid()) {
+            out << "\n" << indent << "  ";
+            a.print(out, verbose, indent + "  ");
+        }
+    }
+    out << ")";
+}
+
+int SpanTree::compare(const SpanTree &other) const {
+    ostringstream out_this, out_other;
+    print(out_this, true, "");
+    other.print(out_other, true, "");
+    return stringref(out_this.str()).compare(stringref(out_other.str()));
+}
+
+}  // namespace document
diff --git a/document/src/vespa/document/annotation/spantree.h b/document/src/vespa/document/annotation/spantree.h
new file mode 100644
index 00000000000..abb8365ef23
--- /dev/null
+++ b/document/src/vespa/document/annotation/spantree.h
@@ -0,0 +1,53 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/document/annotation/annotation.h>
+#include <vespa/vespalib/util/printable.h>
+#include <vector>
+
+namespace document {
+class SpanNode;
+class SpanTreeVisitor;
+
+class SpanTree : public Printable {
+    typedef std::vector<Annotation> AnnotationVector;
+    vespalib::string _name;
+    std::unique_ptr<SpanNode> _root;
+    std::vector<Annotation> _annotations;
+
+public:
+    typedef std::unique_ptr<SpanTree> UP;
+    typedef AnnotationVector::const_iterator const_iterator;
+
+    template <typename T>
+    SpanTree(const vespalib::stringref &name, std::unique_ptr<T> root)
+        : _name(name),
+          _root(std::move(root)) {
+        assert(_root.get());
+    }
+    ~SpanTree();
+
+    // The annotate functions return the annotation index.
+    size_t annotate(std::unique_ptr<Annotation> annotation);
+    size_t annotate(const SpanNode &node, std::unique_ptr<Annotation> a);
+    size_t annotate(const SpanNode &node, const AnnotationType &a_type);
+
+    Annotation & annotation(size_t index) { return _annotations[index]; }
+    const Annotation & annotation(size_t index) const { return _annotations[index]; }
+
+    void accept(SpanTreeVisitor &visitor) const;
+    virtual void print(
+            std::ostream& out, bool verbose, const std::string& indent) const;
+
+    const vespalib::string & getName() const { return _name; }
+    const SpanNode &getRoot() const { return *_root; }
+    size_t numAnnotations() const { return _annotations.size(); }
+    void reserveAnnotations(size_t sz) { _annotations.resize(sz); }
+    const_iterator begin() const { return _annotations.begin(); }
+    const_iterator end() const { return _annotations.end(); }
+    int compare(const SpanTree &other) const;
+};
+
+}  // namespace document
+
diff --git a/document/src/vespa/document/annotation/spantreevisitor.h b/document/src/vespa/document/annotation/spantreevisitor.h
new file mode 100644
index 00000000000..fe77d045a86
--- /dev/null
+++ b/document/src/vespa/document/annotation/spantreevisitor.h
@@ -0,0 +1,21 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+namespace document {
+class AlternateSpanList;
+class Span;
+class SpanList;
+class SimpleSpanList;
+
+struct SpanTreeVisitor {
+    virtual ~SpanTreeVisitor() {}
+
+    virtual void visit(const Span &) = 0;
+    virtual void visit(const SpanList &node) = 0;
+    virtual void visit(const SimpleSpanList &node) = 0;
+    virtual void visit(const AlternateSpanList &node) = 0;
+};
+
+}  // namespace document
+
diff --git a/document/src/vespa/document/base/.gitignore b/document/src/vespa/document/base/.gitignore
new file mode 100644
index 00000000000..64fbfa2e9b2
--- /dev/null
+++ b/document/src/vespa/document/base/.gitignore
@@ -0,0 +1,8 @@
+*.So
+.*.swp
+.depend
+Makefile
+config-documentmanager.cpp
+config-documentmanager.h
+config-documentselectconfig.cpp
+config-documentselectconfig.h
diff --git a/document/src/vespa/document/base/CMakeLists.txt b/document/src/vespa/document/base/CMakeLists.txt
new file mode 100644
index 00000000000..0c389c74e20
--- /dev/null
+++ b/document/src/vespa/document/base/CMakeLists.txt
@@ -0,0 +1,17 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_library(document_base OBJECT
+    SOURCES
+    documentcalculator.cpp
+    documentid.cpp
+    exceptions.cpp
+    field.cpp
+    fieldpath.cpp
+    forcelink.cpp
+    globalid.cpp
+    idstring.cpp
+    testdocman.cpp
+    testdocrepo.cpp
+    DEPENDS
+    AFTER
+    document_documentconfig
+)
diff --git a/document/src/vespa/document/base/documentcalculator.cpp b/document/src/vespa/document/base/documentcalculator.cpp
new file mode 100644
index 00000000000..0d2ac7e81c5
--- /dev/null
+++ b/document/src/vespa/document/base/documentcalculator.cpp
@@ -0,0 +1,39 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/document/base/documentcalculator.h>
+#include <vespa/document/bucket/bucketidfactory.h>
+#include <vespa/document/select/compare.h>
+#include <vespa/document/select/parser.h>
+#include <vespa/document/select/valuenode.h>
+
+namespace document {
+
+DocumentCalculator::DocumentCalculator(
+        const DocumentTypeRepo& repo,
+        const vespalib::string & expression)
+{
+    BucketIdFactory factory;
+    select::Parser parser(repo, factory);
+    _selectionNode = parser.parse(expression + " == 0");
+}
+
+double
+DocumentCalculator::evaluate(const Document& doc, VariableMap& variables)
+{
+    select::Compare& compare(static_cast<select::Compare&>(*_selectionNode));
+    const select::ValueNode& left = compare.getLeft();
+
+    select::Context context(doc);
+    context._variables = variables;
+    std::unique_ptr<select::Value> value = left.getValue(context);
+
+    select::NumberValue* num = dynamic_cast<select::NumberValue*>(value.get());
+
+    if (!num) {
+        throw vespalib::IllegalArgumentException("Expression could not be evaluated - some components of the expression may be missing");
+    }
+
+    return num->getCommonValue();
+}
+
+}
diff --git a/document/src/vespa/document/base/documentcalculator.h b/document/src/vespa/document/base/documentcalculator.h
new file mode 100644
index 00000000000..59b601cf21e
--- /dev/null
+++ b/document/src/vespa/document/base/documentcalculator.h
@@ -0,0 +1,23 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/document/select/node.h>
+#include <vespa/vespalib/stllike/hash_map.h>
+
+namespace document {
+class DocumentTypeRepo;
+
+class DocumentCalculator {
+public:
+    typedef vespalib::hash_map<vespalib::string, double> VariableMap;
+
+    DocumentCalculator(const DocumentTypeRepo& repo,
+                       const vespalib::string& expression);
+    double evaluate(const Document& doc, VariableMap& variables);
+
+private:
+    std::unique_ptr<select::Node> _selectionNode;
+};
+
+}
+
diff --git a/document/src/vespa/document/base/documentid.cpp b/document/src/vespa/document/base/documentid.cpp
new file mode 100644
index 00000000000..13c3b7f724c
--- /dev/null
+++ b/document/src/vespa/document/base/documentid.cpp
@@ -0,0 +1,86 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".document.document-id");
+
+#include "documentid.h"
+
+#include <vespa/vespalib/util/md5.h>
+#include <vespa/vespalib/objects/nbostream.h>
+
+using vespalib::nbostream;
+
+namespace document {
+
+DocumentId::DocumentId()
+    : Printable(),
+      _globalId(),
+      _id(new NullIdString())
+{
+}
+
+DocumentId::DocumentId(const vespalib::stringref & id)
+    : Printable(),
+      _globalId(),
+      _id(IdString::createIdString(id.c_str(), id.size()).release())
+{
+}
+
+DocumentId::DocumentId(vespalib::nbostream & is)
+    : Printable(),
+      _globalId(),
+      _id(IdString::createIdString(is.peek(), strlen(is.peek())).release())
+{
+    is.adjustReadPos(strlen(is.peek()) + 1);
+}
+
+DocumentId::DocumentId(const IdString& id)
+    : Printable(),
+      _globalId(),
+      _id(id.clone())
+{
+}
+
+vespalib::string
+DocumentId::toString() const {
+    return _id->toString();
+}
+
+void
+DocumentId::print(std::ostream& out, bool verbose,
+                  const std::string& indent) const
+{
+    (void) indent;
+    if (verbose) {
+        out << "DocumentId(id = ";
+    }
+    out << _id->toString().c_str();
+    if (verbose) {
+        out << ", " << getGlobalId() << ")";
+    }
+}
+
+size_t
+DocumentId::getSerializedSize() const
+{
+    return _id->toString().size() + 1;
+}
+
+void
+DocumentId::calculateGlobalId() const
+{
+    vespalib::string id(_id->toString());
+
+    unsigned char key[16];
+    fastc_md5sum(reinterpret_cast<const unsigned char*>(id.c_str()),
+                 id.size(), key);
+
+    IdString::LocationType location(_id->getLocation());
+    memcpy(key, &location, 4);
+
+    _globalId.first = true;
+    _globalId.second.set(key);
+}
+
+
+} // document
diff --git a/document/src/vespa/document/base/documentid.h b/document/src/vespa/document/base/documentid.h
new file mode 100644
index 00000000000..86d58d68601
--- /dev/null
+++ b/document/src/vespa/document/base/documentid.h
@@ -0,0 +1,89 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * \class document::DocumentId
+ * \ingroup base
+ *
+ * \brief Class describing a document identifier.
+ *
+ * The document identifier is an URI set by the user. The URI must conform
+ * to one of the accepted document identifier schemes.
+ *
+ * The IdString class represent such a scheme. See the various implementations
+ * of IdString to see legal schemes.
+ *
+ * This class contains the identifier parsed into pieces. Thus, get() functions
+ * are cheap to call.
+ *
+ * It's up to the users to ensure that they use unique document identifiers.
+ */
+
+#pragma once
+
+#include <vespa/document/base/idstring.h>
+#include <vespa/document/base/globalid.h>
+#include <vespa/document/util/printable.h>
+
+namespace vespalib { class nbostream; }
+
+namespace document {
+
+class DocumentType;
+
+class DocumentId : public Printable
+{
+public:
+    typedef std::unique_ptr<DocumentId> UP;
+
+    DocumentId();
+    DocumentId(vespalib::nbostream & os);
+    explicit DocumentId(const IdString& id);
+    /**
+     * Parse the given document identifier given as string, and create an
+     * identifier object from it.
+     *
+     * @throws IdParseException If the identifier given is invalid.
+     */
+    explicit DocumentId(const vespalib::stringref & id);
+
+    void set(const vespalib::stringref & id) {
+        _id.reset(IdString::createIdString(id).release());
+        _globalId.first = false;
+    }
+
+    /**
+       Hides the printable toString() for effiency reasons.
+    */
+    vespalib::string toString() const;
+
+    void print(std::ostream& out, bool verbose,
+               const std::string& indent) const;
+
+    bool operator==(const DocumentId& other) const { return *_id == *other._id; }
+    bool operator!=(const DocumentId& other) const { return ! (*_id == *other._id); }
+
+    const IdString& getScheme() const { return *_id; }
+    bool hasDocType() const { return _id->hasDocType(); }
+    vespalib::string getDocType() const { return _id->getDocType(); }
+
+    const GlobalId& getGlobalId() const {
+        if (!_globalId.first) { calculateGlobalId(); }
+        return _globalId.second;
+    }
+
+    DocumentId* clone() const { return new DocumentId(*this); }
+
+    virtual size_t getSerializedSize() const;
+    void swap(DocumentId & rhs) {
+        _id.swap(rhs._id);
+        std::swap(_globalId, rhs._globalId);
+    }
+
+private:
+    mutable std::pair<bool, GlobalId> _globalId;
+    IdString::CP _id;
+
+    void calculateGlobalId() const;
+};
+
+} // document
+
diff --git a/document/src/vespa/document/base/exceptions.cpp b/document/src/vespa/document/base/exceptions.cpp
new file mode 100644
index 00000000000..d326e54c0c3
--- /dev/null
+++ b/document/src/vespa/document/base/exceptions.cpp
@@ -0,0 +1,110 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/document/base/exceptions.h>
+
+#include <vespa/document/datatype/datatype.h>
+#include <vespa/document/fieldvalue/document.h>
+
+namespace document {
+
+VESPA_IMPLEMENT_EXCEPTION_SPINE(InvalidDataTypeException);
+VESPA_IMPLEMENT_EXCEPTION_SPINE(InvalidDataTypeConversionException);
+VESPA_IMPLEMENT_EXCEPTION_SPINE(DocumentTypeNotFoundException);
+VESPA_IMPLEMENT_EXCEPTION_SPINE(DataTypeNotFoundException);
+VESPA_IMPLEMENT_EXCEPTION_SPINE(AnnotationTypeNotFoundException);
+VESPA_IMPLEMENT_EXCEPTION_SPINE(FieldNotFoundException);
+
+InvalidDataTypeException::InvalidDataTypeException(
+        const DataType& actual,
+        const DataType& expected,
+        const vespalib::string& location)
+    : vespalib::IllegalStateException(
+            vespalib::make_string(
+                "Got %s while expecting %s. These types are not compatible.",
+                actual.toString().c_str(),
+                expected.toString().c_str()
+            ), location, 1),
+      _actual(actual),
+      _expected(expected)
+{
+}
+
+InvalidDataTypeException::~InvalidDataTypeException() throw()
+{
+}
+
+InvalidDataTypeConversionException::InvalidDataTypeConversionException(
+        const DataType &actual,
+        const DataType &expected,
+        const vespalib::string& location)
+    : vespalib::IllegalStateException(
+            vespalib::make_string(
+                "%s can not be converted to %s.",
+                actual.toString().c_str(),
+                expected.toString().c_str()
+            ), location, 1),
+      _actual(actual),
+      _expected(expected)
+{
+}
+
+InvalidDataTypeConversionException::~InvalidDataTypeConversionException() throw()
+{
+}
+
+DocumentTypeNotFoundException::DocumentTypeNotFoundException(const vespalib::string& name, const vespalib::string& location)
+    : Exception("Document type "+name+" not found", location, 1),
+      _type(name)
+{
+}
+
+DataTypeNotFoundException::DataTypeNotFoundException(int id, const vespalib::string& location)
+    : Exception(vespalib::make_string("Data type with id %d not found", id), location, 1)
+{
+}
+
+DataTypeNotFoundException::DataTypeNotFoundException(const vespalib::string& name, const vespalib::string& location)
+    : Exception("Data type with name "+name+" not found.", location, 1)
+{
+}
+
+AnnotationTypeNotFoundException::AnnotationTypeNotFoundException(
+        int id, const vespalib::string& location)
+    : Exception(vespalib::make_string("Data type with id %d not found", id),
+                location, 1) {
+}
+
+FieldNotFoundException::
+FieldNotFoundException(const vespalib::string& fieldName,
+                       const vespalib::string& location)
+    : Exception("Field with name " + fieldName + " not found", location, 1),
+      _fieldName(fieldName),
+      _fieldId(0),
+      _serializationVersion(0)
+{
+}
+
+FieldNotFoundException::
+FieldNotFoundException(int fieldId,
+                       int16_t serializationVersion,
+                       const vespalib::string& location)
+    : Exception((serializationVersion < Document::getNewestSerializationVersion()) ?
+                vespalib::make_string("Field with id %i (serialization version %d) not found", fieldId, serializationVersion) :
+        vespalib::make_string("Field with id %i not found", fieldId), location, 1),
+
+      _fieldName(),
+      _fieldId(fieldId),
+      _serializationVersion(serializationVersion)
+{
+}
+
+FieldNotFoundException::~FieldNotFoundException() throw()
+{
+}
+
+DocumentTypeNotFoundException::~DocumentTypeNotFoundException() throw()
+{
+}
+
+}
diff --git a/document/src/vespa/document/base/exceptions.h b/document/src/vespa/document/base/exceptions.h
new file mode 100644
index 00000000000..785af5f82cd
--- /dev/null
+++ b/document/src/vespa/document/base/exceptions.h
@@ -0,0 +1,143 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <memory>
+#include <vespa/vespalib/util/exceptions.h>
+#include <stdint.h>
+
+namespace document {
+
+class DataType;
+
+/**
+ * \class document::InvalidDataTypeException
+ * \ingroup base
+ *
+ * \brief Exception used to report invalid datatype usage.
+ */
+class InvalidDataTypeException : public vespalib::IllegalStateException
+{
+public:
+    InvalidDataTypeException(const DataType &actual,
+                             const DataType &wanted,
+                             const vespalib::string & location);
+    virtual ~InvalidDataTypeException() throw();
+
+    const DataType& getActualDataType() const { return _actual; }
+    const DataType& getExpectedDataType() const { return _expected; }
+
+    VESPA_DEFINE_EXCEPTION_SPINE(InvalidDataTypeException);
+
+private:
+    const DataType &_actual;
+    const DataType &_expected;
+
+};
+
+/**
+ * \class document::InvalidDataTypeConversionException
+ * \ingroup base
+ *
+ * \brief Exception used to report invalid datatype convertion.
+ */
+class InvalidDataTypeConversionException
+    : public vespalib::IllegalStateException
+{
+public:
+    InvalidDataTypeConversionException(const DataType &actual,
+                                       const DataType &wanted,
+                                       const vespalib::string & location);
+    virtual ~InvalidDataTypeConversionException() throw();
+
+    const DataType& getActualDataType() const { return _actual; }
+    const DataType& getExpectedDataType() const { return _expected; }
+
+    VESPA_DEFINE_EXCEPTION_SPINE(InvalidDataTypeConversionException);
+
+private:
+    const DataType &_actual;
+    const DataType &_expected;
+
+};
+
+/**
+ * \class document::DocumentTypeNotFoundException
+ * \ingroup base
+ *
+ * \brief Exception used when a document type is not found.
+ */
+class DocumentTypeNotFoundException : public vespalib::Exception
+{
+private:
+    vespalib::string _type;
+
+public:
+    DocumentTypeNotFoundException(const vespalib::string & name,
+                                  const vespalib::string& location);
+    virtual ~DocumentTypeNotFoundException() throw();
+
+    const vespalib::string& getDocumentTypeName() const { return _type; }
+
+    VESPA_DEFINE_EXCEPTION_SPINE(DocumentTypeNotFoundException);
+};
+
+/**
+ * \class document::DataTypeNotFoundException
+ * \ingroup base
+ *
+ * \brief Exception used when a data type is not found.
+ */
+class DataTypeNotFoundException : public vespalib::Exception
+{
+public:
+    DataTypeNotFoundException(int id, const vespalib::string& location);
+    DataTypeNotFoundException(const vespalib::string& name, const vespalib::string& location);
+
+    VESPA_DEFINE_EXCEPTION_SPINE(DataTypeNotFoundException);
+};
+
+/**
+ * \class document::AnnotationTypeNotFoundException
+ * \ingroup base
+ *
+ * \brief Exception used when an annotation type is not found.
+ */
+class AnnotationTypeNotFoundException : public vespalib::Exception
+{
+public:
+    AnnotationTypeNotFoundException(int id, const vespalib::string& location);
+
+    VESPA_DEFINE_EXCEPTION_SPINE(AnnotationTypeNotFoundException);
+};
+
+/**
+ * \class document::FieldNotFoundException
+ * \ingroup base
+ *
+ * \brief Typically thrown when accessing non-existing fields in structured
+ *        datatypes.
+ */
+class FieldNotFoundException : public vespalib::Exception
+{
+private:
+    vespalib::string _fieldName;
+    int32_t _fieldId;
+    int16_t _serializationVersion;
+
+public:
+    FieldNotFoundException(const vespalib::string& fieldName,
+                           const vespalib::string& location);
+    FieldNotFoundException(int32_t fieldId,
+                           int16_t serializationVersion,
+                           const vespalib::string& location);
+    ~FieldNotFoundException() throw();
+
+    const vespalib::string& getFieldName() const { return _fieldName; }
+    int32_t getFieldId()              const { return _fieldId; };
+
+    VESPA_DEFINE_EXCEPTION_SPINE(FieldNotFoundException);
+};
+
+}
+
diff --git a/document/src/vespa/document/base/field.cpp b/document/src/vespa/document/base/field.cpp
new file mode 100644
index 00000000000..99254141222
--- /dev/null
+++ b/document/src/vespa/document/base/field.cpp
@@ -0,0 +1,126 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/document/base/field.h>
+
+#include <vespa/document/datatype/datatype.h>
+#include <vespa/vespalib/util/exceptions.h>
+#include <vespa/vespalib/stllike/asciistream.h>
+#include <vespa/vespalib/util/bobhash.h>
+
+namespace document {
+
+Field::Field(const vespalib::stringref & name, int fieldId,
+             const DataType& dataType, bool headerField)
+    : FieldBase(name),
+      _dataType(&dataType),
+      _fieldIdV6(fieldId),
+      _fieldId(fieldId),
+      _isHeaderField(headerField)
+{
+}
+
+Field::Field(const vespalib::stringref & name, int fieldId, int fieldIdV6,
+             const DataType& dataType, bool headerField)
+    : FieldBase(name),
+      _dataType(&dataType),
+      _fieldIdV6(fieldIdV6),
+      _fieldId(fieldId),
+      _isHeaderField(headerField)
+{
+}
+
+Field::Field(const vespalib::stringref & name,
+             const DataType& dataType, bool headerField)
+    : FieldBase(name),
+      _dataType(&dataType),
+      _fieldIdV6(calculateIdV6()),
+      _fieldId(calculateIdV7()),
+      _isHeaderField(headerField)
+{
+}
+
+vespalib::string
+Field::toString(bool verbose) const
+{
+    vespalib::asciistream out;
+    out << "Field(" << getName();
+    if (verbose) {
+        out << ", id " << _fieldId;
+    }
+    out << ", " << _dataType->toString();
+    if (verbose) {
+        out << ", " << (_isHeaderField ? "header" : "body");
+    }
+    out << ")";
+    return out.str();
+}
+
+bool
+Field::contains(const FieldSet& fields) const
+{
+    switch (fields.getType()) {
+    case FIELD:
+        return static_cast<const Field&>(fields).getId(7) == getId(7);
+    case SET:
+    {
+        // Go through each.
+        return false;
+    }
+    case NONE:
+    case DOCID:
+        return true;
+    case HEADER:
+    case BODY:
+    case ALL:
+        return false;
+    }
+
+    return false;
+}
+
+int
+Field::calculateIdV6()
+{
+    int newId =
+        vespalib::BobHash::hash(getName().c_str(), getName().size(), 0);
+        // Highest bit is reserved to tell 7-bit id's from 31-bit ones
+    if (newId < 0) newId = -newId;
+    validateId(newId);
+    return newId;
+}
+
+int
+Field::calculateIdV7()
+{
+    vespalib::asciistream ost;
+    ost << getName();
+    ost << _dataType->getId();
+
+    int newId = vespalib::BobHash::hash(
+            ost.str().c_str(), ost.str().length(), 0);
+        // Highest bit is reserved to tell 7-bit id's from 31-bit ones
+    if (newId < 0) newId = -newId;
+    validateId(newId);
+    return newId;
+}
+
+void
+Field::validateId(int newId) {
+    if (newId >= 100 && newId <= 127) {
+        throw vespalib::IllegalArgumentException(vespalib::make_string(
+                    "Attempt to set the id of %s to %d failed, values from "
+                    "100 to 127 are reserved for internal use",
+                    getName().c_str(), newId));
+    }
+
+    if ((newId & 0x80000000) != 0) // Highest bit must not be set
+    {
+        throw vespalib::IllegalArgumentException(vespalib::make_string(
+                    "Attempt to set the id of %s to %d"
+                    " failed, negative id values are illegal",
+                    getName().c_str(), newId));
+    }
+}
+
+} // document
diff --git a/document/src/vespa/document/base/field.h b/document/src/vespa/document/base/field.h
new file mode 100644
index 00000000000..b2583b0871b
--- /dev/null
+++ b/document/src/vespa/document/base/field.h
@@ -0,0 +1,101 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * \class document::Field
+ * \ingroup base
+ *
+ * \brief Specifies a field within a structured data type.
+ *
+ * A structured data type contains a key - value mapping of predefined
+ * data types. The field class is the key in these maps, and contains
+ * an identifier, in addition to datatype of values.
+ */
+#pragma once
+
+#include <memory>
+#include <vespa/document/datatype/datatype.h>
+#include <vespa/document/fieldvalue/fieldvalue.h>
+#include <vespa/document/fieldset/fieldset.h>
+#include <set>
+
+namespace document {
+
+class Field : public vespalib::FieldBase,
+              public vespalib::Identifiable,
+              public FieldSet
+{
+    const DataType *_dataType;
+
+    int _fieldIdV6; // Field ID for document protocol versions 1-6.
+    int _fieldId;
+
+    bool _isHeaderField;
+public:
+    typedef std::shared_ptr<const Field> CSP;
+    typedef std::shared_ptr<Field> SP;
+
+    struct FieldPtrComparator {
+        bool operator()(const Field* f1, const Field* f2) const {
+            return (*f1 < *f2);
+        }
+    };
+
+    typedef std::set<const Field*, FieldPtrComparator> Set;
+
+    /**
+     * Creates a completely specified field instance.
+     *
+     * @param name The name of the field.
+     * @param fieldId The numeric ID representing the field.
+     * @param type The datatype of the field.
+     * @param headerField Whether or not this is a "header" field.
+     */
+    Field(const vespalib::stringref & name, int fieldId,
+          const DataType &type, bool headerField);
+
+    Field(const vespalib::stringref & name, int fieldId, int fieldIdV6,
+          const DataType &type, bool headerField);
+    Field() :
+        FieldBase(""),
+        _dataType(DataType::INT),
+        _fieldIdV6(0),
+        _fieldId(0),
+        _isHeaderField(false)
+    {
+    }
+
+    /**
+     * Creates a completely specified field instance. Field ids are generated
+     * by hash function.
+     *
+     * @param name The name of the field.
+     * @param dataType The datatype of the field.
+     * @param headerField Whether or not this is a "header" field.
+     */
+    Field(const vespalib::stringref & name, const DataType &dataType, bool headerField);
+
+    FieldSet* clone() const override { return new Field(*this); }
+    FieldValue::UP createValue() const { return _dataType->createFieldValue(); }
+
+    // Note that only id is checked for equality.
+    bool operator==(const Field & other) const { return (_fieldId == other._fieldId); }
+    bool operator!=(const Field & other) const { return (_fieldId != other._fieldId); }
+    bool operator<(const Field & other) const { return (getName() < other.getName()); }
+
+    const DataType &getDataType() const { return *_dataType; }
+
+    int getId() const { return getId(7); }
+    int getId(int version) const { return (version > 6) ? _fieldId : _fieldIdV6; }
+    bool isHeaderField() const { return _isHeaderField; }
+
+    vespalib::string toString(bool verbose=false) const;
+    bool contains(const FieldSet& fields) const;
+    Type getType() const { return FIELD; }
+private:
+    int calculateIdV6();
+    int calculateIdV7();
+
+    void validateId(int newId);
+};
+
+} // document
+
diff --git a/document/src/vespa/document/base/fieldpath.cpp b/document/src/vespa/document/base/fieldpath.cpp
new file mode 100644
index 00000000000..3647fe55f3a
--- /dev/null
+++ b/document/src/vespa/document/base/fieldpath.cpp
@@ -0,0 +1,218 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/document/base/fieldpath.h>
+#include <vespa/document/base/field.h>
+#include <vespa/document/datatype/arraydatatype.h>
+#include <vespa/document/datatype/mapdatatype.h>
+#include <vespa/document/datatype/weightedsetdatatype.h>
+#include <vespa/document/datatype/primitivedatatype.h>
+#include <vespa/vespalib/objects/visit.h>
+
+using vespalib::IllegalArgumentException;
+using vespalib::make_string;
+
+namespace document {
+
+IMPLEMENT_IDENTIFIABLE_NS(document, FieldPathEntry, vespalib::Identifiable)
+
+FieldPathEntry::FieldPathEntry() :
+    _type(NONE),
+    _name(""),
+    _fieldRef(),
+    _dataType(0),
+    _lookupIndex(0),
+    _lookupKey(),
+    _variableName(),
+    _fillInVal()
+{
+}
+
+FieldPathEntry::FieldPathEntry(const DataType & dataType, uint32_t arrayIndex) :
+    _type(ARRAY_INDEX),
+    _name(""),
+    _fieldRef(),
+    _dataType(&dataType),
+    _lookupIndex(arrayIndex),
+    _lookupKey(),
+    _variableName(),
+    _fillInVal()
+{
+    setFillValue(*_dataType);
+}
+
+FieldPathEntry::FieldPathEntry(const Field &fieldRef) :
+    _type(STRUCT_FIELD),
+    _name(fieldRef.getName()),
+    _fieldRef(new Field(fieldRef)),
+    _dataType(&fieldRef.getDataType()),
+    _lookupIndex(0),
+    _lookupKey(),
+    _variableName(),
+    _fillInVal(fieldRef.createValue().release())
+{
+}
+
+FieldPathEntry::FieldPathEntry(const DataType & dataType, const DataType& fillType,
+                               const FieldValueCP & lookupKey) :
+    _type(MAP_KEY),
+    _name("value"),
+    _fieldRef(),
+    _dataType(&dataType),
+    _lookupIndex(0),
+    _lookupKey(lookupKey),
+    _variableName(),
+    _fillInVal()
+{
+    setFillValue(fillType);
+}
+
+void FieldPathEntry::setFillValue(const DataType & dataType)
+{
+    const DataType * dt = & dataType;
+    while (dt->inherits(CollectionDataType::classId) || dt->inherits(MapDataType::classId)) {
+        dt = dt->inherits(CollectionDataType::classId)
+             ? &static_cast<const CollectionDataType *>(dt)->getNestedType()
+             : &static_cast<const MapDataType *>(dt)->getValueType();
+    }
+    if (dt->inherits(PrimitiveDataType::classId)) {
+        _fillInVal.reset(dt->createFieldValue().release());
+    }
+}
+
+FieldPathEntry::FieldPathEntry(const DataType&, const DataType& keyType,
+                               const DataType& valueType, bool keysOnly, bool valuesOnly) :
+    _type(keysOnly ? MAP_ALL_KEYS : MAP_ALL_VALUES),
+    _name(keysOnly ? "key" : "value"),
+    _fieldRef(),
+    _dataType(keysOnly ? &keyType : &valueType),
+    _lookupIndex(0),
+    _lookupKey(),
+    _variableName(),
+    _fillInVal()
+{
+    (void)valuesOnly;
+    setFillValue(*_dataType);
+}
+
+FieldPathEntry::FieldPathEntry(const DataType & dataType, const vespalib::stringref & variableName) :
+    _type(VARIABLE),
+    _name(""),
+    _fieldRef(),
+    _dataType(&dataType),
+    _lookupIndex(0),
+    _lookupKey(),
+    _variableName(variableName),
+    _fillInVal()
+{
+    setFillValue(*_dataType);
+}
+
+const DataType &FieldPathEntry::getDataType() const
+{
+     return _fieldRef.get() ? _fieldRef->getDataType()
+                            : *_dataType;
+}
+
+void
+FieldPathEntry::visitMembers(vespalib::ObjectVisitor &visitor) const
+{
+    visit(visitor, "type", _type);
+    visit(visitor, "name", _name);
+    visit(visitor, "fieldRef", _fieldRef);
+    visit(visitor, "dataType", _dataType);
+    visit(visitor, "lookupIndex", _lookupIndex);
+    visit(visitor, "lookupKey", _lookupKey);
+    visit(visitor, "variableName", _variableName);
+    visit(visitor, "fillInVal", _fillInVal);
+}
+
+vespalib::string FieldPathEntry::parseKey(vespalib::string & key)
+{
+    vespalib::string v;
+    const char *c = key.c_str();
+    const char *e = c + key.size();
+    for(;(c < e) && isspace(c[0]); c++);
+    if ((c < e) && (c[0] == '{')) {
+        for(c++;(c < e) && isspace(c[0]); c++);
+        if ((c < e) && (c[0] == '"')) {
+            for (c++; (c < e) && (c[0] != '"'); c++) {
+                if (c[0] == '\\') {
+                    c++;
+                }
+                if (c < e) {
+                    v += c[0];
+                }
+            }
+            if ((c < e) && (c[0] == '"')) {
+                c++;
+            } else {
+                throw IllegalArgumentException(make_string("Escaped key '%s' is incomplete. No matching '\"'", key.c_str()), VESPA_STRLOC);
+            }
+        } else {
+            for (;(c < e) && (c[0] != '}'); c++) {
+                v += c[0];
+            }
+        }
+        for(;(c < e) && isspace(c[0]); c++);
+        if ((c < e) && (c[0] == '}')) {
+            key = c+1;
+        } else {
+            throw IllegalArgumentException(make_string("Key '%s' is incomplete. No matching '}'", key.c_str()), VESPA_STRLOC);
+        }
+    } else {
+        throw IllegalArgumentException(make_string("key '%s' does not start with '{'", key.c_str()), VESPA_STRLOC);
+    }
+    return v;
+}
+
+FieldPath::FieldPath()
+    : Cloneable(), _path()
+{
+}
+
+FieldPath::FieldPath(const FieldPath& other)
+    : Cloneable(), _path(other._path)
+{
+}
+
+FieldPath&
+FieldPath::operator=(const FieldPath& rhs)
+{
+    if (&rhs != this) {
+        _path = rhs._path;
+    }
+    return *this;
+}
+
+FieldPath::iterator
+FieldPath::insert(iterator pos, const FieldPathEntry& entry)
+{
+    return _path.insert(pos, entry);
+}
+void
+FieldPath::push_back(const FieldPathEntry& entry)
+{
+    _path.push_back(entry);
+}
+
+void
+FieldPath::pop_back()
+{
+    _path.pop_back();
+}
+
+void
+FieldPath::clear()
+{
+    _path.clear();
+}
+
+void
+FieldPath::visitMembers(vespalib::ObjectVisitor& visitor) const
+{
+    for (uint32_t i = 0; i < _path.size(); ++i) {
+        visit(visitor, vespalib::make_string("[%u]", i), _path[i]);
+    }
+}
+
+}
diff --git a/document/src/vespa/document/base/fieldpath.h b/document/src/vespa/document/base/fieldpath.h
new file mode 100644
index 00000000000..e452d7d0294
--- /dev/null
+++ b/document/src/vespa/document/base/fieldpath.h
@@ -0,0 +1,171 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vector>
+#include <vespa/vespalib/objects/cloneable.h>
+#include <vespa/vespalib/util/linkedptr.h>
+#include <memory>
+#include <vespa/document/util/identifiableid.h>
+
+namespace vespalib {
+class ObjectVisitor;
+}
+
+namespace document {
+
+class FieldValue;
+class DataType;
+class MapDataType;
+class WeightedSetDataType;
+class ArrayDataType;
+class Field;
+
+class FieldPathEntry : public vespalib::Identifiable {
+public:
+    DECLARE_IDENTIFIABLE_NS(document, FieldPathEntry);
+    enum Type {
+        STRUCT_FIELD,
+        ARRAY_INDEX,
+        MAP_KEY,
+        MAP_ALL_KEYS,
+        MAP_ALL_VALUES,
+        VARIABLE,
+        NONE
+    };
+    typedef std::shared_ptr<const Field> FieldSP;
+    typedef vespalib::CloneablePtr<DataType> DataTypeCP;
+    typedef vespalib::CloneablePtr<FieldValue> FieldValueCP;
+    typedef vespalib::LinkedPtr<FieldPathEntry> LP;
+
+    /**
+       Creates a empty field path entry.
+    */
+    FieldPathEntry();
+
+    /**
+       Creates a field path entry for a struct field lookup.
+    */
+    FieldPathEntry(const Field &fieldRef);
+
+    /**
+       Creates a field path entry for an array lookup.
+    */
+    FieldPathEntry(const DataType & dataType, uint32_t index);
+
+    /**
+       Creates a field path entry for a map or wset key lookup.
+    */
+    FieldPathEntry(const DataType & dataType, const DataType& fillType,
+                   const FieldValueCP & lookupKey);
+
+    /**
+       Creates a field path entry for a map key or value only traversal.
+    */
+    FieldPathEntry(const DataType & dataType, const DataType& keyType,
+                   const DataType& valueType, bool keysOnly, bool valuesOnly);
+
+    /**
+       Creates a field entry for an array, map or wset traversal using a variable.
+    */
+    FieldPathEntry(const DataType & dataType, const vespalib::stringref & variableName);
+
+    Type getType() const { return _type; }
+    const vespalib::string & getName() const { return _name; }
+
+    const DataType& getDataType() const;
+
+    bool hasField() const { return _fieldRef.get(); }
+    const Field & getFieldRef() const { return *_fieldRef; }
+
+    uint32_t getIndex() const { return _lookupIndex; }
+
+    const FieldValueCP & getLookupKey() const { return _lookupKey; }
+
+    const vespalib::string& getVariableName() const { return _variableName; }
+
+    FieldValue * getFieldValueToSetPtr() const { return _fillInVal.get(); }
+    FieldValue& getFieldValueToSet() const { return *_fillInVal; }
+    virtual void visitMembers(vespalib::ObjectVisitor &visitor) const;
+    /**
+     * Parses a string of the format {["]escaped string["]} to its unescaped value.
+     * @param key is the incoming value, and contains what is left when done.
+     * *return The unescaped value
+     */
+    static vespalib::string parseKey(vespalib::string & key);
+private:
+    void setFillValue(const DataType & dataType);
+    Type              _type;
+    vespalib::string  _name;
+    FieldSP           _fieldRef;
+    const DataType *  _dataType;
+    uint32_t          _lookupIndex;
+    FieldValueCP      _lookupKey;
+    vespalib::string  _variableName;
+    mutable FieldValueCP _fillInVal;
+};
+
+//typedef std::deque<FieldPathEntry> FieldPath;
+// Facade over FieldPathEntry container that exposes cloneability
+class FieldPath : public vespalib::Cloneable {
+    typedef std::vector<FieldPathEntry> Container;
+public:
+    typedef Container::reference reference;
+    typedef Container::const_reference const_reference;
+    typedef Container::iterator iterator;
+    typedef Container::const_iterator const_iterator;
+    typedef Container::reverse_iterator reverse_iterator;
+    typedef Container::const_reverse_iterator const_reverse_iterator;
+    typedef std::unique_ptr<FieldPath> UP;
+
+    FieldPath();
+    FieldPath(const FieldPath& other);
+    FieldPath& operator=(const FieldPath& rhs);
+
+    template <typename InputIterator>
+    FieldPath(InputIterator first, InputIterator last)
+        : _path(first, last)
+    {
+    }
+
+    iterator insert(iterator pos, const FieldPathEntry& entry);
+    void push_back(const FieldPathEntry& entry);
+
+    iterator begin() { return _path.begin(); }
+    iterator end() { return _path.end(); }
+    const_iterator begin() const { return _path.begin(); }
+    const_iterator end() const { return _path.end(); }
+    reverse_iterator rbegin() { return _path.rbegin(); }
+    reverse_iterator rend() { return _path.rend(); }
+    const_reverse_iterator rbegin() const { return _path.rbegin(); }
+    const_reverse_iterator rend() const { return _path.rend(); }
+
+    reference front() { return _path.front(); }
+    const_reference front() const { return _path.front(); }
+    reference back() { return _path.back(); }
+    const_reference back() const { return _path.back(); }
+
+    void pop_back();
+    void clear();
+
+    Container::size_type size() const { return _path.size(); }
+    bool empty() const { return _path.empty(); }
+    reference operator[](Container::size_type i) {
+        return _path[i];
+    }
+
+    const_reference operator[](Container::size_type i) const {
+        return _path[i];
+    }
+
+    FieldPath* clone() const {
+        return new FieldPath(*this);
+    }
+
+    virtual void visitMembers(vespalib::ObjectVisitor &visitor) const;
+
+private:
+    Container _path;
+};
+
+}
+
diff --git a/document/src/vespa/document/base/forcelink.cpp b/document/src/vespa/document/base/forcelink.cpp
new file mode 100644
index 00000000000..f42ae0fe1c8
--- /dev/null
+++ b/document/src/vespa/document/base/forcelink.cpp
@@ -0,0 +1,26 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/document/base/forcelink.h>
+
+#include <vespa/document/update/updates.h>
+#include <vespa/document/fieldvalue/fieldvalues.h>
+
+namespace document {
+
+ForceLink::ForceLink(void)
+{
+    if (time(NULL) == 0) {
+        DocumentType          type("foo", 1);
+        Document              document(type, DocumentId("doc:ns:bar"));
+        DocumentUpdate        documentUpdate(type, DocumentId("doc:ns:bar"));
+        MapValueUpdate        mapValueUpdate(IntFieldValue(3), ClearValueUpdate());
+        AddValueUpdate        addValueUpdate(IntFieldValue(3));
+        RemoveValueUpdate     removeValueUpdate(IntFieldValue(3));
+        AssignValueUpdate     assignValueUpdate(IntFieldValue(3));
+        ClearValueUpdate      clearValueUpdate;
+        ArithmeticValueUpdate arithmeticValueUpdate(ArithmeticValueUpdate::Add, 3);
+    }
+}
+
+}
+
diff --git a/document/src/vespa/document/base/forcelink.h b/document/src/vespa/document/base/forcelink.h
new file mode 100644
index 00000000000..81fadaedcfd
--- /dev/null
+++ b/document/src/vespa/document/base/forcelink.h
@@ -0,0 +1,21 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * \class document::ForceLink
+ * \ingroup base
+ *
+ * \brief Class used to include some document functionality that programs that
+ *        needs to get it linked, but doesn't use it can include.
+ *
+ * Many codebits include this, but who really depends on it?
+ */
+#pragma once
+
+namespace document {
+
+class ForceLink {
+public:
+    ForceLink();
+};
+
+} // document
+
diff --git a/document/src/vespa/document/base/globalid.cpp b/document/src/vespa/document/base/globalid.cpp
new file mode 100644
index 00000000000..68e83b23b09
--- /dev/null
+++ b/document/src/vespa/document/base/globalid.cpp
@@ -0,0 +1,198 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".globalid");
+
+#include <vespa/document/base/globalid.h>
+#include <vespa/document/bucket/bucketid.h>
+#include <vespa/vespalib/util/exceptions.h>
+#include <vespa/vespalib/stllike/asciistream.h>
+
+namespace {
+
+bool
+validateHex(char c)
+{
+    return
+        (c >= '0' && c <= '9') ||
+        (c >= 'a' && c <= 'f') ||
+        (c >= 'A' && c <= 'F');
+}
+
+uint16_t
+getHexVal(char c)
+{
+    if (c >= '0' && c <= '9') {
+        return (c - '0');
+    } else if (c >= 'a' && c <= 'f') {
+        return (c - 'a' + 10);
+    } else if (c >= 'A' && c <= 'F') {
+        return (c - 'A' + 10);
+    }
+    LOG_ASSERT(validateHex(c));
+    abort();
+    abort();
+}
+
+}
+
+namespace document {
+
+bool
+GlobalId::BucketOrderCmp::operator()(const GlobalId &lhs, const GlobalId &rhs) const
+{
+    const unsigned char * __restrict__ a = lhs._gid._buffer;
+    const unsigned char * __restrict__ b = rhs._gid._buffer;
+    int diff;
+    if ((diff = compare(a[0], b[0])) != 0) {
+        return diff < 0;
+    }
+    if ((diff = compare(a[1], b[1])) != 0) {
+        return diff < 0;
+    }
+    if ((diff = compare(a[2], b[2])) != 0) {
+        return diff < 0;
+    }
+    if ((diff = compare(a[3], b[3])) != 0) {
+        return diff < 0;
+    }
+    if ((diff = compare(a[8], b[8])) != 0) {
+        return diff < 0;
+    }
+    if ((diff = compare(a[9], b[9])) != 0) {
+        return diff < 0;
+    }
+    if ((diff = compare(a[10], b[10])) != 0) {
+        return diff < 0;
+    }
+    if ((diff = compare(a[11], b[11])) != 0) {
+        return diff < 0;
+    }
+    return lhs < rhs;
+}
+
+vespalib::string GlobalId::toString() const {
+    vespalib::asciistream out;
+    out << "gid(0x" << vespalib::hex;
+    for (int i = 0; i < (int)LENGTH; ++i) {
+        unsigned short s1 = _gid._buffer[i] >> 4;
+        unsigned short s2 = _gid._buffer[i] & 0xF;
+        out << s1 << s2;
+    }
+    out << ")" << vespalib::dec;
+    return out.str();
+}
+
+GlobalId
+GlobalId::parse(const vespalib::stringref & source)
+{
+    if (source.substr(0, 6) != "gid(0x") {
+        throw vespalib::IllegalArgumentException(
+                "A gid must start with \"gid(0x\". Invalid source: '" + source
+                + "'.", VESPA_STRLOC);
+    }
+    if (source.size() != 2 * LENGTH + 7) {
+        std::ostringstream ost;
+        ost << "A gid string representation must be exactly "
+            << (2 * LENGTH + 7) << " bytes long. Invalid source: '"
+            << source << "'.";
+        throw vespalib::IllegalArgumentException(ost.str(), VESPA_STRLOC);
+    }
+    if (source.substr(2 * LENGTH + 6, 1) != ")") {
+        throw vespalib::IllegalArgumentException(
+                "A gid must end in \")\". Invalid source: '" + source
+                + "'.", VESPA_STRLOC);
+    }
+    GlobalId id;
+    for (uint32_t i = 0; i < LENGTH; ++i) {
+        char c1 = source[6 + 2*i];
+        char c2 = source[6 + 2*i + 1];
+        if (!validateHex(c1) || !validateHex(c2)) {
+            throw vespalib::IllegalArgumentException(
+                    "A gid can only contain hexidecimal characters [0-9a-fA-F]."
+                    " Invalid source: '" + source + "'.", VESPA_STRLOC);
+        }
+        id._gid._buffer[i] = (getHexVal(c1) << 4) | getHexVal(c2);
+        /*
+        std::cerr << "Hexval of " << source[6 + 2*i] << " and "
+                  << source[6 + 2*i + 1] << " is " << getHexVal(c1) << " and "
+                  << getHexVal(c2) << ", which forms the byte "
+                  << (int) id._gid._buffer[i] << "\n";
+        */
+    }
+    return id;
+}
+
+bool
+GlobalId::containedInBucket(const BucketId &bucket) const
+{
+    return bucket.contains(convertToBucketId());
+}
+
+GlobalId
+GlobalId::calculateFirstInBucket(const BucketId& bucket)
+{
+    //std::cerr << "First: Calculating gid from " << bucket << "\n" << std::hex;
+    BucketId::Type location, gid;
+    uint32_t usedBits(bucket.getUsedBits());
+    if (usedBits > 32) {
+        BucketId::Type gidMask(0x03ffffff00000000ull);
+        BucketId::Type locationMask(0x00000000ffffffffull);
+        uint32_t usedGidBits = usedBits - 32;
+        gidMask = gidMask << (32 - usedGidBits) >> (32 - usedGidBits);
+        gid = bucket.getRawId() & gidMask;
+        location = bucket.getRawId() & locationMask;
+    } else {
+        BucketId::Type locationMask(0x00000000ffffffffull);
+        locationMask = locationMask << (64 - usedBits) >> (64 - usedBits);
+        gid = 0;
+        location = bucket.getRawId() & locationMask;
+    }
+    //std::cerr << "First: Got location " << location << " and gid " << gid
+    //          << "\n";
+    uint8_t raw[GlobalId::LENGTH];
+    memcpy(&raw[0], &location, 4);
+    memcpy(&raw[0] + 4, &gid, 8);
+    //std::cerr << "First: " << GlobalId(raw) << ", bucket "
+    //          << GlobalId(raw).convertToBucketId() << "\n" << std::dec;
+    return GlobalId(raw);
+}
+
+GlobalId
+GlobalId::calculateLastInBucket(const BucketId& bucket)
+{
+    //std::cerr << "Last: Calculating gid from " << bucket << "\n" << std::hex;
+    BucketId::Type location, gid;
+    uint32_t usedBits(bucket.getUsedBits());
+    if (usedBits > 32) {
+        BucketId::Type gidMask(0x03ffffff00000000ull);
+        BucketId::Type locationMask(0x00000000ffffffffull);
+        uint32_t usedGidBits = usedBits - 32;
+        gidMask = gidMask << (32 - usedGidBits) >> (32 - usedGidBits);
+        BucketId::Type gidRevMask(gidMask ^ 0xffffffffffffffffull);
+        gid = (bucket.getRawId() & gidMask) | gidRevMask;
+        location = bucket.getRawId() & locationMask;
+    } else {
+        BucketId::Type locationMask(0x00000000ffffffffull);
+        locationMask = locationMask << (64 - usedBits) >> (64 - usedBits);
+        BucketId::Type locationRevMask(locationMask ^ 0xffffffffffffffffull);
+        gid = 0xffffffffffffffffull;
+        location = (bucket.getRawId() & locationMask) | locationRevMask;
+    }
+    //std::cerr << "Last: Got location " << location << " and gid " << gid
+    //          << "\n";
+    uint8_t raw[GlobalId::LENGTH];
+    memcpy(&raw[0], &location, 4);
+    memcpy(&raw[0] + 4, &gid, 8);
+    //std::cerr << "Last: " << GlobalId(raw) << ", bucket "
+    //          << GlobalId(raw).convertToBucketId() << "\n" << std::dec;
+    return GlobalId(raw);
+}
+
+void
+GlobalId::print(std::ostream& out) const
+{
+    out << toString();
+}
+
+} // document
diff --git a/document/src/vespa/document/base/globalid.h b/document/src/vespa/document/base/globalid.h
new file mode 100644
index 00000000000..f87e989b135
--- /dev/null
+++ b/document/src/vespa/document/base/globalid.h
@@ -0,0 +1,237 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * \class document::GlobalId
+ * \ingroup base
+ *
+ * \brief Representation of a global ID.
+ *
+ * The global ID, is a hash of the document ID, used where we need to
+ * distinguish between documents, but where storing a variable length string is
+ * not practical. VESPA search currently assumes that there will be no hash
+ * collisions, and if one should occur, the latest document will be the only
+ * one present in the indexes. VESPA document storage handles global ID
+ * collisions, but optimize code for the instances where it doesn't happen.
+ *
+ * It's a 96 bit MD5 checksum, so the chances of a collision is very very small.
+ *
+ * This class should not inherit from anything, as clients may use it's memory
+ * representation directly to save it.
+ *
+ * The interface for creating or modifying these objects is not user friendly.
+ * You should create it by calling DocumentId::getGlobalId() and you should not
+ * need to modify it, unless by standard copy constructor or assignment
+ * operator.
+ */
+#pragma once
+
+#include <iostream>
+#include <stdint.h>
+#include <vespa/vespalib/stllike/string.h>
+#include <vespa/document/bucket/bucketid.h>
+
+namespace document {
+
+class GlobalId {
+public:
+    /**
+     * The number of bytes used to represent a global id.
+     */
+    static const unsigned int LENGTH = 12;
+
+    /** Hash function that can be used to put global ids in hash set/maps. */
+    struct hash {
+        size_t operator () (const GlobalId & g) const {
+            return g._gid._bucketId._gid;
+        }
+    };
+
+private:
+    struct BucketIdS {
+        uint32_t _location;
+        uint64_t _gid;
+    } __attribute__((packed));
+    union {
+        unsigned char _buffer[LENGTH];
+        BucketIdS     _bucketId;
+        uint32_t      _nums[LENGTH/sizeof(uint32_t)];
+    } _gid;
+
+public:
+    /**
+     * Defines a comparator object that can be used for sorting global ids based on bucket order. An std::map
+     * using this comparator to order gid keys can use
+     * map.lower_bound(GlobalId::calculateFirstInBucket(bucket)) and
+     * map.upper_bound(GlobalId::calculateLastInBucket(bucket)) to traverse only those gids that belong to the
+     * given bucket.
+     */
+    struct BucketOrderCmp {
+        bool operator()(const GlobalId &lhs, const GlobalId &rhs) const;
+        //These 2 compare methods are exposed only for testing
+        static int compareRaw(unsigned char a, unsigned char b) {
+            return a - b;
+        }
+        static int compare(unsigned char a, unsigned char b) {
+            return compareRaw(document::reverseBitTable[a], document::reverseBitTable[b]);
+        }
+    };
+
+    /**
+     * Constructs a new global id with all 0 bits.
+     */
+    GlobalId() { set("\0\0\0\0\0\0\0\0\0\0\0\0"); }
+
+
+
+    /**
+     * Constructs a new global id with initial content. This copies the first LENGTH bytes from the given
+     * address into the internal buffer.
+     *
+     * @param gid The address to the data to copy.
+     */
+    explicit GlobalId(const void *gid) { set(gid); }
+
+    /**
+     * Implements the assignment operator.
+     *
+     * @param other The global id whose value to copy to this.
+     * @return This.
+     */
+    GlobalId& operator=(const GlobalId& other) { memcpy(_gid._buffer, other._gid._buffer, sizeof(_gid._buffer)); return *this; }
+
+    /**
+     * Implements the equality operator.
+     *
+     * @param other The global id to compare to.
+     * @return True if this equals the other, false otherwise.
+     */
+    bool operator==(const GlobalId& other) const { return (memcmp(_gid._buffer, other._gid._buffer, sizeof(_gid._buffer)) == 0); }
+
+    /**
+     * Implements the inequality operator.
+     *
+     * @param other The global id to compare to.
+     * @return True if this does NOT equal the other, false otherwise.
+     */
+    bool operator!=(const GlobalId& other) const { return (memcmp(_gid._buffer, other._gid._buffer, sizeof(_gid._buffer)) != 0); }
+
+    /**
+     * Implements the less-than operator. If you intend to map global ids in such a way that they can
+     * efficiently be looked up based on corresponding bucket ids, you should use the {@link BucketOrderCmp}
+     * for ordering your collection.
+     *
+     * @param other The global id to compare to.
+     * @return True if comparing the bits of this to the other yields a negative result.
+     */
+    bool operator<(const GlobalId& other) const { return (memcmp(_gid._buffer, other._gid._buffer, sizeof(_gid._buffer)) < 0); }
+
+    /**
+     * Copies the first LENGTH bytes from the given address into the internal byte array.
+     *
+     * @param id The bytes to set.
+     */
+    void set(const void *id) { memcpy(_gid._buffer, id, sizeof(_gid._buffer)); }
+
+    /**
+     * Returns the raw byte array that constitutes this global id.
+     */
+    const unsigned char *get() const { return _gid._buffer; }
+
+    /**
+     * If a GID has been generated from a document ID with a location (n=, g=),
+     * the returned value will be deterministic based on the location, and two
+     * different document IDs with the same location will return the same value.
+     * If not, the value will not have any usable semantics but will still be
+     * deterministic in the sense that two identical document IDs will generate
+     * the same returned value.
+     */
+    uint32_t getLocationSpecificBits() const noexcept {
+        return _gid._bucketId._location;
+    }
+
+    /**
+     * Writes a string representation of this global id to the given output stream.
+     *
+     * @param out The stream to write to.
+     */
+    void print(std::ostream &out) const;
+
+    /**
+     * Returns a string representation of this global id.
+     */
+    vespalib::string toString() const;
+
+    /**
+     * Parse the source string to generate a global id object.  The source is expected to contain exactly what
+     * toString() creates from a global identifier.
+     *
+     * @param str The string to parse.
+     * @throws vespalib::IllegalArgumentException Thrown if input is not in GID format.
+     */
+    static GlobalId parse(const vespalib::stringref &str);
+
+    /**
+     * Returns the most specified bucket id to which this global id belongs.
+     *
+     * This function should not be used as it puts too strict limitations on
+     * what the bucket identifier can be.. Simplifying implementation to not
+     * use bucket id internals. Function should be removed before we can change
+     * bucket id.
+     *
+     * @return The bucket id.
+     */
+    BucketId convertToBucketId() const {
+        uint64_t location(_gid._bucketId._location);
+        uint64_t gid(_gid._bucketId._gid);
+        return BucketId(58, (gid & 0xFFFFFFFF00000000ull)
+                            | (location & 0xFFFFFFFF));
+    }
+
+    /**
+     * Returns whether or not this global id is contained in the given bucket.
+     *
+     * @param bucket The bucket to check.
+     * @return True, if this gid is contained in the bucket.
+     */
+    bool containedInBucket(const BucketId &bucket) const;
+
+    /**
+     * Given a list of global identifiers sorted in bucket order (see {@link BucketOrderCmp}), this function
+     * returns the global id that is the smallest that can exist in the given bucket.
+     *
+     * @param bucket The bucket id whose lowest gid to find.
+     * @return The first global id of the bucket.
+     */
+    static GlobalId calculateFirstInBucket(const BucketId &bucket);
+
+    /**
+     * Given a list of global identifiers sorted in bucket order (see {@link BucketOrderCmp}), this function
+     * returns the global id that is the largest that can exist in the given bucket.
+     *
+     * @param bucket The bucket id whose largest gid to find.
+     * @return The last global id of the bucket.
+     */
+    static GlobalId calculateLastInBucket(const BucketId &bucket);
+};
+
+/**
+ * Implements stream writing of a global id.
+ *
+ * @param out The stream to write to.
+ * @param gid The global id to write.
+ * @return The output stream.
+ */
+inline std::ostream&
+operator<<(std::ostream& out, const GlobalId& gid)
+{
+    gid.print(out);
+    return out;
+}
+
+inline vespalib::asciistream &
+operator << (vespalib::asciistream & os, const GlobalId & gid)
+{
+    return os << gid.toString();
+}
+
+} // document
+
diff --git a/document/src/vespa/document/base/idstring.cpp b/document/src/vespa/document/base/idstring.cpp
new file mode 100644
index 00000000000..e82deb8d049
--- /dev/null
+++ b/document/src/vespa/document/base/idstring.cpp
@@ -0,0 +1,430 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/document/base/idstring.h>
+
+#include <vespa/vespalib/util/md5.h>
+#include <stdlib.h>
+#include <vespa/document/bucket/bucketid.h>
+#include <vespa/vespalib/util/vstringfmt.h>
+#include <vespa/vespalib/util/optimized.h>
+
+using vespalib::string;
+using vespalib::stringref;
+using vespalib::make_string;
+
+namespace document {
+
+VESPA_IMPLEMENT_EXCEPTION(IdParseException, vespalib::Exception);
+
+namespace {
+
+string _G_typeName[6] = {
+    "doc",
+    "userdoc",
+    "groupdoc",
+    "orderdoc",
+    "id",
+    "null"
+};
+
+}
+
+const string &
+IdString::getTypeName(Type t)
+{
+    return _G_typeName[t];
+}
+
+string
+IdString::getSchemeName() const
+{
+    return getTypeName(getType());
+}
+
+const string &
+IdString::toString() const
+{
+    return _rawId;
+}
+
+namespace {
+
+void reportError(const char* part) __attribute__((noinline));
+void reportError(const stringref & s) __attribute__((noinline));
+void reportError(const stringref & s, const char* part) __attribute__((noinline));
+void reportTooShortDocId(const char * id, size_t sz) __attribute__((noinline));
+void reportNoSchemeSeparator(const char * id) __attribute__((noinline));
+
+void reportError(const char* part)
+{
+    throw IdParseException(make_string("Unparseable id: No %s separator ':' found", part), VESPA_STRLOC);
+}
+void reportError(const stringref & s, const char* part)
+{
+    throw IdParseException(make_string("Unparseable %s '%s': Not an unsigned 64-bit number", part,
+                                   string(s).c_str()), VESPA_STRLOC);
+}
+
+void reportError(const stringref & s)
+{
+    throw IdParseException(make_string("Unparseable order doc scheme '%s': Scheme must contain parameters on the form (width, division)",
+                                       string(s).c_str()), VESPA_STRLOC);
+}
+void reportNoSchemeSeparator(const char * id)
+{
+    throw IdParseException(make_string("Unparseable id '%s': No scheme separator ':' found", id), VESPA_STRLOC);
+}
+
+void reportTooShortDocId(const char * id, size_t sz)
+{
+    throw IdParseException(
+            make_string(
+                    "Unparseable id '%s': It is too short(%li) "
+                    "to make any sense", id, sz), VESPA_STRLOC);
+}
+
+uint64_t getAsNumber(const stringref & s, const char* part) {
+    char* errPos = NULL;
+    uint64_t value = strtoull(s.c_str(), &errPos, 10);
+
+    if (s.c_str() + s.size() != errPos) {
+        reportError(s, part);
+    }
+    return value;
+}
+
+void
+getOrderDocBits(const stringref& scheme, uint16_t & widthBits, uint16_t & divisionBits)
+{
+    const char* parenPos = reinterpret_cast<const char*>(
+            memchr(scheme.c_str(), '(', scheme.size()));
+    const char* endParenPos = reinterpret_cast<const char*>(
+            memchr(scheme.c_str(), ')', scheme.size()));
+
+    if (parenPos == NULL || endParenPos == NULL || endParenPos < parenPos) {
+        reportError(scheme);
+    }
+
+    const char* separatorPos = reinterpret_cast<const char*>(
+            memchr(parenPos + 1, ',', endParenPos - parenPos - 1));
+    if (separatorPos == NULL) {
+        reportError(scheme);
+    }
+
+    char* endptr = NULL;
+    // strtoul stops at first non-numeric char (in this case ',' and ')'), so don't
+    // require any extra zero-terminated buffers
+    errno = 0;
+    widthBits = static_cast<uint16_t>(strtoul(parenPos + 1, &endptr, 10));
+    if (errno != 0 || *endptr != ',') {
+        reportError(scheme);
+    }
+    errno = 0;
+    divisionBits = static_cast<uint16_t>(strtoul(separatorPos + 1, &endptr, 10));
+    if (errno != 0 || *endptr != ')') {
+        reportError(scheme);
+    }
+}
+
+union TwoByte {
+    char     asChar[2];
+    uint16_t as16;
+};
+
+union FourByte {
+    char     asChar[4];
+    uint32_t as32;
+};
+
+union EightByte {
+    char     asChar[8];
+    uint64_t as64;
+};
+
+const FourByte _G_doc = {{'d', 'o', 'c', ':'}};
+const FourByte _G_null = {{'n', 'u', 'l', 'l'}};
+const EightByte _G_userdoc = {{'u', 's', 'e', 'r', 'd', 'o', 'c', ':'}};
+const EightByte _G_groupdoc = {{'g', 'r', 'o', 'u', 'p', 'd', 'o', 'c'}};
+const EightByte _G_orderdoc = {{'o', 'r', 'd', 'e', 'r', 'd', 'o', 'c'}};
+const TwoByte _G_id = {{'i', 'd'}};
+
+typedef char v16qi __attribute__ ((__vector_size__(16)));
+
+v16qi _G_zero  = { ':', ':', ':', ':', ':', ':', ':', ':', ':', ':', ':', ':', ':', ':', ':', ':' };
+
+//const char * fmemchr_stdc(const char * s, const char * e) __attribute__((noinline));
+
+inline const char *
+fmemchr(const char * s, const char * e)
+{
+    while (s+15 < e) {
+        v16qi tmpCurrent = __builtin_ia32_loaddqu(s);
+        v16qi tmp0       = __builtin_ia32_pcmpeqb128(tmpCurrent, _G_zero);
+        uint32_t charMap = __builtin_ia32_pmovmskb128(tmp0); // 1 in charMap equals to '\0' in input buffer
+        if (__builtin_expect(charMap, 1)) {
+            return s + vespalib::Optimized::lsbIdx(charMap);
+        }
+        s+=16;
+    }
+
+    const char c(':');
+    while (s+3 < e) {
+        if (s[0] == c) {
+            return s;
+        }
+        if (s[1] == c) {
+            return s+1;
+        }
+        if (s[2] == c) {
+            return s+2;
+        }
+        if (s[3] == c) {
+            return s+3;
+        }
+        s+=4;
+    }
+    while (s < e) {
+        if (s[0] == c) {
+            return s;
+        }
+        s++;
+    }
+    return NULL;
+}
+
+}  // namespace
+
+
+IdString::Offsets::Offsets(uint32_t maxComponents, uint32_t namespaceOffset, const stringref & id)
+{
+    _offsets[0] = namespaceOffset;
+    size_t index(1);
+    const char * s(id.c_str() + namespaceOffset);
+    const char * e(id.c_str() + id.size());
+    for(s=fmemchr(s, e);
+        (s != NULL) && (index < maxComponents);
+        s = fmemchr(s+1, e))
+    {
+        _offsets[index++] = s - id.c_str() + 1;
+    }
+    _numComponents = index;
+    for (;index < VESPA_NELEMS(_offsets); index++) {
+        _offsets[index] = id.size() + 1; // 1 is added due to the implicitt accounting for ':'
+    }
+    _offsets[maxComponents] = id.size() + 1; // 1 is added due to the implicitt accounting for ':'
+}
+
+IdString::IdString(uint32_t maxComponents, uint32_t namespaceOffset, const stringref & rawId) :
+    _offsets(maxComponents, namespaceOffset, rawId),
+    _rawId(rawId)
+{
+}
+
+void
+IdString::validate() const
+{
+    if (_offsets.numComponents() < 2) {
+        reportError("namespace");
+    }
+}
+
+void
+IdIdString::validate() const
+{
+    IdString::validate();
+    if (getNumComponents() < 3) {
+        reportError("document type");
+    }
+    if (getNumComponents() < 4) {
+        reportError("key/value-pairs");
+    }
+}
+
+IdString::UP
+IdString::createIdString(const char * id, size_t sz_)
+{
+    if (sz_ > 4) {
+        if (_G_doc.as32 == *reinterpret_cast<const uint32_t *>(id)) {
+            return IdString::UP(new DocIdString(stringref(id, sz_)));
+        } else if ((sz_ == 6) && (_G_null.as32 == *reinterpret_cast<const uint32_t *>(id)) && (id[4] == ':') && (id[5] == ':')) {
+            return IdString::UP(new NullIdString());
+        } else if (_G_id.as16 == *reinterpret_cast<const uint16_t *>(id) && id[2] == ':') {
+            return IdString::UP(new IdIdString(stringref(id, sz_)));
+        } else if (sz_ > 8) {
+            if (_G_userdoc.as64 == *reinterpret_cast<const uint64_t *>(id)) {
+                return IdString::UP(new UserDocIdString(stringref(id, sz_)));
+            } else if (_G_groupdoc.as64 == *reinterpret_cast<const uint64_t *>(id) && (id[8] == ':')) {
+                return IdString::UP(new GroupDocIdString(stringref(id, sz_)));
+            } else if (_G_orderdoc.as64 == *reinterpret_cast<const uint64_t *>(id) && (id[8] == '(')) {
+                return IdString::UP(new OrderDocIdString(stringref(id, sz_)));
+            } else {
+                reportNoSchemeSeparator(id);
+            }
+        } else {
+            reportTooShortDocId(id, 8);
+        }
+    } else {
+        reportTooShortDocId(id, 5);
+    }
+    return IdString::UP();
+}
+
+namespace {
+union LocationUnion {
+    uint8_t _key[16];
+    IdString::LocationType _location[2];
+};
+
+IdString::LocationType makeLocation(const stringref &s) {
+    LocationUnion location;
+    fastc_md5sum(reinterpret_cast<const unsigned char*>(s.c_str()), s.size(), location._key);
+    return location._location[0];
+}
+
+uint64_t parseNumber(const stringref &number) {
+    char* errPos = NULL;
+    errno = 0;
+    uint64_t n = strtoul(number.c_str(), &errPos, 10);
+    if (*errPos) {
+        throw IdParseException(
+                "'n'-value must be a 64-bit number. It was " +
+                number, VESPA_STRLOC);
+    }
+    if (errno == ERANGE) {
+        throw IdParseException("'n'-value out of range "
+                               "(" + number + ")", VESPA_STRLOC);
+    }
+    return n;
+}
+
+void setLocation(IdString::LocationType &loc, IdString::LocationType val,
+                 bool &has_set_location, const stringref &key_values) {
+    if (has_set_location) {
+        throw IdParseException("Illegal key combination in "
+                               + key_values);
+    }
+    loc = val;
+    has_set_location = true;
+}
+
+
+}  // namespace
+
+IdIdString::IdIdString(const stringref & id)
+    : IdString(4, 3, id),
+      _location(0),
+      _groupOffset(0),
+      _has_number(false)
+{
+    // TODO(magnarn): Require that keys are lexicographically ordered.
+    validate();
+
+    stringref key_values(getComponent(2));
+    char key(0);
+    string::size_type pos = 0;
+    bool has_set_location = false;
+    for (string::size_type i = 0; i < key_values.size(); ++i) {
+        if (key_values[i] == '=') {
+            key = key_values[i-1];
+            pos = i + 1;
+        } else if (key_values[i] == ',' || i == key_values.size() - 1) {
+            stringref value(key_values.substr(pos, i - pos + (i == key_values.size() - 1)));
+            if (key == 'n') {
+                char tmp=value[value.size()];
+                const_cast<char &>(value[value.size()]) = 0;
+                setLocation(_location, parseNumber(value), has_set_location, key_values);
+                _has_number = true;
+                const_cast<char &>(value[value.size()]) = tmp;
+            } else if (key == 'g') {
+                setLocation(_location, makeLocation(value), has_set_location, key_values);
+                _groupOffset = offset(2) + pos;
+            } else {
+                throw IdParseException(make_string("Illegal key '%c'", key));
+            }
+            pos = i + 1;
+        }
+    }
+
+    if (!has_set_location) {
+        _location = makeLocation(getNamespaceSpecific());
+    }
+}
+
+IdString::LocationType
+DocIdString::getLocation() const
+{
+    return makeLocation(toString());
+}
+
+IdString::LocationType
+GroupDocIdString::getLocation() const
+{
+    return makeLocation(getGroup());
+}
+
+DocIdString::DocIdString(const stringref & ns, const stringref & id) :
+    IdString(2, 4, "doc:" + ns + ":" + id)
+{
+    validate();
+}
+
+DocIdString::DocIdString(const stringref & rawId) :
+    IdString(2, 4, rawId)
+{
+    validate();
+}
+
+UserDocIdString::UserDocIdString(const stringref & rawId) :
+    IdString(3, 8, rawId),
+    _userId(getAsNumber(rawId.substr(offset(1), offset(2) - offset(1) - 1), "userid"))
+{
+    validate();
+}
+
+GroupDocIdString::GroupDocIdString(const stringref & rawId) :
+    IdString(3, 9, rawId)
+{
+    validate();
+}
+
+IdString::LocationType
+GroupDocIdString::locationFromGroupName(vespalib::stringref name)
+{
+    return makeLocation(name);
+}
+
+OrderDocIdString::OrderDocIdString(const stringref & rawId) :
+    IdString(4, static_cast<const char *>(memchr(rawId.c_str(), ':', rawId.size())) - rawId.c_str() + 1, rawId),
+    _widthBits(0),
+    _divisionBits(0),
+    _ordering(getAsNumber(rawId.substr(offset(2), offset(3) - offset(2) - 1), "ordering"))
+{
+    validate();
+    getOrderDocBits(rawId.substr(0, offset(0) - 1), _widthBits, _divisionBits);
+   
+    string group(rawId.substr(offset(1), offset(2) - offset(1) - 1)); 
+    char* errStr = NULL;
+    _location = strtoull(group.c_str(), &errStr, 0);
+    if (*errStr != '\0') {
+        LocationUnion location;
+        fastc_md5sum((const unsigned char*) group.c_str(), group.size(), location._key);
+        _location = location._location[0];
+    }
+}
+
+std::pair<int16_t, int64_t>
+OrderDocIdString::getGidBitsOverride() const
+{
+    int usedBits = _widthBits - _divisionBits;
+    int64_t gidBits = (_ordering << (64 - _widthBits));
+    gidBits = BucketId::reverse(gidBits);
+    gidBits &= (std::numeric_limits<uint64_t>::max() >> (64 - usedBits));
+    return std::pair<int16_t, int64_t>(usedBits, gidBits);
+}
+
+string
+OrderDocIdString::getSchemeName() const {
+    return make_string("%s(%d,%d)", getTypeName(getType()).c_str(), _widthBits, _divisionBits);
+}
+
+} // document
diff --git a/document/src/vespa/document/base/idstring.h b/document/src/vespa/document/base/idstring.h
new file mode 100644
index 00000000000..ec7ec3c6ecf
--- /dev/null
+++ b/document/src/vespa/document/base/idstring.h
@@ -0,0 +1,243 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * @file idstring.h
+ *
+ * Contains the various URI schemes accepted in document identifiers.
+ */
+
+#pragma once
+
+#include <vespa/fastos/fastos.h>
+#include <memory>
+#include <vespa/vespalib/util/exception.h>
+#include <vespa/vespalib/util/memory.h>
+#include <vespa/vespalib/objects/cloneable.h>
+#include <vespa/vespalib/stllike/string.h>
+
+namespace document {
+
+/**
+ * \class document::IdParseException
+ * \ingroup base
+ *
+ * \brief Exception used to indicate failure to parse a %document identifier
+ * URI.
+ */
+VESPA_DEFINE_EXCEPTION(IdParseException, vespalib::Exception);
+
+/**
+ * \class document::IdString
+ * \ingroup base
+ *
+ * \brief Superclass for all document identifier schemes.
+ */
+class IdString : public vespalib::Cloneable {
+public:
+    typedef std::unique_ptr<IdString> UP;
+    typedef vespalib::CloneablePtr<IdString> CP;
+    typedef uint64_t LocationType;
+    enum Type { DOC=0, USERDOC, GROUPDOC, ORDERDOC, ID, NULLID };
+    static const vespalib::string & getTypeName(Type t);
+
+    /** @throws document::IdParseException If parsing of id scheme failed. */
+    static IdString::UP createIdString(const vespalib::stringref & id) { return createIdString(id.c_str(), id.size()); }
+    static IdString::UP createIdString(const char *id, size_t sz);
+
+    virtual ~IdString() {}
+    IdString* clone() const = 0;
+
+    virtual Type        getType() const = 0;
+    vespalib::stringref getNamespace() const { return getComponent(0); }
+    virtual vespalib::stringref getNamespaceSpecific() const = 0;
+    virtual LocationType getLocation() const = 0;
+    virtual std::pair<int16_t, int64_t> getGidBitsOverride() const { return std::pair<int16_t, int64_t>(0, 0); }
+    virtual bool hasDocType() const { return false; }
+    virtual vespalib::stringref getDocType() const { return ""; }
+    virtual bool hasNumber() const { return false; }
+    virtual uint64_t getNumber() const { return 0; }
+    virtual bool hasGroup() const { return false; }
+    virtual vespalib::stringref getGroup() const { return ""; }
+
+    bool operator==(const IdString& other) const
+        { return toString() == other.toString(); }
+
+    const vespalib::string & toString() const;
+
+protected:
+    IdString(uint32_t maxComponents, uint32_t namespaceOffset, const vespalib::stringref & rawId);
+    virtual vespalib::string getSchemeName() const;
+    size_t offset(size_t index) const { return _offsets[index]; }
+    size_t size(size_t index) const { return _offsets[index+1] - _offsets[index] - 1; }
+    vespalib::stringref getComponent(size_t index) const { return vespalib::stringref(_rawId.c_str() + offset(index), size(index)); }
+    const vespalib::string & getRawId() const { return _rawId; }
+    virtual void validate() const;
+    size_t getNumComponents() const { return _offsets.numComponents(); }
+
+private:
+    class Offsets {
+    public:
+        Offsets(uint32_t maxComponents, uint32_t first, const vespalib::stringref & id);
+        uint16_t first() const { return _offsets[0]; }
+        uint16_t operator [] (size_t i) const { return _offsets[i]; }
+        size_t numComponents() const { return _numComponents; }
+    private:
+        uint16_t _offsets[5];
+        uint32_t _numComponents;
+    };
+    Offsets _offsets;
+    uint32_t _nssComponentId;
+    vespalib::string _rawId;
+};
+
+class NullIdString : public IdString
+{
+public:
+    NullIdString() : IdString(2, 5, "null::") { }
+private:
+    IdString* clone() const { return new NullIdString(); }
+    virtual LocationType getLocation() const { return 0; }
+    virtual Type getType() const { return NULLID; }
+    virtual vespalib::stringref getNamespaceSpecific() const { return getComponent(1); }
+};
+
+/**
+ * \class document::IdIdString
+ * \ingroup base
+ *
+ * \brief New scheme for documents with no forced distribution.
+ *
+ * By using this scheme, documents will be evenly distributed within VDS,
+ * as the location of a doc identifier is a hash of the entire URI.
+ * This scheme also contains the DocumentType.
+ */
+class IdIdString : public IdString {
+    LocationType _location;
+    uint16_t     _groupOffset;
+    bool         _has_number;
+
+public:
+    IdIdString(const vespalib::stringref &ns);
+
+    virtual bool hasDocType() const { return true; }
+    virtual vespalib::stringref getDocType() const { return getComponent(1); }
+    virtual IdIdString* clone() const { return new IdIdString(*this); }
+    virtual LocationType getLocation() const { return _location; }
+    virtual bool hasNumber() const { return _has_number; }
+    virtual uint64_t getNumber() const { return _location; }
+    virtual bool hasGroup() const { return _groupOffset != 0; }
+
+    virtual vespalib::stringref getGroup() const { return vespalib::stringref(getRawId().c_str() + _groupOffset, offset(3) - _groupOffset - 1); }
+private:
+    virtual void validate() const;
+    virtual Type getType() const { return ID; }
+    virtual vespalib::stringref getNamespaceSpecific() const { return getComponent(3); }
+};
+
+/**
+ * \class document::DocIdString
+ * \ingroup base
+ *
+ * \brief Scheme for documents with no forced distribution.
+ *
+ * By using this scheme, documents will be evenly distributed within VDS,
+ * as the location of a doc identifier is a hash of the entire URI.
+ */
+class DocIdString : public IdString {
+public:
+    DocIdString(const vespalib::stringref & ns, const vespalib::stringref & id);
+    DocIdString(const vespalib::stringref & rawId);
+private:
+    virtual DocIdString* clone() const { return new DocIdString(*this); }
+    virtual Type getType() const { return DOC; }
+    virtual LocationType getLocation() const;
+    virtual vespalib::stringref getNamespaceSpecific() const { return getComponent(1); }
+};
+
+/**
+ * \class document::UserDocIdString
+ * \ingroup base
+ *
+ * \brief Scheme for distributing documents based on a 64 bit number.
+ *
+ * The location of a userdoc identifier is the 64 bit id given. The
+ * name "userdoc" is purely syntactical; Vespa does not care what the source
+ * of the number is.
+ */
+class UserDocIdString : public IdString {
+public:
+    UserDocIdString(const vespalib::stringref & rawId);
+
+    virtual int64_t getUserId() const { return _userId; }
+    virtual bool hasNumber() const { return true; }
+    uint64_t getNumber() const { return _userId; }
+    virtual LocationType getLocation() const { return _userId; }
+
+private:
+    virtual UserDocIdString* clone()   const { return new UserDocIdString(*this); }
+    virtual Type getType() const { return USERDOC; }
+    virtual vespalib::stringref getNamespaceSpecific() const { return getComponent(2); }
+
+    int64_t _userId;
+};
+
+/**
+ * \class document::OrderDocIdString
+ * \ingroup base
+ * \brief Scheme for distributing documents based on a group and a parametrized ordering.
+ */
+class OrderDocIdString : public IdString {
+public:
+    OrderDocIdString(const vespalib::stringref& rawId);
+
+    int64_t  getUserId() const { return _location; }
+    uint16_t getWidthBits() const { return _widthBits; }
+    uint16_t getDivisionBits() const { return _divisionBits; }
+    uint64_t getOrdering() const { return _ordering; }
+    std::pair<int16_t, int64_t> getGidBitsOverride() const;
+    vespalib::string getSchemeName() const;
+    virtual bool hasNumber() const { return true; }
+    uint64_t getNumber() const { return _location; }
+    virtual bool hasGroup() const { return true; }
+    virtual vespalib::stringref getGroup() const { return getComponent(1); }
+
+private:
+    virtual LocationType getLocation() const { return _location; }
+    virtual OrderDocIdString* clone() const { return new OrderDocIdString(*this); }
+    virtual Type getType() const { return ORDERDOC; }
+    virtual vespalib::stringref getNamespaceSpecific() const { return getComponent(3); }
+
+    LocationType _location;
+    uint16_t _widthBits;
+    uint16_t _divisionBits;
+    uint64_t _ordering;
+};
+
+/**
+ * \class document::GroupDocIdString
+ * \ingroup base
+ *
+ * \brief Scheme for distributing documents based on a group string.
+ *
+ * The location of a groupdoc identifier is a hash of the group string.
+ */
+class GroupDocIdString : public IdString {
+public:
+    GroupDocIdString(const vespalib::stringref & rawId);
+    virtual bool hasGroup() const { return true; }
+    virtual vespalib::stringref getGroup() const { return getComponent(1); }
+    virtual LocationType getLocation() const;
+
+    /**
+     * Extract the location for the group-specific part of a document ID.
+     * i.e. `name` here must match the `group` in ID "id::foo:g=group:".
+     */
+    static LocationType locationFromGroupName(vespalib::stringref name);
+
+private:
+    virtual vespalib::stringref getNamespaceSpecific() const { return getComponent(2); }
+    virtual GroupDocIdString* clone() const { return new GroupDocIdString(*this); }
+    virtual Type getType() const { return GROUPDOC; }
+};
+
+} // document
+
diff --git a/document/src/vespa/document/base/testdocman.cpp b/document/src/vespa/document/base/testdocman.cpp
new file mode 100644
index 00000000000..538f1b04c0e
--- /dev/null
+++ b/document/src/vespa/document/base/testdocman.cpp
@@ -0,0 +1,146 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/document/base/testdocman.h>
+
+#include <boost/random.hpp>
+#include <vespa/document/base/exceptions.h>
+#include <vespa/document/datatype/datatypes.h>
+
+namespace document {
+
+namespace {
+    std::vector<char> createBuffer() {
+        // Text from Shakespeare's Hamlet, http://www.ibiblio.org/pub/docs/books/gutenberg/etext98/2ws2610.txt
+        const char* content =
+            "To be, or not to be: that is the question:\n"
+            "Whether 'tis nobler in the mind to suffer\n"
+            "The slings and arrows of outrageous fortune,\n"
+            "Or to take arms against a sea of troubles,\n"
+            "And by opposing end them? To die: to sleep;\n"
+            "No more; and by a sleep to say we end\n"
+            "The heart-ache and the thousand natural shocks\n"
+            "That flesh is heir to, 'tis a consummation\n"
+            "Devoutly to be wish'd. To die, to sleep;\n"
+            "To sleep: perchance to dream: ay, there's the rub;\n"
+            "For in that sleep of death what dreams may come\n"
+            "When we have shuffled off this mortal coil,\n"
+            "Must give us pause: there's the respect\n"
+            "That makes calamity of so long life;\n"
+            "For who would bear the whips and scorns of time,\n"
+            "The oppressor's wrong, the proud man's contumely,\n"
+            "The pangs of despised love, the law's delay,\n"
+            "The insolence of office and the spurns\n"
+            "That patient merit of the unworthy takes,\n"
+            "When he himself might his quietus make\n"
+            "With a bare bodkin? who would fardels bear,\n"
+            "To grunt and sweat under a weary life,\n"
+            "But that the dread of something after death,\n"
+            "The undiscover'd country from whose bourn\n"
+            "No traveller returns, puzzles the will\n"
+            "And makes us rather bear those ills we have\n"
+            "Than fly to others that we know not of?\n"
+            "Thus conscience does make cowards of us all;\n"
+            "And thus the native hue of resolution\n"
+            "Is sicklied o'er with the pale cast of thought,\n"
+            "And enterprises of great pith and moment\n"
+            "With this regard their currents turn awry,\n"
+            "And lose the name of action. - Soft you now!\n"
+            "The fair Ophelia! Nymph, in thy orisons\n"
+            "Be all my sins remember'd.\n\n";
+        std::vector<char> buffer(content, content + strlen(content));
+        return buffer;
+    }
+}
+
+std::vector<char> TestDocMan::_buffer = createBuffer();
+
+TestDocMan::TestDocMan()
+    : _test_repo(),
+      _repo(_test_repo.getTypeRepoSp()),
+      _typeCfg(&_test_repo.getTypeConfig())
+{
+}
+
+TestDocMan::~TestDocMan() {
+}
+
+void
+TestDocMan::setTypeRepo(const DocumentTypeRepo::SP &repo)
+{
+    _repo = repo;
+    _typeCfg = NULL;
+}
+
+Document::UP
+TestDocMan::createDocument(
+        const std::string& content,
+        const std::string& id,
+        const std::string& type) const
+{
+    const DocumentType *type_ptr = 0;
+    type_ptr = _repo->getDocumentType(type);
+    assert(type_ptr);
+    Document::UP doc(new Document(*type_ptr, DocumentId(id)));
+    doc->setValue(doc->getField("content"), StringFieldValue(content.c_str()));
+    return doc;
+}
+
+Document::UP
+TestDocMan::createRandomDocument(int seed, int maxContentSize) const
+{
+    // Currently only one document type added.
+    return createRandomDocument("testdoctype1", seed, maxContentSize);
+}
+
+Document::UP
+TestDocMan::createRandomDocumentAtLocation(
+                int location, int seed, int maxContentSize) const
+{
+    boost::rand48 rnd(seed);
+    std::ostringstream id;
+    id << "id:mail:testdoctype1:n=" << location << ":" << (rnd() % 0x10000)
+       << ".html";
+    return createDocument(generateRandomContent(rnd() % maxContentSize),
+                          id.str(), "testdoctype1");
+}
+
+Document::UP
+TestDocMan::createRandomDocumentAtLocation(
+        int location, int seed, int minContentSize, int maxContentSize) const
+{
+    boost::rand48 rnd(seed);
+    std::ostringstream id;
+    id << "id:mail:testdoctype1:n=" << location << ":" << (rnd() % 0x10000)
+       << ".html";
+
+    int size = maxContentSize > minContentSize ?
+               rnd() % (maxContentSize - minContentSize) + minContentSize :
+               minContentSize;
+    return
+        createDocument(generateRandomContent(size), id.str(), "testdoctype1");
+}
+
+Document::UP
+TestDocMan::createRandomDocument(
+        const std::string& type, int seed, int maxContentSize) const
+{
+    boost::rand48 rnd(seed);
+    std::ostringstream id;
+    id << "id:mail:" << type << ":n=" << (rnd() % 0xFFFF);
+    id << ":" << (rnd() % 256) << ".html";
+    return createDocument(generateRandomContent(rnd() % maxContentSize), id.str(), type);
+}
+
+std::string
+TestDocMan::generateRandomContent(uint32_t size)
+{
+    std::ostringstream content;
+    for (uint32_t i=0; i<size; i += _buffer.size()) {
+        uint32_t sz = (i + _buffer.size() > size ? size - i : _buffer.size());
+        content << std::string(&_buffer[0], sz);
+    }
+    return content.str();
+}
+
+} // document
diff --git a/document/src/vespa/document/base/testdocman.h b/document/src/vespa/document/base/testdocman.h
new file mode 100644
index 00000000000..c95e29c20e2
--- /dev/null
+++ b/document/src/vespa/document/base/testdocman.h
@@ -0,0 +1,68 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * \class document::TestDocMan
+ * \ingroup base
+ *
+ * \brief Utility for unit tests that need document manager and documents.
+ *
+ * This test class sets up a document manager, and defines a few document types
+ * for use in testing.
+ *
+ * The following document types are defined by this manager
+ * (add more when needed):
+ *
+ * testtype1
+ *     headerval int (header variable)
+ *     content string (body variable)
+ */
+
+#pragma once
+
+#include "testdocrepo.h"
+#include <vespa/document/datatype/datatypes.h>
+#include <vespa/document/fieldvalue/fieldvalues.h>
+#include <vespa/document/repo/documenttyperepo.h>
+#include <memory>
+#include <vector>
+
+namespace document {
+    class TestDocMan {
+        static std::vector<char> _buffer;
+        TestDocRepo _test_repo;
+        DocumentTypeRepo::SP _repo;
+        const DocumenttypesConfig *_typeCfg;
+
+    public:
+        TestDocMan();
+        ~TestDocMan();
+
+        void setTypeRepo(const DocumentTypeRepo::SP &repo);
+
+        const DocumentTypeRepo& getTypeRepo() const { return *_repo; }
+        const DocumentTypeRepo::SP getTypeRepoSP() const { return _repo; }
+        const DocumenttypesConfig *getTypeConfig() const { return _typeCfg; }
+
+        /** Create test document. */
+        Document::UP createDocument(
+                const std::string& content = "This is the contents of "
+                        "the test document.\nIt ain't much.\n",
+                const std::string& uri = "id:test:testdoctype1::test",
+                const std::string& type = "testdoctype1") const;
+
+        /** Create random document from given seed. */
+        Document::UP createRandomDocument(
+                int seed = 0, int maxContentSize = 0x80) const;
+        /** Create random document from given seed belonging to given location */
+        Document::UP createRandomDocumentAtLocation(
+                int location, int seed = 0, int maxContentSize = 0x80) const;
+        Document::UP createRandomDocumentAtLocation(
+                int location, int seed, int minContentSize, int maxContentSize) const;
+        /** Create random document of given type from given seed. */
+        Document::UP createRandomDocument(
+                const std::string& type, int seed = 0,
+                int maxContentSize = 0x80) const;
+        static std::string generateRandomContent(uint32_t size);
+    };
+
+} // document
+
diff --git a/document/src/vespa/document/base/testdocrepo.cpp b/document/src/vespa/document/base/testdocrepo.cpp
new file mode 100644
index 00000000000..8421fc6e281
--- /dev/null
+++ b/document/src/vespa/document/base/testdocrepo.cpp
@@ -0,0 +1,73 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".testdocrepo");
+
+#include "testdocrepo.h"
+#include <vespa/document/config/config-documenttypes.h>
+#include <vespa/document/datatype/documenttype.h>
+#include <vespa/document/repo/configbuilder.h>
+
+using document::config_builder::Struct;
+using document::config_builder::Wset;
+using document::config_builder::Array;
+using document::config_builder::Map;
+
+namespace document {
+TestDocRepo::TestDocRepo()
+    :   _cfg(getDefaultConfig()),
+        _repo(new DocumentTypeRepo(_cfg)) {
+}
+
+DocumenttypesConfig TestDocRepo::getDefaultConfig() {
+    const int type1_id = 238423572;
+    const int type2_id = 238424533;
+    const int type3_id = 1088783091;
+    const int mystruct_id = -2092985851;
+    const int structarray_id = 759956026;
+    config_builder::DocumenttypesConfigBuilderHelper builder;
+    builder.document(type1_id, "testdoctype1",
+                     Struct("testdoctype1.header")
+                     .addField("headerval", DataType::T_INT)
+                     .addField("headerlongval", DataType::T_LONG)
+                     .addField("hfloatval", DataType::T_FLOAT)
+                     .addField("hstringval", DataType::T_STRING)
+                     .addField("mystruct", Struct("mystruct")
+                               .setId(mystruct_id)
+                               .addField("key", DataType::T_INT)
+                               .addField("value", DataType::T_STRING))
+                     .addField("tags", Array(DataType::T_STRING))
+                     .addField("stringweightedset", Wset(DataType::T_STRING))
+                     .addField("stringweightedset2", DataType::T_TAG)
+                     .addField("byteweightedset", Wset(DataType::T_BYTE))
+                     .addField("mymap",
+                               Map(DataType::T_INT, DataType::T_STRING))
+                     .addField("structarrmap", Map(
+                                     DataType::T_STRING,
+                                     Array(mystruct_id).setId(structarray_id)))
+                     .addField("title", DataType::T_STRING)
+                     .addField("byteval", DataType::T_BYTE),
+                     Struct("testdoctype1.body")
+                     .addField("content", DataType::T_STRING)
+                     .addField("rawarray", Array(DataType::T_RAW))
+                     .addField("structarray", structarray_id));
+    builder.document(type2_id, "testdoctype2",
+                     Struct("testdoctype2.header")
+                     .addField("onlyinchild", DataType::T_INT),
+                     Struct("testdoctype2.body"))
+        .inherit(type1_id);
+    builder.document(type3_id, "_test_doctype3_",
+                     Struct("_test_doctype3_.header")
+                     .addField("_only_in_child_", DataType::T_INT),
+                     Struct("_test_doctype3_.body"))
+        .inherit(type1_id);
+    return builder.config();
+}
+
+const DataType*
+TestDocRepo::getDocumentType(const vespalib::string &t) const {
+    return _repo->getDocumentType(t);
+}
+
+}  // namespace document
diff --git a/document/src/vespa/document/base/testdocrepo.h b/document/src/vespa/document/base/testdocrepo.h
new file mode 100644
index 00000000000..51c45765514
--- /dev/null
+++ b/document/src/vespa/document/base/testdocrepo.h
@@ -0,0 +1,27 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/document/config/config-documenttypes.h>
+#include <vespa/document/datatype/datatype.h>
+#include <vespa/document/repo/documenttyperepo.h>
+
+namespace document {
+
+class TestDocRepo {
+    DocumenttypesConfig _cfg;
+    DocumentTypeRepo::SP _repo;
+
+public:
+    TestDocRepo();
+
+    static DocumenttypesConfig getDefaultConfig();
+
+    const DocumentTypeRepo& getTypeRepo() const { return *_repo; }
+    const DocumentTypeRepo::SP getTypeRepoSp() const { return _repo; }
+    const DocumenttypesConfig& getTypeConfig() const { return _cfg; }
+    const DataType* getDocumentType(const vespalib::string &name) const;
+};
+
+}  // namespace document
+
diff --git a/document/src/vespa/document/bucket/.gitignore b/document/src/vespa/document/bucket/.gitignore
new file mode 100644
index 00000000000..987027bf12f
--- /dev/null
+++ b/document/src/vespa/document/bucket/.gitignore
@@ -0,0 +1,6 @@
+*.So
+.*.swp
+.depend
+Makefile
+config-bucket.cpp
+config-bucket.h
diff --git a/document/src/vespa/document/bucket/CMakeLists.txt b/document/src/vespa/document/bucket/CMakeLists.txt
new file mode 100644
index 00000000000..6a1372f6607
--- /dev/null
+++ b/document/src/vespa/document/bucket/CMakeLists.txt
@@ -0,0 +1,11 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_library(document_bucket OBJECT
+    SOURCES
+    bucketid.cpp
+    bucketidfactory.cpp
+    bucketselector.cpp
+    bucketdistribution.cpp
+    DEPENDS
+    AFTER
+    document_documentconfig
+)
diff --git a/document/src/vespa/document/bucket/bucketdistribution.cpp b/document/src/vespa/document/bucket/bucketdistribution.cpp
new file mode 100644
index 00000000000..49585fe1092
--- /dev/null
+++ b/document/src/vespa/document/bucket/bucketdistribution.cpp
@@ -0,0 +1,116 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/document/bucket/bucketdistribution.h>
+#include <vespa/log/log.h>
+
+LOG_SETUP(".bucketdistribution");
+
+namespace document {
+
+BucketDistribution::BucketDistribution(uint32_t numColumns, uint32_t numBucketBits) :
+    _numColumns(0),
+    _numBucketBits(numBucketBits),
+    _bucketToColumn(),
+    _lock()
+{
+    _bucketToColumn.resize(getNumBuckets());
+    reset();
+    setNumColumns(numColumns);
+}
+
+void
+BucketDistribution::getBucketCount(uint32_t numColumns, uint32_t numBucketBits, std::vector<uint32_t> &ret)
+{
+    ret.resize(numColumns);
+    uint32_t cnt = getNumBuckets(numBucketBits) / numColumns;
+    uint32_t rst = getNumBuckets(numBucketBits) % numColumns;
+    for (uint32_t i = 0; i < numColumns; ++i) {
+        ret[i] = cnt + (i < rst ? 1 : 0);
+    }
+}
+
+void
+BucketDistribution::getBucketMigrateCount(uint32_t numColumns, uint32_t numBucketBits, std::vector<uint32_t> &ret)
+{
+    getBucketCount(numColumns++, numBucketBits, ret);
+    uint32_t cnt = getNumBuckets(numBucketBits) / numColumns;
+    uint32_t rst = getNumBuckets(numBucketBits) % numColumns;
+    for (uint32_t i = 0; i < numColumns - 1; ++i) {
+        ret[i] -= cnt + (i < rst ? 1 : 0);
+    }
+}
+
+void
+BucketDistribution::reset()
+{
+    for (std::vector<uint32_t>::iterator it = _bucketToColumn.begin();
+         it != _bucketToColumn.end(); ++it) {
+        *it = 0;
+    }
+    _numColumns = 1;
+}
+
+void
+BucketDistribution::addColumn()
+{
+    uint32_t newColumns = _numColumns + 1;
+    std::vector<uint32_t> migrate;
+    getBucketMigrateCount(_numColumns, _numBucketBits, migrate);
+    uint32_t numBuckets = getNumBuckets(_numBucketBits);
+    for (uint32_t i = 0; i < numBuckets; ++i) {
+        uint32_t old = _bucketToColumn[i];
+        if (migrate[old] > 0) {
+            _bucketToColumn[i] = _numColumns; // move this bucket to the new column
+            migrate[old]--;
+        }
+    }
+    _numColumns = newColumns;
+}
+
+void
+BucketDistribution::setNumColumns(uint32_t numColumns)
+{
+    vespalib::LockGuard guard(_lock);
+    if (numColumns < _numColumns) {
+        reset();
+    }
+    if (numColumns == _numColumns) {
+        return;
+    }
+    for (int i = numColumns - _numColumns; --i >= 0; ) {
+        addColumn();
+    }
+}
+
+void
+BucketDistribution::setNumBucketBits(uint32_t numBucketBits)
+{
+    uint32_t numColumns;
+    {
+        vespalib::LockGuard guard(_lock);
+        if (numBucketBits == _numBucketBits) {
+            return;
+        }
+        _numBucketBits = numBucketBits;
+        _bucketToColumn.resize(getNumBuckets(numBucketBits));
+        numColumns = _numColumns;
+        reset();
+    }
+    setNumColumns(numColumns);
+}
+
+uint32_t
+BucketDistribution::getColumn(const document::BucketId &bucketId) const
+{
+    uint32_t ret = (uint32_t)(bucketId.getId() & (getNumBuckets(_numBucketBits) - 1));
+    if (ret >= _bucketToColumn.size()) {
+        LOG(error,
+            "The bucket distribution map is not in sync with the number of bucket bits. "
+            "This should never happen! Distribution is broken!!");
+        return 0;
+    }
+    return _bucketToColumn[ret];
+}
+
+}
diff --git a/document/src/vespa/document/bucket/bucketdistribution.h b/document/src/vespa/document/bucket/bucketdistribution.h
new file mode 100644
index 00000000000..cc7232c81e6
--- /dev/null
+++ b/document/src/vespa/document/bucket/bucketdistribution.h
@@ -0,0 +1,136 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * \class document::BucketDistribution
+ * \ingroup bucket
+ *
+ * Stable algorithmic hash distribution; this class assigns hash buckets to
+ * targets. The number of hash buckets should be large compared to the number
+ * of targets. The mapping from hash value to hash bucket is performed outside
+ * this class.
+ *
+ * This is used to determine which search column a bucket should go to.
+ */
+#pragma once
+
+#include <vector>
+#include <vespa/document/bucket/bucketid.h>
+#include <vespa/vespalib/util/sync.h>
+
+namespace document {
+
+class BucketDistribution {
+public:
+    /**
+     * Constructs a new bucket distribution object with a given number of
+     * columns and buckets.
+     *
+     * @param numColumns    The number of columns to distribute to.
+     * @param numBucketBits The number of bits to use for bucket id.
+     */
+    BucketDistribution(uint32_t numColumns, uint32_t numBucketBits);
+
+    /**
+     * Returns the number of buckets that the given number of bucket bits will
+     * allow.
+     *
+     * @param numBucketBits The number of bits to use for bucket id.
+     * @return              The number of buckets allowed.
+     */
+    static uint32_t getNumBuckets(uint32_t numBucketBits) { return 1 << numBucketBits; }
+
+    /**
+     * This method returns a list that contains the distributions of the given
+     * number of buckets over the given number of columns.
+     *
+     * @param numColumns    The number of columns to distribute to.
+     * @param numBucketBits The number of bits to use for bucket id.
+     * @param ret           List to fill with the bucket distribution.
+     */
+    static void getBucketCount(uint32_t numColumns, uint32_t numBucketBits,
+                               std::vector<uint32_t> &ret);
+
+    /**
+     * This method returns a list similar to getBucketCount(int,int), except
+     * that the returned list contains the number of buckets that will have to
+     * be migrated from each column if an additional column was added.
+     *
+     * @param numColumns    The original number of columns.
+     * @param numBucketBits The number of bits to use for bucket id.
+     * @param ret           List to fill with the number of buckets to migrate,
+     *                      one value per column.
+     */
+    static void getBucketMigrateCount(uint32_t numColumns,
+                        uint32_t numBucketBits, std::vector<uint32_t> &ret);
+
+    /**
+     * Sets the number of columns to distribute to to 1, and resets the content
+     * of the internal bucket-to-column map so that it all buckets point to
+     * that single column.
+     */
+    void reset();
+
+    /**
+     * Sets the number of columns to use for this document distribution object.
+     * This will reset and setup this object from scratch. The original number
+     * of buckets is maintained.
+     *
+     * @param numColumns The new number of columns to distribute to.
+     */
+    void setNumColumns(uint32_t numColumns);
+
+    /**
+     * Returns the number of columns to distribute to.
+     *
+     * @return The number of columns.
+     */
+    uint32_t getNumColumns() const { return _numColumns; }
+
+    /**
+     * Sets the number of buckets to use for this document distribution object.
+     * This will reset and setup this object from scratch. The original number
+     * of columns is maintained.
+     *
+     * @param numBucketBits The new number of bits to use for bucket id.
+     */
+    void setNumBucketBits(uint32_t numBucketBits);
+
+    /**
+     * Returns the number of bits used for bucket identifiers.
+     *
+     * @return The number of bits.
+     */
+    uint32_t getNumBucketBits() const { return _numBucketBits; }
+
+    /**
+     * Returns the number of buckets available using the configured number of
+     * bucket bits.
+     *
+     * @return The number of buckets.
+     */
+    uint32_t getNumBuckets() const { return getNumBuckets(_numBucketBits); }
+
+    /**
+     * This method maps the given bucket id to its corresponding column.
+     *
+     * @param bucketId The bucket whose column to lookup.
+     * @return         The column to distribute the bucket to.
+     */
+    uint32_t getColumn(const document::BucketId &bucketId) const;
+
+private:
+    /**
+     * Adds a single column to this bucket distribution object. This will
+     * modify the internal bucket-to-column map so that it takes into account
+     * the new column.
+     */
+    void addColumn();
+
+private:
+    uint32_t              _numColumns;     // The number of columns to distribute to.
+    uint32_t              _numBucketBits;  // The number of bits to use for bucket identification.
+    std::vector<uint32_t> _bucketToColumn; // A map from bucket id to column index.
+    vespalib::Lock        _lock;
+};
+
+} // document
+
diff --git a/document/src/vespa/document/bucket/bucketid.cpp b/document/src/vespa/document/bucket/bucketid.cpp
new file mode 100644
index 00000000000..14d86d84307
--- /dev/null
+++ b/document/src/vespa/document/bucket/bucketid.cpp
@@ -0,0 +1,155 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/document/bucket/bucketid.h>
+
+#include <iomanip>
+#include <sstream>
+#include <vespa/vespalib/util/exceptions.h>
+#include <vespa/vespalib/objects/nbostream.h>
+#include <vespa/vespalib/stllike/asciistream.h>
+
+using vespalib::nbostream;
+
+namespace document {
+
+
+const unsigned char reverseBitTable[256] =
+{
+  0x00, 0x80, 0x40, 0xC0, 0x20, 0xA0, 0x60, 0xE0, 0x10, 0x90, 0x50, 0xD0, 0x30, 0xB0, 0x70, 0xF0,
+  0x08, 0x88, 0x48, 0xC8, 0x28, 0xA8, 0x68, 0xE8, 0x18, 0x98, 0x58, 0xD8, 0x38, 0xB8, 0x78, 0xF8,
+  0x04, 0x84, 0x44, 0xC4, 0x24, 0xA4, 0x64, 0xE4, 0x14, 0x94, 0x54, 0xD4, 0x34, 0xB4, 0x74, 0xF4,
+  0x0C, 0x8C, 0x4C, 0xCC, 0x2C, 0xAC, 0x6C, 0xEC, 0x1C, 0x9C, 0x5C, 0xDC, 0x3C, 0xBC, 0x7C, 0xFC,
+  0x02, 0x82, 0x42, 0xC2, 0x22, 0xA2, 0x62, 0xE2, 0x12, 0x92, 0x52, 0xD2, 0x32, 0xB2, 0x72, 0xF2,
+  0x0A, 0x8A, 0x4A, 0xCA, 0x2A, 0xAA, 0x6A, 0xEA, 0x1A, 0x9A, 0x5A, 0xDA, 0x3A, 0xBA, 0x7A, 0xFA,
+  0x06, 0x86, 0x46, 0xC6, 0x26, 0xA6, 0x66, 0xE6, 0x16, 0x96, 0x56, 0xD6, 0x36, 0xB6, 0x76, 0xF6,
+  0x0E, 0x8E, 0x4E, 0xCE, 0x2E, 0xAE, 0x6E, 0xEE, 0x1E, 0x9E, 0x5E, 0xDE, 0x3E, 0xBE, 0x7E, 0xFE,
+  0x01, 0x81, 0x41, 0xC1, 0x21, 0xA1, 0x61, 0xE1, 0x11, 0x91, 0x51, 0xD1, 0x31, 0xB1, 0x71, 0xF1,
+  0x09, 0x89, 0x49, 0xC9, 0x29, 0xA9, 0x69, 0xE9, 0x19, 0x99, 0x59, 0xD9, 0x39, 0xB9, 0x79, 0xF9,
+  0x05, 0x85, 0x45, 0xC5, 0x25, 0xA5, 0x65, 0xE5, 0x15, 0x95, 0x55, 0xD5, 0x35, 0xB5, 0x75, 0xF5,
+  0x0D, 0x8D, 0x4D, 0xCD, 0x2D, 0xAD, 0x6D, 0xED, 0x1D, 0x9D, 0x5D, 0xDD, 0x3D, 0xBD, 0x7D, 0xFD,
+  0x03, 0x83, 0x43, 0xC3, 0x23, 0xA3, 0x63, 0xE3, 0x13, 0x93, 0x53, 0xD3, 0x33, 0xB3, 0x73, 0xF3,
+  0x0B, 0x8B, 0x4B, 0xCB, 0x2B, 0xAB, 0x6B, 0xEB, 0x1B, 0x9B, 0x5B, 0xDB, 0x3B, 0xBB, 0x7B, 0xFB,
+  0x07, 0x87, 0x47, 0xC7, 0x27, 0xA7, 0x67, 0xE7, 0x17, 0x97, 0x57, 0xD7, 0x37, 0xB7, 0x77, 0xF7,
+  0x0F, 0x8F, 0x4F, 0xCF, 0x2F, 0xAF, 0x6F, 0xEF, 0x1F, 0x9F, 0x5F, 0xDF, 0x3F, 0xBF, 0x7F, 0xFF
+};
+
+std::vector<BucketId::Type>
+BucketId::getUsedMasks()
+{
+    typedef BucketId::Type Type;
+    uint8_t maxBits = BucketId::maxNumBits();
+    std::vector<Type> masks(maxBits + 1);
+    for (uint32_t usedBits = 0; usedBits <= maxBits; ++usedBits) {
+        uint8_t notused = 8 * sizeof(Type) - usedBits;
+        masks[usedBits] = (std::numeric_limits<Type>::max() << notused)
+                          >> notused;
+    }
+    return masks;
+}
+
+std::vector<BucketId::Type>
+BucketId::getStripMasks()
+{
+    typedef BucketId::Type Type;
+    uint8_t maxBits = BucketId::maxNumBits();
+    std::vector<Type> masks(maxBits + 1);
+    for (uint32_t usedBits = 0; usedBits <= maxBits; ++usedBits) {
+        uint8_t notused = 8 * sizeof(Type) - usedBits;
+        Type usedMask = (std::numeric_limits<Type>::max() << notused)
+                        >> notused;
+        Type countMask = (std::numeric_limits<Type>::max() >> maxBits)
+                         << maxBits;
+        masks[usedBits] = usedMask | countMask;
+    }
+    return masks;
+}
+
+std::vector<BucketId::Type> BucketId::_usedMasks = getUsedMasks();
+std::vector<BucketId::Type> BucketId::_stripMasks = getStripMasks();
+
+void
+BucketId::print(std::ostream& out) const
+{
+    out << toString();
+}
+
+vespalib::string
+BucketId::toString() const
+{
+    vespalib::asciistream stream;
+    stream << "BucketId(0x" << vespalib::hex << vespalib::setw(sizeof(Type)*2)
+           << vespalib::setfill('0') << getId() << ")" << vespalib::dec;
+    return stream.str();
+}
+
+void BucketId::throwFailedSetUsedBits(uint32_t used, uint32_t availBits) {
+    throw vespalib::IllegalArgumentException(vespalib::make_string(
+            "Failed to set used bits to %u, max is %u.",
+            used, availBits), VESPA_STRLOC);
+}
+
+BucketId::Type
+BucketId::reverse(Type id)
+{
+    Type retVal;
+    int bytes = sizeof(Type);
+
+    for (int i = 0; i < bytes; i++) {
+        ((unsigned char*)&retVal)[bytes - i - 1] = reverseBitTable[((const unsigned char*)&id)[i]];
+    }
+
+    return retVal;
+}
+
+BucketId::Type
+BucketId::keyToBucketId(Type key)
+{
+    Type retVal = reverse(key);
+
+    Type usedCountMSB = key << maxNumBits();
+    retVal <<= CountBits;
+    retVal >>= CountBits;
+    retVal |= usedCountMSB;
+
+    return retVal;
+}
+
+bool
+BucketId::contains(const BucketId& id) const
+{
+    if (id.getUsedBits() < getUsedBits()) {
+        return false;
+    }
+    BucketId copy(getUsedBits(), id.getRawId());
+    return (copy.getId() == getId());
+}
+
+vespalib::asciistream& operator<<(vespalib::asciistream& os, const BucketId& id)
+{
+    return os << id.toString();
+}
+
+std::ostream& operator<<(std::ostream& os, const BucketId& id)
+{
+    id.print(os);
+    return os;
+}
+
+
+nbostream &
+operator<<(nbostream &os, const BucketId &bucketId)
+{
+    os << bucketId._id;
+    return os;
+}
+
+
+nbostream &
+operator>>(nbostream &is, BucketId &bucketId)
+{
+    is >> bucketId._id;
+    return is;
+}
+
+
+} // document
diff --git a/document/src/vespa/document/bucket/bucketid.h b/document/src/vespa/document/bucket/bucketid.h
new file mode 100644
index 00000000000..f4ec15ff7c1
--- /dev/null
+++ b/document/src/vespa/document/bucket/bucketid.h
@@ -0,0 +1,200 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * \class document::BucketId
+ * \ingroup bucket
+ *
+ * \brief The document space is divided into buckets, this identifies a chunk.
+ *
+ * The legacy bucket id internals are:
+ *   - A 64 bit internal representation.
+ *   - The 6 MSB bits is a number where value 0-58 specifies how many of the
+ *     other bits are in use. Values 59+ are invalid.
+ *   - The 32 LSB bits are the location. This part may be overridden by
+ *     document id schemes to create a first level sorting criteria.
+ *   - The remaining 28 bits are GID bits (calculated from MD5), used to split
+ *     up buckets with the same location bits. Orderdoc overrides some of these
+ *     bits to represent a secondary order.
+ *
+ * Bucket identifiers are created by the bucket id factory, such that some
+ * non-static state can be kept to optimize the generation.
+ */
+
+#pragma once
+
+#include <vespa/fastos/types.h>
+#include <limits>
+#include <vector>
+#include <vespa/vespalib/stllike/string.h>
+#include <vespa/vespalib/stllike/asciistream.h>
+#include <vespa/vespalib/util/array.h>
+
+namespace vespalib {
+    class nbostream;
+}
+
+namespace document {
+
+extern const unsigned char reverseBitTable[256];
+
+class BucketId
+{
+public:
+    struct hash {
+        size_t operator () (const BucketId& g) const {
+            return g.getId();
+        }
+    };
+
+    /**
+     * The primitive type used to store bucket identifiers. If you use the
+     * typedef when needed we can alter this later with less code changes.
+     */
+    typedef uint64_t Type;
+    typedef vespalib::Array<BucketId, vespalib::DefaultAlloc> List;
+    /** Create an initially unset bucket id. */
+    BucketId() : _id(0) {}
+    /** Create a bucket id with the given raw unchecked content. */
+    explicit BucketId(Type id) : _id(id) {}
+    /** Create a bucket id using a set of bits from a raw unchecked value. */
+    BucketId(uint32_t useBits, Type id) : _id(createUsedBits(useBits, id)) { }
+
+    bool operator<(const BucketId& id) const {
+        return getId() < id.getId();
+    }
+    bool operator==(const BucketId& id) const { return getId() == id.getId(); }
+    bool operator!=(const BucketId& id) const { return getId() != id.getId(); }
+
+    void print(std::ostream& out) const;
+    vespalib::string toString() const;
+
+    bool valid() const {
+        uint32_t usedBits(getUsedBits());
+        return (usedBits >= minNumBits() && usedBits <= maxNumBits());
+    }
+
+    bool isSet() const {
+        return _id != 0u;
+    }
+    /**
+     * Create a bucket id that set all unused bits to zero. If you want to
+     * verify that two different documents belong to the same bucket given some
+     * level of bucket splitting, use this to ignore the unused bits.
+     */
+    BucketId stripUnused() const { return BucketId(getUsedBits(), getId());    }
+
+    /**
+     * Checks whether the given bucket is contained within this bucket. That is,
+     * if it is the same bucket, or if it is a bucket using more bits, which is
+     * identical to this one if set to use as many bits as this one.
+     */
+    bool contains(const BucketId& id) const;
+
+// Functions exposing internals we want to make users independent of
+
+// private: Setting these private when trying to stop code from using it
+    /** Number of MSB bits used to count LSB bits used. */
+    enum { CountBits = 6 };
+
+    static uint32_t maxNumBits() { return (8 * sizeof(Type) - CountBits);}
+    static uint32_t minNumBits() { return 1u; }	// See comment above.
+
+    uint32_t getUsedBits() const { return _id >> maxNumBits(); }
+
+    void setUsedBits(uint32_t used) {
+        uint32_t availBits = maxNumBits();
+        if (used > availBits) {
+            throwFailedSetUsedBits(used, availBits);
+        }
+        Type usedCount(used);
+        usedCount <<= availBits;
+
+        _id <<= CountBits;
+        _id >>= CountBits;
+        _id |= usedCount;
+    }
+
+    /** Get the bucket id value stripped of the bits that are not in use. */
+    Type getId() const { return (_id & getStripMask()); }
+
+    /**
+     * Get the bucket id value stripped of the count bits plus the bits that
+     * are not in use.
+     */
+    Type withoutCountBits() const { return (_id & getUsedMask()); }
+
+    Type getRawId() const { return _id; }
+
+    /**
+     * Reverses the bits in the given number, except the countbits part.
+     * Used for sorting in the bucket database as we want related buckets
+     * to be sorted next to each other.
+     */
+    static Type bucketIdToKey(Type id) {
+        Type retVal = reverse(id);
+
+        Type usedCountLSB = id >> maxNumBits();
+        retVal >>= CountBits;
+        retVal <<= CountBits;
+        retVal |= usedCountLSB;
+
+        return retVal;
+    }
+
+    static Type keyToBucketId(Type key);
+
+    /**
+     * Reverses the bucket id bitwise, except the countbits part,
+     * and returns the value,
+     */
+    Type toKey() const { return bucketIdToKey(getId()); };
+
+    /**
+     * Reverses the order of the bits in the bucket id.
+     */
+    static Type reverse(Type id);
+
+    /**
+     * Returns the value of the Nth bit, counted in the reverse order of the
+     * bucket id.
+     */
+    uint8_t getBit(uint32_t n) const {
+        return (_id & ((Type)1 << n)) == 0 ? 0 : 1;
+    }
+
+private:
+    static std::vector<Type> _usedMasks;
+    static std::vector<Type> _stripMasks;
+    static std::vector<BucketId::Type> getUsedMasks();
+    static std::vector<BucketId::Type> getStripMasks();
+
+    Type _id;
+
+    Type getUsedMask() const {
+        return _usedMasks[getUsedBits()];
+    }
+
+    Type getStripMask() const {
+        return _stripMasks[getUsedBits()];
+    }
+
+    static Type createUsedBits(uint32_t used, Type id) {
+        uint32_t availBits = maxNumBits();
+        Type usedCount(used);
+        usedCount <<= availBits;
+
+        id <<= CountBits;
+        id >>= CountBits;
+        id |= usedCount;
+        return id;
+    }
+
+    static void throwFailedSetUsedBits(uint32_t used, uint32_t availBits);
+
+    friend vespalib::nbostream& operator<<(vespalib::nbostream&, const BucketId&);
+    friend vespalib::nbostream& operator>>(vespalib::nbostream&, BucketId&);
+};
+
+std::ostream& operator<<(std::ostream&, const BucketId&);
+vespalib::asciistream& operator<<(vespalib::asciistream&, const BucketId&);
+
+} // document
diff --git a/document/src/vespa/document/bucket/bucketidfactory.cpp b/document/src/vespa/document/bucket/bucketidfactory.cpp
new file mode 100644
index 00000000000..559495acdf1
--- /dev/null
+++ b/document/src/vespa/document/bucket/bucketidfactory.cpp
@@ -0,0 +1,84 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/document/bucket/bucketidfactory.h>
+
+#include <vespa/document/bucket/bucketid.h>
+#include <vespa/document/base/documentid.h>
+
+namespace document {
+
+BucketIdFactory::BucketIdFactory()
+    : _locationBits(32),
+      _gidBits(26),
+      _countBits(6),
+      _locationMask(0),
+      _gidMask(0),
+      _initialCount(0)
+{
+    initializeMasks();
+}
+
+void
+BucketIdFactory::initializeMasks()
+{
+    assert(_countBits == 6);
+    _locationMask = _gidMask = std::numeric_limits<uint64_t>::max();
+
+    _locationMask <<= (_gidBits + _countBits);
+    _locationMask >>= (_gidBits + _countBits);
+
+    _gidMask >>= _locationBits;
+    _gidMask <<= (_locationBits + _countBits);
+    _gidMask >>= _countBits;
+
+    _initialCount = _locationBits + _gidBits;
+    _initialCount <<= 58;
+}
+
+BucketId
+BucketIdFactory::getBucketId(const DocumentId& id) const
+{
+    uint64_t location = id.getScheme().getLocation();
+    assert(GlobalId::LENGTH >= sizeof(uint64_t) + 4u);
+    uint64_t gid = reinterpret_cast<const uint64_t&>(
+            *(id.getGlobalId().get() + 4));
+
+    std::pair<int16_t, int64_t> gidOverride(
+            id.getScheme().getGidBitsOverride());
+
+    if (gidOverride.first > 0) {
+        return BucketId(_locationBits + _gidBits,
+            _initialCount
+            | (gidOverride.second << _locationBits)
+            | ((_gidMask << gidOverride.first) & gid)
+            | (_locationMask & location));
+    } else {
+        return BucketId(_locationBits + _gidBits,
+            _initialCount
+            | (_gidMask & gid)
+            | (_locationMask & location));
+    }
+}
+
+void
+BucketIdFactory::print(std::ostream& out, bool verbose,
+                       const std::string& indent) const
+{
+    out << "BucketIdFactory("
+        << _locationBits << " location bits, "
+        << _gidBits << " gid bits, "
+        << _countBits << " count bits";
+    if (verbose) {
+        out << std::hex;
+        out << ",\n" << indent << "                location mask: "
+            << _locationMask;
+        out << ",\n" << indent << "                gid mask: "
+            << _gidMask;
+        out << ",\n" << indent << "                initial count: "
+            << _initialCount;
+        out << std::dec;
+    }
+    out << ")";
+}
+
+} // document
diff --git a/document/src/vespa/document/bucket/bucketidfactory.h b/document/src/vespa/document/bucket/bucketidfactory.h
new file mode 100644
index 00000000000..a09bb4156a6
--- /dev/null
+++ b/document/src/vespa/document/bucket/bucketidfactory.h
@@ -0,0 +1,47 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * \class document::BucketIdFactory
+ * \ingroup bucket
+ *
+ * \brief Class to generate bucket identifiers from document identifiers.
+ *
+ * To be able to take advantage of some pregenerated state to make bucket id
+ * creation efficient, this class exist to prevent the need for static objects
+ * in the bucket identifier.
+ *
+ * \see BucketId for more information.
+ */
+
+#pragma once
+
+#include <vespa/document/util/printable.h>
+#include <vespa/fastos/fastos.h>
+
+namespace document {
+
+class BucketId;
+class DocumentId;
+
+class BucketIdFactory : public document::Printable
+{
+    uint16_t _locationBits;
+    uint16_t _gidBits;
+    uint16_t _countBits;
+    uint64_t _locationMask;
+    uint64_t _distributionMask;
+    uint64_t _gidMask;
+    uint64_t _initialCount;
+
+public:
+    BucketIdFactory();
+
+    BucketId getBucketId(const DocumentId&) const;
+
+    void print(std::ostream& out, bool verbose,
+               const std::string& indent) const;
+
+private:
+    void initializeMasks();
+};
+
+} // document
diff --git a/document/src/vespa/document/bucket/bucketselector.cpp b/document/src/vespa/document/bucket/bucketselector.cpp
new file mode 100644
index 00000000000..cfb5b7c001c
--- /dev/null
+++ b/document/src/vespa/document/bucket/bucketselector.cpp
@@ -0,0 +1,273 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/document/bucket/bucketselector.h>
+
+#include <algorithm>
+#include <vespa/document/bucket/bucketidfactory.h>
+#include <vespa/document/base/documentid.h>
+#include <vespa/document/base/idstring.h>
+#include <vespa/document/select/node.h>
+#include <vespa/document/select/valuenode.h>
+#include <vespa/document/select/visitor.h>
+#include <vespa/document/select/branch.h>
+#include <vespa/document/select/compare.h>
+#include <math.h>
+
+namespace document {
+
+using namespace document::select;
+
+//namespace {
+    /**
+     * Visitor class that is used for visiting a node tree generated by a
+     * document selection expression.
+     *
+     * The visitor class contains the set of buckets expression can match.
+     */
+    struct BucketVisitor : public document::select::Visitor {
+        const BucketIdFactory& _factory;
+        std::vector<BucketId> _buckets;
+            // If set to false, the buckets to visit is set in _buckets.
+        bool _unknown;
+
+        BucketVisitor(const BucketIdFactory& factory)
+            : _factory(factory), _buckets(), _unknown(true) {}
+
+        void visitAndBranch(const document::select::And& node) {
+            BucketVisitor left(_factory);
+            node.getLeft().visit(left);
+            node.getRight().visit(*this);
+                // If either part is unknown we can just return other part.
+            if (left._unknown) {
+                return;
+            }
+                // If only left part is known return that part.
+            if (_unknown) {
+                _buckets.swap(left._buckets);
+                _unknown = false;
+                return;
+            }
+            std::vector<BucketId> result;
+            std::set_intersection(left._buckets.begin(), left._buckets.end(),
+                                  _buckets.begin(), _buckets.end(),
+                                  back_inserter(result));
+            _buckets.swap(result);
+            return;
+        }
+
+        void visitOrBranch(const document::select::Or& node) {
+            BucketVisitor left(_factory);
+            node.getLeft().visit(left);
+            node.getRight().visit(*this);
+                // If one part is unknown we have to keep unknown status
+            if (left._unknown || _unknown) {
+                _unknown = true;
+                return;
+            }
+                // If both parts are known return both sets
+            std::vector<BucketId> result;
+            std::set_union(left._buckets.begin(), left._buckets.end(),
+                           _buckets.begin(), _buckets.end(),
+                           back_inserter(result));
+            _buckets.swap(result);
+        }
+
+        void visitNotBranch(const document::select::Not&) {
+            // Since selected locations doesn't include everything at that
+            // location, we can't reverse the selection. Any NOT branch must
+            // end up specifying all
+        }
+
+        void compare(const select::IdValueNode& node,
+                     const select::ValueNode& valnode,
+                     const select::Operator& op)
+        {
+            if (node.getType() == IdValueNode::ALL) {
+                const StringValueNode* val(
+                        dynamic_cast<const StringValueNode*>(&valnode));
+                if (!val) return;
+                vespalib::string docId(val->getValue());
+                if (op == FunctionOperator::EQ ||
+                    !GlobOperator::containsVariables(docId))
+                {
+                    IdString::UP id(IdString::createIdString(docId));
+                    _buckets.push_back(BucketId(58, id->getLocation()));
+                    _unknown = false;
+                }
+            } else if (node.getType() == IdValueNode::USER) {
+                const IntegerValueNode* val(
+                        dynamic_cast<const IntegerValueNode*>(&valnode));
+                if (!val) return;
+                UserDocIdString id(vespalib::make_string("userdoc::%lu:", val->getValue()));
+                _buckets.push_back(BucketId(32, id.getLocation()));
+                _unknown = false;
+            } else if (node.getType() == IdValueNode::GROUP) {
+                const StringValueNode* val(
+                        dynamic_cast<const StringValueNode*>(&valnode));
+                if (!val) return;
+                vespalib::string group(val->getValue());
+                if (op == FunctionOperator::EQ ||
+                    !GlobOperator::containsVariables(group))
+                {
+                    GroupDocIdString id("groupdoc::" + group + ":");
+                    _buckets.push_back(BucketId(32, id.getLocation()));
+                    _unknown = false;
+                }
+            } else if (node.getType() == IdValueNode::GID) {
+                const StringValueNode* val(
+                        dynamic_cast<const StringValueNode*>(&valnode));
+
+                vespalib::string gid(val->getValue());
+                if (op == FunctionOperator::EQ ||
+                    !GlobOperator::containsVariables(gid))
+                {
+                    BucketId bid = document::GlobalId::parse(gid).convertToBucketId();
+                    _buckets.push_back(BucketId(32, bid.getRawId()));
+                    _unknown = false;
+                }
+            } else if (node.getType() == IdValueNode::BUCKET) {
+                const IntegerValueNode* val(
+                        dynamic_cast<const IntegerValueNode*>(&valnode));
+                if (!val) return;
+
+                BucketId bid(val->getValue());
+                if (!bid.getUsedBits()) {
+                    bid.setUsedBits(32);
+                }
+                _buckets.push_back(bid);
+                _unknown = false;
+            }
+        }
+
+        void compare(const select::SearchColumnValueNode& node,
+                     const select::ValueNode& valnode,
+                     const select::Operator& op) {
+            if (op == FunctionOperator::EQ || op == document::select::GlobOperator::GLOB) {
+                int bucketCount = (int)pow(2, 16);
+                const IntegerValueNode* val(
+                        dynamic_cast<const IntegerValueNode*>(&valnode));
+
+                int64_t rval = val->getValue();
+
+                for (int i = 0; i < bucketCount; i++) {
+                    int64_t column = node.getValue(BucketId(16, i));
+                    if (column == rval) {
+                        _buckets.push_back(BucketId(16, i));
+                    }
+                }
+
+                _unknown = false;
+            }
+        }
+
+        void visitComparison(const document::select::Compare& node) {
+            if (node.getOperator() != document::select::FunctionOperator::EQ &&
+                node.getOperator() != document::select::GlobOperator::GLOB)
+            {
+                return;
+            }
+            const IdValueNode* lid(dynamic_cast<const IdValueNode*>(
+                        &node.getLeft()));
+            const SearchColumnValueNode* sc(dynamic_cast<const SearchColumnValueNode*>(
+                                                    &node.getLeft()));
+            if (lid) {
+                compare(*lid, node.getRight(), node.getOperator());
+            } else if (sc) {
+                compare(*sc, node.getRight(), node.getOperator());
+            } else {
+                const IdValueNode* rid(dynamic_cast<const IdValueNode*>(
+                            &node.getRight()));
+                if (rid) {
+                    compare(*rid, node.getLeft(), node.getOperator());
+                }
+            }
+        }
+
+        void visitConstant(const document::select::Constant&) {
+        }
+
+        virtual void
+        visitInvalidConstant(const document::select::InvalidConstant &)
+        {
+        }
+
+        void visitDocumentType(const document::select::DocType&) {
+        }
+
+        virtual void
+        visitArithmeticValueNode(const ArithmeticValueNode &)
+        {
+        }
+
+        virtual void
+        visitFunctionValueNode(const FunctionValueNode &)
+        {
+        }
+
+        virtual void
+        visitIdValueNode(const IdValueNode &)
+        {
+        }
+
+        virtual void
+        visitSearchColumnValueNode(const SearchColumnValueNode &)
+        {
+        }
+
+        virtual void
+        visitFieldValueNode(const FieldValueNode &)
+        {
+        }
+
+        virtual void
+        visitFloatValueNode(const FloatValueNode &)
+        {
+        }
+
+        virtual void
+        visitVariableValueNode(const VariableValueNode &)
+        {
+        }
+
+        virtual void
+        visitIntegerValueNode(const IntegerValueNode &)
+        {
+        }
+
+        virtual void
+        visitCurrentTimeValueNode(const CurrentTimeValueNode &)
+        {
+        }
+
+        virtual void
+        visitStringValueNode(const StringValueNode &)
+        {
+        }
+
+        virtual void
+        visitNullValueNode(const NullValueNode &)
+        {
+        }
+
+        virtual void
+        visitInvalidValueNode(const InvalidValueNode &)
+        {
+        }
+    };
+//}
+
+BucketSelector::BucketSelector(const document::BucketIdFactory& factory)
+    : _factory(factory)
+{
+}
+
+std::unique_ptr<BucketSelector::BucketVector>
+BucketSelector::select(const document::select::Node& expression) const
+{
+    BucketVisitor v(_factory);
+    expression.visit(v);
+    return std::unique_ptr<BucketVector>(v._unknown
+            ? 0 : new BucketVector(v._buckets));
+}
+
+} // document
diff --git a/document/src/vespa/document/bucket/bucketselector.h b/document/src/vespa/document/bucket/bucketselector.h
new file mode 100644
index 00000000000..141b951d41f
--- /dev/null
+++ b/document/src/vespa/document/bucket/bucketselector.h
@@ -0,0 +1,51 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * \class document::BucketSelector
+ * \ingroup bucket
+ *
+ * \brief Calculates which buckets correspond to a document selection.
+ *
+ * When you want to visit a subset of documents in VDS you specify a
+ * document selection expression. Some of these expressions limit what
+ * buckets may contain matching documents.
+ *
+ * This class is used to calculate which set of buckets we need to visit
+ * to be sure we find all existing data.
+ *
+ * \see BucketId For more information on buckets
+ * \see document::select::Parser For more information about the selection
+ * language
+ */
+
+#pragma once
+
+#include <memory>
+#include <vector>
+
+namespace document {
+namespace select {
+    class Node;
+}
+class BucketId;
+class BucketIdFactory;
+
+class BucketSelector {
+    const BucketIdFactory& _factory;
+
+public:
+    explicit BucketSelector(const BucketIdFactory& factory);
+
+    typedef std::vector<BucketId> BucketVector;
+    /**
+     * Get a list of bucket ids that needs to be visited to be sure to find
+     * all data matching given expression. Note that we can only detect
+     * some common expressions. We guarantuee that you get all buckets
+     * that may contain data, but not that you get the minimal bucket set.
+     *
+     * If a small bucket set can not be identified, a null pointer is returned
+     * to indicate all buckets needs to be visited.
+     */
+    std::unique_ptr<BucketVector> select(const select::Node& expression) const;
+};
+
+} // document
diff --git a/document/src/vespa/document/config/.gitignore b/document/src/vespa/document/config/.gitignore
new file mode 100644
index 00000000000..0d235ff140b
--- /dev/null
+++ b/document/src/vespa/document/config/.gitignore
@@ -0,0 +1,11 @@
+.depend
+Makefile
+config-bucket.So
+config-bucket.cpp
+config-bucket.h
+config-documentmanager.So
+config-documentmanager.cpp
+config-documentmanager.h
+config-documenttypes.So
+config-documenttypes.cpp
+config-documenttypes.h
diff --git a/document/src/vespa/document/config/CMakeLists.txt b/document/src/vespa/document/config/CMakeLists.txt
new file mode 100644
index 00000000000..02bedc39525
--- /dev/null
+++ b/document/src/vespa/document/config/CMakeLists.txt
@@ -0,0 +1,10 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_library(document_documentconfig OBJECT
+    SOURCES
+    DEPENDS
+)
+vespa_generate_config(document_documentconfig documenttypes.def)
+install(FILES
+    documenttypes.def
+    documentmanager.def
+    DESTINATION var/db/vespa/config_server/serverdb/classes)
diff --git a/document/src/vespa/document/config/documentmanager.def b/document/src/vespa/document/config/documentmanager.def
new file mode 100644
index 00000000000..267273cfa21
--- /dev/null
+++ b/document/src/vespa/document/config/documentmanager.def
@@ -0,0 +1,97 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+version=15
+
+namespace=document.config
+
+## Whether to enable compression in this process.
+enablecompression bool default=false
+
+## The Id of the datatype. Must be unique, including not
+## overlapping with the internal datatypes (defined in datatype.h)
+datatype[].id int
+
+## Use if this datatype is an array type, for instance an int
+## or double array. Specifies the datatype we have an array of.
+## This can be a built-in datatype or a new one created here, meaning
+## that for instance arrays of arrays are allowed.
+datatype[].arraytype[].datatype int
+
+## Map type. Keys and values can be built in types or types created here.
+datatype[].maptype[].keytype int
+datatype[].maptype[].valtype int
+
+## Use if this datatype is a weighted set type, for instance an int
+## or double weighted set. Specifies the datatype we have a weighted set of.
+## This can be a built-in datatype or a new one craeted here, meaning
+## that for instance weighted sets of weighted sets are allowed.
+datatype[].weightedsettype[].datatype int
+
+## Should an update to a nonexistant element cause it to be created
+datatype[].weightedsettype[].createifnonexistant bool default=false
+
+## Should an element in a weighted set be removed if an update changes the weight to 0
+datatype[].weightedsettype[].removeifzero bool default=false
+
+## Specify the name of the document type. Must be unique.
+datatype[].structtype[].name string
+
+## Verison is not in use
+datatype[].structtype[].version int default=0
+
+## Specify which compression to use if compression is enabled above
+datatype[].structtype[].compresstype enum { NONE, UNCOMPRESSABLE, LZ4 } default=NONE
+
+## Specify the compression level to use if compression is enabled
+datatype[].structtype[].compresslevel int default=0
+
+## Specify the minimum reduction required from compression in order to keep the compressed version (maximum percentage of original size)
+datatype[].structtype[].compressthreshold int default=95
+
+## Specify the minimum size of the struct data before we even try to compress it
+datatype[].structtype[].compressminsize int default=800
+
+## Specify a document field name. Must be unique within the document type.
+datatype[].structtype[].field[].name string
+
+## Specify a document field id. If not specified, this is generated by a hash function
+datatype[].structtype[].field[].id[].id int
+
+## Specify the datatype of the field. Can be a built-in datatype or
+## one specified in config.
+datatype[].structtype[].field[].datatype int
+
+## Specify a document type to inherit
+datatype[].structtype[].inherits[].name string
+
+## Version is not in use
+datatype[].structtype[].inherits[].version int default=0
+
+## Specity an annotation reference type and the name of the annotation type it is referencing
+datatype[].annotationreftype[].annotation string
+
+## Specify the name of the document type. Must be unique.
+datatype[].documenttype[].name string
+
+## Version is not in use
+datatype[].documenttype[].version int default=0
+
+## Specify a document type to inherit
+datatype[].documenttype[].inherits[].name string
+
+## Version is not in use
+datatype[].documenttype[].inherits[].version int default=0
+
+## Name of header struct defining document header.
+datatype[].documenttype[].headerstruct int
+
+## Specify a document field id. Must be unique within the document type.
+datatype[].documenttype[].bodystruct int
+
+## Field sets
+datatype[].documenttype[].fieldsets{}.fields[] string
+
+## The Id of the annotation type. Must be unique.
+annotationtype[].id int
+annotationtype[].name string
+annotationtype[].datatype int default=-1
+annotationtype[].inherits[].id int
diff --git a/document/src/vespa/document/config/documenttypes.def b/document/src/vespa/document/config/documenttypes.def
new file mode 100644
index 00000000000..5e0c5e4e528
--- /dev/null
+++ b/document/src/vespa/document/config/documenttypes.def
@@ -0,0 +1,98 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+version=15
+
+namespace=document
+
+## Whether to enable compression in this process.
+enablecompression bool default=false
+
+## The Id of the documenttype. Must be unique among all document types.
+documenttype[].id int
+
+## Specify the name of the document type. Must be unique.
+documenttype[].name string
+
+## Version is not used
+documenttype[].version int default=0
+
+## Name of header struct defining document header.
+documenttype[].headerstruct int
+
+## Specify a document field id. Must be unique within the document type.
+documenttype[].bodystruct int
+
+## Specify a document type to inherit (id)
+documenttype[].inherits[].id int
+
+## This is the id of a datatype defined in the document.
+documenttype[].datatype[].id int
+
+## This is the type of the datatype.
+documenttype[].datatype[].type enum {STRUCT, ARRAY, WSET, MAP, ANNOTATIONREF, PRIMITIVE}
+
+## This is the id of the datatype of the element in the array.
+documenttype[].datatype[].array.element.id int default=0
+
+## This is the id of the datatype of the key in the map.
+documenttype[].datatype[].map.key.id int default=0
+
+## This is the id of the datatype of the value in the map.
+documenttype[].datatype[].map.value.id int default=0
+
+## This is the id of the datatype of the key in the wset.
+documenttype[].datatype[].wset.key.id int default=0
+
+## Should an update to a nonexistent element cause it to be created 
+documenttype[].datatype[].wset.createifnonexistent bool default=false
+
+## Should an element in a weighted set be removed if an update changes the weight to 0
+documenttype[].datatype[].wset.removeifzero bool default=false
+
+## This is the id of the referenced annotation.
+documenttype[].datatype[].annotationref.annotation.id int default=0
+
+## Specify the name of the struct type. Must be unique within documenttype.
+documenttype[].datatype[].sstruct.name string default=""
+
+## Version is not used
+documenttype[].datatype[].sstruct.version int default=0
+
+## Specify which compression to use if compression is enabled above (0 = NONE, 6 = LZ4)
+documenttype[].datatype[].sstruct.compression.type enum {NONE, LZ4} default=NONE
+
+## Specify the compression level to use if compression is enabled
+documenttype[].datatype[].sstruct.compression.level int default=0
+
+## Specify the minimum reduction required from compression in order to keep the compressed version (maximum percentage of original size)
+documenttype[].datatype[].sstruct.compression.threshold int default=95
+
+## Specify the minimum size before trying compression.
+documenttype[].datatype[].sstruct.compression.minsize int default=200
+
+## Specify a struct field name. Must be unique within the struct type.
+documenttype[].datatype[].sstruct.field[].name string
+
+## Specify a document field id. If not specified, this is generated by a hash function
+documenttype[].datatype[].sstruct.field[].id int
+
+## Specify a document field id for serialization version 6. If not specified, this is generated by a hash function
+documenttype[].datatype[].sstruct.field[].id_v6 int
+
+## Specify the datatype of the field. Can only be a type defined in document
+## or one of its inherited document types.
+documenttype[].datatype[].sstruct.field[].datatype int
+
+## The id of the annotation type.
+documenttype[].annotationtype[].id int
+
+## The name of the annotation type.
+documenttype[].annotationtype[].name string
+
+## The contained datatype of the annotation type.
+documenttype[].annotationtype[].datatype int default=-1
+
+## The annotation type that this type inherits. (0 or 1 type)
+documenttype[].annotationtype[].inherits[].id int
+
+## Field sets
+documenttype[].fieldsets{}.fields[] string
diff --git a/document/src/vespa/document/datatype/.gitignore b/document/src/vespa/document/datatype/.gitignore
new file mode 100644
index 00000000000..dfa09296ddb
--- /dev/null
+++ b/document/src/vespa/document/datatype/.gitignore
@@ -0,0 +1,4 @@
+*.So
+.*.swp
+.depend
+Makefile
diff --git a/document/src/vespa/document/datatype/CMakeLists.txt b/document/src/vespa/document/datatype/CMakeLists.txt
new file mode 100644
index 00000000000..e6826075762
--- /dev/null
+++ b/document/src/vespa/document/datatype/CMakeLists.txt
@@ -0,0 +1,21 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_library(document_datatypes OBJECT
+    SOURCES
+    annotationreferencedatatype.cpp
+    annotationtype.cpp
+    arraydatatype.cpp
+    collectiondatatype.cpp
+    datatype.cpp
+    documenttype.cpp
+    mapdatatype.cpp
+    numericdatatype.cpp
+    positiondatatype.cpp
+    primitivedatatype.cpp
+    structdatatype.cpp
+    structureddatatype.cpp
+    urldatatype.cpp
+    weightedsetdatatype.cpp
+    DEPENDS
+    AFTER
+    document_documentconfig
+)
diff --git a/document/src/vespa/document/datatype/annotationreferencedatatype.cpp b/document/src/vespa/document/datatype/annotationreferencedatatype.cpp
new file mode 100644
index 00000000000..3a989f0814b
--- /dev/null
+++ b/document/src/vespa/document/datatype/annotationreferencedatatype.cpp
@@ -0,0 +1,47 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".annotationreferencedatatype");
+
+#include "annotationreferencedatatype.h"
+#include <vespa/document/fieldvalue/annotationreferencefieldvalue.h>
+
+using std::unique_ptr;
+using std::ostream;
+
+namespace document {
+
+IMPLEMENT_IDENTIFIABLE(AnnotationReferenceDataType, DataType);
+
+AnnotationReferenceDataType::AnnotationReferenceDataType(
+        const AnnotationType &type, int id)
+    : DataType("annotationreference<" + type.getName() + ">", id),
+      _type(&type) {
+}
+
+const AnnotationType &AnnotationReferenceDataType::getAnnotationType() const {
+    assert(_type);
+    return *_type;
+}
+
+void
+AnnotationReferenceDataType::print(ostream &out, bool, const std::string &) const {
+    out << "AnnotationReferenceDataType("
+        << getName() << ", " << getId() << ")";
+}
+
+AnnotationReferenceDataType *AnnotationReferenceDataType::clone() const {
+    return new AnnotationReferenceDataType(*this);
+}
+
+unique_ptr<FieldValue> AnnotationReferenceDataType::createFieldValue() const {
+    return FieldValue::UP(new AnnotationReferenceFieldValue(*this, 0));
+}
+
+unique_ptr<FieldPath> AnnotationReferenceDataType::onBuildFieldPath( const vespalib::stringref &) const {
+    return unique_ptr<FieldPath>(new FieldPath);
+}
+
+
+}  // namespace document
diff --git a/document/src/vespa/document/datatype/annotationreferencedatatype.h b/document/src/vespa/document/datatype/annotationreferencedatatype.h
new file mode 100644
index 00000000000..498c931f715
--- /dev/null
+++ b/document/src/vespa/document/datatype/annotationreferencedatatype.h
@@ -0,0 +1,31 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "annotationtype.h"
+#include <memory>
+#include <vespa/document/datatype/datatype.h>
+
+namespace document {
+
+class AnnotationReferenceDataType : public DataType {
+    const AnnotationType *_type;
+
+public:
+    typedef std::shared_ptr<AnnotationReferenceDataType> SP;
+
+    AnnotationReferenceDataType() {}
+    AnnotationReferenceDataType(const AnnotationType &type, int id);
+
+    const AnnotationType &getAnnotationType() const;
+    virtual void print(std::ostream &out, bool verbose,
+                       const std::string &indent) const;
+    virtual AnnotationReferenceDataType *clone() const;
+    virtual std::unique_ptr<FieldValue> createFieldValue() const;
+    virtual FieldPath::UP onBuildFieldPath(const vespalib::stringref &remainFieldName) const;
+
+    DECLARE_IDENTIFIABLE(AnnotationReferenceDataType);
+};
+
+}  // namespace document
+
diff --git a/document/src/vespa/document/datatype/annotationtype.cpp b/document/src/vespa/document/datatype/annotationtype.cpp
new file mode 100644
index 00000000000..4f7f3b0ff41
--- /dev/null
+++ b/document/src/vespa/document/datatype/annotationtype.cpp
@@ -0,0 +1,42 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".annotationtype");
+
+#include "annotationtype.h"
+
+#include "numericdatatype.h"
+#include "primitivedatatype.h"
+#include <vespa/vespalib/stllike/string.h>
+
+using std::vector;
+using vespalib::string;
+
+namespace document {
+namespace {
+AnnotationType makeType(int id, string name, const DataType &type) {
+    AnnotationType annotation_type(id, name);
+    annotation_type.setDataType(type);
+    return annotation_type;
+}
+
+const PrimitiveDataType STRING_OBJ(DataType::T_STRING);
+const NumericDataType INT_OBJ(DataType::T_INT);
+
+const AnnotationType TERM_OBJ(makeType(1, "term", STRING_OBJ));
+const AnnotationType TOKEN_TYPE_OBJ(makeType(2, "token_type", INT_OBJ));
+
+}  // namespace
+
+const AnnotationType *const AnnotationType::TERM(&TERM_OBJ);
+const AnnotationType *const AnnotationType::TOKEN_TYPE(&TOKEN_TYPE_OBJ);
+
+vector<const AnnotationType *> AnnotationType::getDefaultAnnotationTypes() {
+    vector<const AnnotationType *> types;
+    types.push_back(TERM);
+    types.push_back(TOKEN_TYPE);
+    return types;
+}
+
+}  // namespace document
diff --git a/document/src/vespa/document/datatype/annotationtype.h b/document/src/vespa/document/datatype/annotationtype.h
new file mode 100644
index 00000000000..637bd185f74
--- /dev/null
+++ b/document/src/vespa/document/datatype/annotationtype.h
@@ -0,0 +1,49 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "datatype.h"
+#include <memory>
+#include <string>
+#include <vector>
+
+namespace document {
+
+class AnnotationType {
+    int _id;
+    vespalib::string _name;
+    const DataType *_type;
+
+public:
+    typedef std::unique_ptr<AnnotationType> UP;
+    typedef std::shared_ptr<AnnotationType> SP;
+
+    AnnotationType(int id, const vespalib::stringref &name)
+        : _id(id), _name(name), _type(0) {}
+    void setDataType(const DataType &type) { _type = &type; }
+
+    const vespalib::string & getName() const { return _name; }
+    int getId() const { return _id; }
+    const DataType *getDataType() const { return _type; }
+
+    static const AnnotationType *const TERM;
+    static const AnnotationType *const TOKEN_TYPE;
+
+    /** Used by type manager to fetch default types to register. */
+    static std::vector<const AnnotationType *> getDefaultAnnotationTypes();
+};
+
+inline bool operator==(const AnnotationType &a1, const AnnotationType &a2) {
+    return a1.getId() == a2.getId() && a1.getName() == a2.getName();
+}
+
+inline bool operator!=(const AnnotationType &a1, const AnnotationType &a2) {
+    return !(a1 == a2);
+}
+
+inline std::ostream &operator<<(std::ostream &out, const AnnotationType &a) {
+    return out << "AnnotationType(" << a.getId() << ", " << a.getName() << ")";
+}
+
+}  // namespace document
+
diff --git a/document/src/vespa/document/datatype/arraydatatype.cpp b/document/src/vespa/document/datatype/arraydatatype.cpp
new file mode 100644
index 00000000000..7e851453c78
--- /dev/null
+++ b/document/src/vespa/document/datatype/arraydatatype.cpp
@@ -0,0 +1,75 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/document/datatype/arraydatatype.h>
+
+#include <vespa/document/fieldvalue/arrayfieldvalue.h>
+
+namespace document {
+
+IMPLEMENT_IDENTIFIABLE(ArrayDataType, CollectionDataType);
+
+ArrayDataType::ArrayDataType(const DataType &nestedType, int32_t id)
+    : CollectionDataType("Array<" + nestedType.getName() + ">", nestedType, id)
+{
+}
+
+ArrayDataType::ArrayDataType(const DataType& nestedType)
+    : CollectionDataType("Array<" + nestedType.getName() + ">", nestedType)
+{
+}
+
+FieldValue::UP
+ArrayDataType::createFieldValue() const
+{
+    return FieldValue::UP(new ArrayFieldValue(*this));
+}
+
+void
+ArrayDataType::print(std::ostream& out, bool verbose,
+                     const std::string& indent) const
+{
+    out << "ArrayDataType(\n" << indent << "    ";
+    getNestedType().print(out, verbose, indent + "    ");
+    out << ", id " << getId() << ")";
+}
+
+bool
+ArrayDataType::operator==(const DataType& other) const
+{
+    if (this == &other) return true;
+    if (!CollectionDataType::operator==(other)) return false;
+    return other.inherits(ArrayDataType::classId);
+}
+
+FieldPath::UP
+ArrayDataType::onBuildFieldPath(const vespalib::stringref & remainFieldName) const
+{
+    if (remainFieldName[0] == '[') {
+        size_t endPos = remainFieldName.find(']');
+        if (endPos == vespalib::stringref::npos) {
+            throw vespalib::IllegalArgumentException("Array subscript must be closed with ]");
+        } else {
+            int pos = endPos + 1;
+            if (remainFieldName[pos] == '.') {
+                pos++;
+            }
+
+            FieldPath::UP path = getNestedType().buildFieldPath(remainFieldName.substr(pos));
+            if (!path.get()) {
+                return FieldPath::UP();
+            }
+            if (remainFieldName[1] == '$') {
+                path->insert(path->begin(), FieldPathEntry(getNestedType(), remainFieldName.substr(2, endPos - 2)));
+            } else {
+                path->insert(path->begin(), FieldPathEntry(getNestedType(), atoi(remainFieldName.substr(1, endPos - 1).c_str())));
+            }
+
+            return path;
+        }
+    }
+
+    return getNestedType().buildFieldPath(remainFieldName);
+}
+
+} // document
diff --git a/document/src/vespa/document/datatype/arraydatatype.h b/document/src/vespa/document/datatype/arraydatatype.h
new file mode 100644
index 00000000000..706579ae2f6
--- /dev/null
+++ b/document/src/vespa/document/datatype/arraydatatype.h
@@ -0,0 +1,40 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * \class document::ArrayDataType
+ * \ingroup datatype
+ *
+ * \brief A datatype specifying what can be contained in an array field value.
+ */
+#pragma once
+
+#include <vespa/document/datatype/collectiondatatype.h>
+
+namespace document {
+
+class ArrayDataType : public CollectionDataType {
+protected:
+    // Protected to help you avoid calling the copy constructor when
+    // you think you're calling the regular constructor with a nested
+    // ArrayDataType.
+    ArrayDataType(const ArrayDataType &o) : CollectionDataType(o) {}
+
+public:
+    ArrayDataType() {}
+    explicit ArrayDataType(const DataType &nestedType);
+    ArrayDataType(const DataType &nestedType, int32_t id);
+
+        // CollectionDataType implementation
+    virtual std::unique_ptr<FieldValue> createFieldValue() const;
+    virtual void print(std::ostream&, bool verbose,
+                       const std::string& indent) const;
+    virtual bool operator==(const DataType& other) const;
+    virtual ArrayDataType* clone() const { return new ArrayDataType(*this); }
+
+    FieldPath::UP onBuildFieldPath(
+            const vespalib::stringref & remainFieldName) const;
+
+    DECLARE_IDENTIFIABLE(ArrayDataType);
+};
+
+} // document
+
diff --git a/document/src/vespa/document/datatype/collectiondatatype.cpp b/document/src/vespa/document/datatype/collectiondatatype.cpp
new file mode 100644
index 00000000000..903875e303c
--- /dev/null
+++ b/document/src/vespa/document/datatype/collectiondatatype.cpp
@@ -0,0 +1,55 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/document/datatype/collectiondatatype.h>
+
+#include <vespa/document/util/stringutil.h>
+#include <vespa/vespalib/util/exceptions.h>
+
+namespace document {
+
+IMPLEMENT_IDENTIFIABLE_ABSTRACT(CollectionDataType, DataType);
+
+CollectionDataType::CollectionDataType(const CollectionDataType& other)
+    : DataType(other),
+      _nestedType(other._nestedType)
+{
+}
+
+CollectionDataType&
+CollectionDataType::operator=(const CollectionDataType& other)
+{
+    if (this != &other) {
+        DataType::operator=(other);
+        _nestedType = other._nestedType;
+    }
+    return *this;
+}
+
+CollectionDataType::CollectionDataType(const vespalib::stringref& name,
+                                       const DataType& nestedType)
+    : DataType(name),
+      _nestedType(&nestedType) {
+}
+
+CollectionDataType::CollectionDataType(const vespalib::stringref& name,
+                                       const DataType& nestedType,
+                                       int32_t id)
+    : DataType(name, id),
+      _nestedType(&nestedType) {
+}
+
+CollectionDataType::~CollectionDataType()
+{
+}
+
+bool
+CollectionDataType::operator==(const DataType& other) const
+{
+    if (!DataType::operator==(other)) return false;
+    const CollectionDataType* o(
+            Identifiable::cast<const CollectionDataType*>(&other));
+    return o != 0 && *_nestedType == *o->_nestedType;
+}
+
+} // document
diff --git a/document/src/vespa/document/datatype/collectiondatatype.h b/document/src/vespa/document/datatype/collectiondatatype.h
new file mode 100644
index 00000000000..59fbe2c6d5a
--- /dev/null
+++ b/document/src/vespa/document/datatype/collectiondatatype.h
@@ -0,0 +1,40 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * \class document::CollectionDataType
+ * \ingroup datatype
+ *
+ * \brief Data type used for collections of data with similar types.
+ *
+ * This contains common functionality for array and weighted set datatypes.
+ */
+#pragma once
+
+#include <vespa/document/datatype/datatype.h>
+
+namespace document {
+
+class CollectionDataType : public DataType {
+    const DataType *_nestedType;
+
+protected:
+    CollectionDataType() : _nestedType(0) {}
+    CollectionDataType(const CollectionDataType&);
+    CollectionDataType& operator=(const CollectionDataType&);
+    CollectionDataType(const vespalib::stringref & name,
+                       const DataType &nestedType);
+    CollectionDataType(const vespalib::stringref & name,
+                       const DataType &nestedType, int32_t id);
+
+public:
+    virtual ~CollectionDataType();
+
+    bool operator==(const DataType&) const;
+
+    const DataType &getNestedType() const { return *_nestedType; }
+
+    DECLARE_IDENTIFIABLE_ABSTRACT(CollectionDataType);
+};
+
+} // document
+
+
diff --git a/document/src/vespa/document/datatype/datatype.cpp b/document/src/vespa/document/datatype/datatype.cpp
new file mode 100644
index 00000000000..b53162f25ba
--- /dev/null
+++ b/document/src/vespa/document/datatype/datatype.cpp
@@ -0,0 +1,184 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/document/datatype/datatype.h>
+#include <vespa/document/datatype/numericdatatype.h>
+#include <vespa/document/datatype/primitivedatatype.h>
+#include <vespa/document/datatype/documenttype.h>
+#include <vespa/document/datatype/weightedsetdatatype.h>
+#include <vespa/document/fieldvalue/fieldvalues.h>
+#include <vespa/vespalib/text/lowercase.h>
+#include <stdexcept>
+
+namespace document {
+
+IMPLEMENT_IDENTIFIABLE_ABSTRACT(DataType, vespalib::Identifiable);
+
+namespace {
+NumericDataType BYTE_OBJ(DataType::T_BYTE);
+NumericDataType SHORT_OBJ(DataType::T_SHORT);
+NumericDataType INT_OBJ(DataType::T_INT);
+NumericDataType LONG_OBJ(DataType::T_LONG);
+NumericDataType FLOAT_OBJ(DataType::T_FLOAT);
+NumericDataType DOUBLE_OBJ(DataType::T_DOUBLE);
+PrimitiveDataType STRING_OBJ(DataType::T_STRING);
+PrimitiveDataType RAW_OBJ(DataType::T_RAW);
+DocumentType DOCUMENT_OBJ("document");
+WeightedSetDataType TAG_OBJ(*DataType::STRING, true, true);
+PrimitiveDataType URI_OBJ(DataType::T_URI);
+PrimitiveDataType PREDICATE_OBJ(DataType::T_PREDICATE);
+PrimitiveDataType TENSOR_OBJ(DataType::T_TENSOR);
+
+}  // namespace
+
+const DataType *const DataType::BYTE(&BYTE_OBJ);
+const DataType *const DataType::SHORT(&SHORT_OBJ);
+const DataType *const DataType::INT(&INT_OBJ);
+const DataType *const DataType::LONG(&LONG_OBJ);
+const DataType *const DataType::FLOAT(&FLOAT_OBJ);
+const DataType *const DataType::DOUBLE(&DOUBLE_OBJ);
+const DataType *const DataType::STRING(&STRING_OBJ);
+const DataType *const DataType::RAW(&RAW_OBJ);
+const DocumentType *const DataType::DOCUMENT(&DOCUMENT_OBJ);
+const DataType *const DataType::TAG(&TAG_OBJ);
+const DataType *const DataType::URI(&URI_OBJ);
+const DataType *const DataType::PREDICATE(&PREDICATE_OBJ);
+const DataType *const DataType::TENSOR(&TENSOR_OBJ);
+
+namespace {
+
+class DataType2FieldValueId
+{
+public:
+    DataType2FieldValueId();
+    unsigned int getFieldValueId(unsigned int id) const {
+        return id < sizeof(_type2FieldValueId)/sizeof(_type2FieldValueId[0])
+               ? _type2FieldValueId[id]
+               : 0;
+    }
+private:
+    unsigned int _type2FieldValueId[DataType::MAX];
+};
+
+DataType2FieldValueId::DataType2FieldValueId()
+{
+    for (size_t i(0); i < sizeof(_type2FieldValueId)/sizeof(_type2FieldValueId[0]); i++) {
+        _type2FieldValueId[i] = 0;
+    }
+    _type2FieldValueId[DataType::T_BYTE]  = ByteFieldValue::classId;
+    _type2FieldValueId[DataType::T_SHORT] = ShortFieldValue::classId;
+    _type2FieldValueId[DataType::T_INT] = IntFieldValue::classId;
+    _type2FieldValueId[DataType::T_LONG] = LongFieldValue::classId;
+    _type2FieldValueId[DataType::T_FLOAT] = FloatFieldValue::classId;
+    _type2FieldValueId[DataType::T_DOUBLE] = DoubleFieldValue::classId;
+    _type2FieldValueId[DataType::T_STRING] = StringFieldValue::classId;
+    _type2FieldValueId[DataType::T_RAW] = RawFieldValue::classId;
+    _type2FieldValueId[DataType::T_URI] = StringFieldValue::classId;
+    _type2FieldValueId[DataType::T_PREDICATE] = PredicateFieldValue::classId;
+    _type2FieldValueId[DataType::T_TENSOR] = TensorFieldValue::classId;
+}
+
+DataType2FieldValueId _G_type2FieldValueId;
+
+}
+
+bool DataType::isValueType(const FieldValue & fv) const
+{
+    if ((_dataTypeId >= 0) && _dataTypeId < MAX) {
+        const uint32_t cid(_G_type2FieldValueId.getFieldValueId(_dataTypeId));
+        if (cid != 0) {
+            return cid == fv.getClass().id();
+        }
+    }
+    return _dataTypeId == fv.getDataType()->getId();
+}
+
+std::vector<const DataType *>
+DataType::getDefaultDataTypes()
+{
+    std::vector<const DataType *> types;
+    types.push_back(BYTE);
+    types.push_back(SHORT);
+    types.push_back(INT);
+    types.push_back(LONG);
+    types.push_back(FLOAT);
+    types.push_back(DOUBLE);
+    types.push_back(STRING);
+    types.push_back(RAW);
+    types.push_back(DOCUMENT);
+    types.push_back(TAG);
+    types.push_back(URI);
+    types.push_back(PREDICATE);
+    types.push_back(TENSOR);
+    return types;
+}
+
+namespace {
+// This should be equal to java implementation if name only has 7-bit
+// ASCII characters. Probably screwed up otherwise, but generated ids
+// should only be used in testing anyways. In production this will be
+// set from the document manager config.
+uint32_t crappyJavaStringHash(const vespalib::stringref & value) {
+    uint32_t h = 0;
+    for (uint32_t i = 0; i < value.size(); ++i) {
+        h = 31 * h + value[i];
+    }
+    return h;
+}
+
+int32_t createId(const vespalib::stringref & name)
+{
+    if (name == "Tag") {
+        return DataType::T_TAG;
+    }
+    return crappyJavaStringHash(vespalib::LowerCase::convert(name));
+}
+
+} // anon namespace
+
+DataType::DataType()
+    : _dataTypeId(-1),
+      _name("invalid")
+{
+}
+
+DataType::DataType(const vespalib::stringref & name, int dataTypeId)
+    : _dataTypeId(dataTypeId),
+      _name(name)
+{
+}
+
+DataType::DataType(const vespalib::stringref & name)
+    : _dataTypeId(createId(name)),
+      _name(name)
+{
+}
+
+DataType::~DataType()
+{
+}
+
+bool
+DataType::operator==(const DataType& other) const
+{
+    return _dataTypeId == other._dataTypeId && _name == other._name;
+}
+
+bool
+DataType::operator<(const DataType& other) const
+{
+    if (this == &other) return false;
+    return (_dataTypeId < other._dataTypeId);
+}
+
+FieldPath::UP
+DataType::buildFieldPath(const vespalib::stringref  & remainFieldName) const
+{
+//    LOG(debug, "remainFieldName = %s, dataType=%s", remainFieldName.c_str(), getClass().name());
+    if ( !remainFieldName.empty() ) {
+        return onBuildFieldPath(remainFieldName);
+    }
+    return FieldPath::UP(new FieldPath());
+}
+
+} // document
diff --git a/document/src/vespa/document/datatype/datatype.h b/document/src/vespa/document/datatype/datatype.h
new file mode 100644
index 00000000000..be9f8f1ded6
--- /dev/null
+++ b/document/src/vespa/document/datatype/datatype.h
@@ -0,0 +1,144 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * \class document::DataType
+ * \ingroup datatype
+ *
+ * \brief Specifies what is legal to store in a given field value.
+ */
+#pragma once
+
+#include <memory>
+
+#include <vespa/vespalib/objects/cloneable.h>
+#include <vespa/vespalib/objects/identifiable.h>
+
+#include <vespa/document/util/identifiableid.h>
+#include <vespa/document/util/printable.h>
+#include <memory>
+#include <vector>
+#include <vespa/document/base/fieldpath.h>
+
+namespace document {
+
+class FieldValue;
+class Field;
+class NumericDataType;
+class PrimitiveDataType;
+class DocumentType;
+class WeightedSetDataType;
+
+class DataType : public vespalib::Cloneable,
+                 public Printable,
+                 public vespalib::Identifiable
+{
+    int _dataTypeId;
+    vespalib::string _name;
+
+protected:
+    DataType();
+    /**
+     * Creates a datatype. Note that datatypes must be configured to work with
+     * the entire system, you can't just create them on the fly and expect
+     * everyone to be able to use them. Only tests and the type manager reading
+     * config should need to create datatypes.
+     */
+    DataType(const vespalib::stringref& name, int dataTypeId);
+
+    /**
+     * Creates a datatype using the hash of name as the id.
+     */
+    explicit DataType(const vespalib::stringref& name);
+
+public:
+    virtual ~DataType();
+    typedef std::unique_ptr<DataType> UP;
+    typedef std::shared_ptr<DataType> SP;
+    typedef vespalib::CloneablePtr<DataType> CP;
+
+    /**
+     * Enumeration of primitive data type identifiers. (Complex types uses
+     * hashed identifiers.
+     *
+     * <b>NOTE:</b> These types are also defined in the java source (in file
+     * "document/src/java/com/yahoo/document/DataType.java". Changes done
+     * here must also be applied there.
+     */
+    enum Type {
+        T_INT         =  0,
+        T_FLOAT       =  1,
+        T_STRING      =  2,
+        T_RAW         =  3,
+        T_LONG        =  4,
+        T_DOUBLE      =  5,
+        T_DOCUMENT    =  8, // Type of super document type Document.0 that all documents inherit.
+        // T_TIMESTAMP   =  9,  // Not used anymore, Id should probably not be reused
+        T_URI         = 10,
+        // T_EXACTSTRING = 11,  // Not used anymore, Id should probably not be reused
+        // T_CONTENT     = 12,  // Not used anymore, Id should probably not be reused
+        // T_CONTENTMETA = 13,  // Not used anymore, Id should probably not be reused
+        // T_MAILADDRESS = 14,  // Not used anymore, Id should probably not be reused
+        // T_TERMBOOST   = 15,  // Not used anymore, Id should probably not be reused
+        T_BYTE        = 16,
+        T_TAG         = 18,
+        T_SHORT       = 19,
+        T_PREDICATE   = 20,
+        T_TENSOR      = 21,
+        MAX
+    };
+
+    static const DataType *const BYTE;
+    static const DataType *const SHORT;
+    static const DataType *const INT;
+    static const DataType *const LONG;
+    static const DataType *const FLOAT;
+    static const DataType *const DOUBLE;
+    static const DataType *const STRING;
+    static const DataType *const RAW;
+    static const DocumentType *const DOCUMENT;
+    static const DataType *const TAG;
+    static const DataType *const URI;
+    static const DataType *const PREDICATE;
+    static const DataType *const TENSOR;
+
+    /** Used by type manager to fetch default types to register. */
+    static std::vector<const DataType *> getDefaultDataTypes();
+
+
+    const vespalib::string& getName() const { return _name; }
+    int getId() const { return _dataTypeId; }
+    bool isValueType(const FieldValue & fv) const;
+
+    /**
+     * Create a field value using this datatype.
+     */
+    virtual std::unique_ptr<FieldValue> createFieldValue() const = 0;
+    virtual DataType* clone() const = 0;
+
+    /**
+     * Whether another datatype is a supertype of this one. Document types may
+     * be due to inheritance. For other types, they must be identical for this
+     * to match.
+     */
+    virtual bool isA(const DataType& other) const { return (*this == other); }
+
+    virtual bool operator==(const DataType&) const;
+    virtual bool operator<(const DataType&) const;
+    bool operator != (const DataType & rhs) const { return !(*this == rhs); }
+
+    /**
+     * This takes a . separated fieldname and gives you back the path of
+     * fields you have to apply to get to your leaf.
+     * @param remainFieldName. The remaining part of the fieldname that you want the path of.
+     * @return pointer to field path or null if an error occured
+     */
+    FieldPath::UP buildFieldPath(
+            const vespalib::stringref & remainFieldName) const;
+
+    DECLARE_IDENTIFIABLE_ABSTRACT(DataType);
+private:
+    virtual FieldPath::UP onBuildFieldPath(
+            const vespalib::stringref & remainFieldName) const = 0;
+};
+
+} // document
+
diff --git a/document/src/vespa/document/datatype/datatypes.h b/document/src/vespa/document/datatype/datatypes.h
new file mode 100644
index 00000000000..19dfe2b1f97
--- /dev/null
+++ b/document/src/vespa/document/datatype/datatypes.h
@@ -0,0 +1,12 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/document/datatype/numericdatatype.h>
+#include <vespa/document/datatype/primitivedatatype.h>
+#include <vespa/document/datatype/arraydatatype.h>
+#include <vespa/document/datatype/mapdatatype.h>
+#include <vespa/document/datatype/weightedsetdatatype.h>
+#include <vespa/document/datatype/structdatatype.h>
+#include <vespa/document/datatype/documenttype.h>
+
diff --git a/document/src/vespa/document/datatype/documenttype.cpp b/document/src/vespa/document/datatype/documenttype.cpp
new file mode 100644
index 00000000000..0a679d75e96
--- /dev/null
+++ b/document/src/vespa/document/datatype/documenttype.cpp
@@ -0,0 +1,252 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/document/datatype/documenttype.h>
+
+#include <vespa/document/fieldvalue/document.h>
+#include <iomanip>
+#include <vespa/log/log.h>
+
+LOG_SETUP(".document.datatype.document");
+
+using vespalib::IllegalArgumentException;
+using vespalib::make_string;
+using vespalib::stringref;
+
+namespace document {
+
+IMPLEMENT_IDENTIFIABLE(DocumentType, StructuredDataType);
+
+DocumentType::DocumentType()
+{
+}
+
+DocumentType::DocumentType(const stringref& name, int32_t id)
+    : StructuredDataType(name, id),
+      _inheritedTypes(),
+      _ownedFields(new StructDataType(name + ".header")),
+      _fields(_ownedFields.get()),
+      _fieldSets()
+{
+    if (name != "document") {
+        _inheritedTypes.push_back(DataType::DOCUMENT);
+    }
+}
+
+DocumentType::DocumentType(const stringref& name, int32_t id, const StructDataType& fields)
+    : StructuredDataType(name, id),
+      _inheritedTypes(),
+      _fields(&fields),
+      _fieldSets()
+{
+    if (name != "document") {
+        _inheritedTypes.push_back(DataType::DOCUMENT);
+    }
+}
+
+DocumentType::DocumentType(const stringref& name)
+    : StructuredDataType(name),
+      _inheritedTypes(),
+      _ownedFields(new StructDataType(name + ".header")),
+      _fields(_ownedFields.get()),
+      _fieldSets()
+{
+    if (name != "document") {
+        _inheritedTypes.push_back(DataType::DOCUMENT);
+    }
+}
+
+DocumentType::DocumentType(const stringref& name, const StructDataType& fields)
+    : StructuredDataType(name),
+      _inheritedTypes(),
+      _fields(&fields),
+      _fieldSets()
+{
+    if (name != "document") {
+        _inheritedTypes.push_back(DataType::DOCUMENT);
+    }
+}
+
+DocumentType::~DocumentType()
+{
+}
+
+DocumentType &
+DocumentType::addFieldSet(const vespalib::string & name, const FieldSet::Fields & fields)
+{
+    for (FieldSet::Fields::const_iterator it(fields.begin()), mt(fields.end()); it != mt; it++) {
+        if ( ! _fields->hasField(*it) ) {
+            FieldPath::UP fieldPath = _fields->buildFieldPath(*it);
+            if (fieldPath.get() == nullptr) {
+                throw IllegalArgumentException("Fieldset '" + name + "': No field with name '" + *it +
+                                               "' in document type '" + getName() + "'.", VESPA_STRLOC);
+            }
+        }
+    }
+    _fieldSets[name] = FieldSet(name, fields);
+    return *this;
+}
+
+const DocumentType::FieldSet *
+DocumentType::getFieldSet(const vespalib::string & name) const
+{
+    FieldSetMap::const_iterator it(_fieldSets.find(name));
+    return (it != _fieldSets.end()) ? & it->second : NULL;
+}
+
+void
+DocumentType::addField(const Field& field)
+{
+    if (_fields->hasField(field.getName())) {
+        throw IllegalArgumentException( "A field already exists with name " + field.getName(),
+                VESPA_STRLOC);
+    } else if (_fields->hasField(field)) {
+        throw IllegalArgumentException(make_string(
+                "A field already exists with id %i (or deprecated id %i).",
+                field.getId(7), field.getId(6)), VESPA_STRLOC);
+    } else if (!_ownedFields.get()) {
+        throw vespalib::IllegalStateException(make_string(
+                        "Cannot add field %s to a DocumentType that does not "
+                        "own its fields.", field.getName().c_str()),
+                                              VESPA_STRLOC);
+    }
+    _ownedFields->addField(field);
+}
+
+void
+DocumentType::inherit(const DocumentType &docType) {
+    if (docType.getName() == "document") {
+        return;
+    }
+    if (docType.isA(*this)) {
+        throw IllegalArgumentException(
+                "Document type " + docType.toString() + " already inherits type "
+                + toString() + ". Cannot add cyclic dependencies.", VESPA_STRLOC);
+    }
+        // If we already inherits this type, there is no point in adding it
+        // again.
+    if (isA(docType)) {
+            // If we already directly inherits it, complain
+        for (std::vector<const DocumentType *>::const_iterator
+                it = _inheritedTypes.begin(); it != _inheritedTypes.end(); ++it)
+        {
+            if (**it == docType) {
+                throw IllegalArgumentException(
+                        "DocumentType " + getName() + " already inherits "
+                        "document type " + docType.getName(), VESPA_STRLOC);
+            }
+        }
+            // Indirectly already inheriting it is oki, as this can happen
+            // due to inherited documents inheriting the same type.
+        LOG(info, "Document type %s inherits document type %s from multiple "
+                  "types.", getName().c_str(), docType.getName().c_str());
+        return;
+    }
+        // Add non-conflicting types.
+    Field::Set fs = docType._fields->getFieldSet();
+    for (Field::Set::const_iterator it = fs.begin(); it != fs.end(); ++it) {
+        if (!_ownedFields.get()) {
+            _ownedFields.reset(_fields->clone());
+            _fields = _ownedFields.get();
+        }
+        _ownedFields->addInheritedField(**it);
+    }
+    // If we inherit default document type Document.0, remove that if adding
+    // another parent, as that has to also inherit Document
+    if (_inheritedTypes.size() == 1 && *_inheritedTypes[0] == *DataType::DOCUMENT) {
+        _inheritedTypes.clear();
+    }
+    _inheritedTypes.push_back(&docType);
+}
+
+bool
+DocumentType::isA(const DataType& other) const
+{
+    for (std::vector<const DocumentType *>::const_iterator
+         it = _inheritedTypes.begin(); it != _inheritedTypes.end(); ++it)
+    {
+        if ((*it)->isA(other)) return true;
+    }
+    return (*this == other);
+}
+
+FieldValue::UP
+DocumentType::createFieldValue() const
+{
+    return FieldValue::UP(new Document(*this, DocumentId("doc::")));
+}
+
+void
+DocumentType::print(std::ostream& out, bool verbose,
+                    const std::string& indent) const
+{
+    out << "DocumentType(" << getName();
+    if (verbose) {
+        out << ", id " << getId();
+    }
+    out << ")";
+    if (verbose) {
+        if (!_inheritedTypes.empty()) {
+            std::vector<const DocumentType *>::const_iterator it(
+                    _inheritedTypes.begin());
+            out << "\n" << indent << "    : ";
+            (*it)->print(out, false, "");
+            while (++it != _inheritedTypes.end()) {
+                out << ",\n" << indent << "      ";
+                (*it)->print(out, false, "");
+            }
+        }
+        out << " {\n" << indent << "  ";
+        _fields->print(out, verbose, indent + "  ");
+        out << "\n" << indent << "}";
+    }
+}
+
+bool
+DocumentType::operator==(const DataType& other) const
+{
+    if (&other == this) return true;
+    if (!DataType::operator==(other)) return false;
+    const DocumentType* o(dynamic_cast<const DocumentType*>(&other));
+    if (o == 0) return false;
+    if (*_fields != *o->_fields) return false;
+    if (_inheritedTypes.size() != o->_inheritedTypes.size()) return false;
+    std::vector<const DocumentType *>::const_iterator it1(
+            _inheritedTypes.begin());
+    std::vector<const DocumentType *>::const_iterator it2(
+            o->_inheritedTypes.begin());
+    while (it1 != _inheritedTypes.end()) {
+        if (**it1 != **it2) return false;
+        ++it1;
+        ++it2;
+    }
+    return true;
+}
+
+const Field&
+DocumentType::getField(const stringref& name) const
+{
+    return _fields->getField(name);
+}
+
+const Field&
+DocumentType::getField(int fieldId, int version) const
+{
+    return _fields->getField(fieldId, version);
+}
+
+bool DocumentType::hasField(const stringref &name) const {
+    return _fields->hasField(name);
+}
+
+bool DocumentType::hasField(int fieldId, int version) const {
+    return _fields->hasField(fieldId, version);
+}
+
+Field::Set
+DocumentType::getFieldSet() const
+{
+    return _fields->getFieldSet();
+}
+
+} // document
diff --git a/document/src/vespa/document/datatype/documenttype.h b/document/src/vespa/document/datatype/documenttype.h
new file mode 100644
index 00000000000..73292b339fa
--- /dev/null
+++ b/document/src/vespa/document/datatype/documenttype.h
@@ -0,0 +1,100 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * \class document::DocumentType
+ * \ingroup datatype
+ *
+ * \brief A class describing what can be contained in a document of this type.
+ *
+ * A document type can inherit other document types. All document types inherit
+ * "document" type.
+ */
+
+#pragma once
+
+#include <vespa/document/datatype/structdatatype.h>
+
+#include <vector>
+
+namespace document {
+
+class Field;
+class DocumentType;
+
+class DocumentType : public StructuredDataType {
+public:
+    class FieldSet {
+    public:
+        typedef std::set<vespalib::string> Fields;
+        FieldSet() : _name(), _fields() {}
+        FieldSet(const vespalib::string & name) : _name(name), _fields() {}
+        FieldSet(const vespalib::string & name, const Fields & fields) : _name(name), _fields(fields) {}
+        const vespalib::string & getName() const { return _name; }
+        const Fields & getFields() const { return _fields; }
+        FieldSet & add(vespalib::string & field) {
+            _fields.insert(field);
+            return *this;
+        }
+    private:
+        vespalib::string _name;
+        Fields           _fields;
+    };
+    typedef std::map<vespalib::string, FieldSet> FieldSetMap;
+    std::vector<const DocumentType *> _inheritedTypes;
+    StructDataType::SP _ownedFields;
+    const StructDataType* _fields;
+    FieldSetMap _fieldSets;
+
+public:
+    typedef std::unique_ptr<DocumentType> UP;
+    typedef std::shared_ptr<DocumentType> SP;
+
+    DocumentType();
+    DocumentType(const vespalib::stringref &name, int32_t id);
+    DocumentType(const vespalib::stringref &name, int32_t id,
+                 const StructDataType& fields);
+
+    DocumentType(const vespalib::stringref &name);
+    DocumentType(const vespalib::stringref &name,
+                 const StructDataType& fields);
+
+    virtual ~DocumentType();
+
+    const StructDataType& getFieldsType() const { return *_fields; }
+
+    void addField(const Field&);
+
+    /**
+     * Add a documenttype this type inherits from. The order inherited
+     * types are added decides which parent fields are used if
+     * multiple parents define the same fields.
+     */
+    void inherit(const DocumentType &docType);
+
+    virtual bool isA(const DataType& other) const;
+
+    const std::vector<const DocumentType *> & getInheritedTypes() const
+        { return _inheritedTypes; };
+
+        // Implementation of StructuredDataType
+    virtual FieldValue::UP createFieldValue() const;
+    virtual void print(
+            std::ostream&, bool verbose, const std::string& indent) const;
+    virtual bool operator==(const DataType& type) const;
+    virtual uint32_t getFieldCount() const {
+        return _fields->getFieldCount();
+    }
+    virtual const Field & getField(const vespalib::stringref & name) const;
+    virtual const Field & getField(int fieldId, int version) const;
+    virtual bool hasField(const vespalib::stringref &name) const;
+    virtual bool hasField(int fieldId, int version) const;
+    virtual Field::Set getFieldSet() const;
+    virtual DocumentType* clone() const { return new DocumentType(*this); }
+
+    DocumentType & addFieldSet(const vespalib::string & name, const FieldSet::Fields & fields);
+    const FieldSet * getFieldSet(const vespalib::string & name) const;
+
+    DECLARE_IDENTIFIABLE(DocumentType);
+};
+
+} // document
+
diff --git a/document/src/vespa/document/datatype/mapdatatype.cpp b/document/src/vespa/document/datatype/mapdatatype.cpp
new file mode 100644
index 00000000000..c56e72d1d71
--- /dev/null
+++ b/document/src/vespa/document/datatype/mapdatatype.cpp
@@ -0,0 +1,126 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/document/datatype/mapdatatype.h>
+
+#include <vespa/document/datatype/primitivedatatype.h>
+#include <vespa/document/fieldvalue/mapfieldvalue.h>
+#include <vespa/vespalib/stllike/asciistream.h>
+
+namespace document {
+
+IMPLEMENT_IDENTIFIABLE(MapDataType, DataType);
+
+namespace {
+vespalib::string createName(const DataType& keyType, const DataType& valueType)
+{
+    vespalib::asciistream ost;
+    ost << "Map<" << keyType.getName() << "," << valueType.getName() << ">";
+    return ost.str();
+}
+}  // namespace
+
+MapDataType::MapDataType(const DataType &key, const DataType &value)
+    : DataType(createName(key, value)),
+      _keyType(&key),
+      _valueType(&value) {
+}
+
+MapDataType::MapDataType(const DataType &key, const DataType &value, int id)
+    : DataType(createName(key, value), id),
+      _keyType(&key),
+      _valueType(&value) {
+}
+
+FieldValue::UP MapDataType::createFieldValue() const {
+    return FieldValue::UP(new MapFieldValue(*this));
+}
+
+void
+MapDataType::print(std::ostream& out, bool verbose,
+                   const std::string& indent) const
+{
+    out << "MapDataType(";
+    getKeyType().print(out, verbose, indent + "    ");
+    out << ", ";
+    getValueType().print(out, verbose, indent + "    ");
+    out << ", id " << getId() << ")";
+}
+
+bool
+MapDataType::operator==(const DataType& other) const
+{
+    if (this == &other) return true;
+    if (!DataType::operator==(other)) return false;
+    const MapDataType* w(Identifiable::cast<const MapDataType*>(&other));
+    return (*_keyType == *w->_keyType) && (*_valueType == *w->_valueType);
+}
+
+FieldPath::UP
+MapDataType::buildFieldPathImpl(const DataType &dataType,
+                                const vespalib::stringref &remainFieldName,
+                                const DataType &keyType,
+                                const DataType &valueType)
+{
+    if (!remainFieldName.empty() && remainFieldName[0] == '{') {
+        vespalib::string rest = remainFieldName;
+        vespalib::string keyValue = FieldPathEntry::parseKey(rest);
+
+        FieldPath::UP path =
+            valueType.buildFieldPath((rest[0] == '.') ? rest.substr(1) : rest);
+        if (!path.get()) {
+            return FieldPath::UP();
+        }
+
+        if (remainFieldName[1] == '$') {
+            path->insert(path->begin(),
+                         FieldPathEntry(valueType, keyValue.substr(1)));
+        } else {
+            FieldValue::UP fv = keyType.createFieldValue();
+            *fv = keyValue;
+            path->insert(path->begin(), FieldPathEntry(valueType, dataType,
+                                 vespalib::CloneablePtr<FieldValue>(fv.release())));
+        }
+
+        return path;
+    } else if (memcmp(remainFieldName.c_str(), "key", 3) == 0) {
+        size_t endPos = 3;
+        if (remainFieldName[endPos] == '.') {
+            endPos++;
+        }
+
+        FieldPath::UP path
+            = keyType.buildFieldPath(remainFieldName.substr(endPos));
+        if (!path.get()) {
+            return FieldPath::UP();
+        }
+        path->insert(path->begin(), FieldPathEntry(dataType, keyType,
+                                                   valueType, true, false));
+        return path;
+    } else if (memcmp(remainFieldName.c_str(), "value", 5) == 0) {
+        size_t endPos = 5;
+        if (remainFieldName[endPos] == '.') {
+            endPos++;
+        }
+
+        FieldPath::UP path
+            = valueType.buildFieldPath(remainFieldName.substr(endPos));
+        if (!path.get()) {
+            return FieldPath::UP();
+        }
+        path->insert(path->begin(), FieldPathEntry(dataType, keyType,
+                                                   valueType, false, true));
+        return path;
+    }
+
+    return keyType.buildFieldPath(remainFieldName);
+}
+
+FieldPath::UP
+MapDataType::onBuildFieldPath(const vespalib::stringref &remainFieldName) const
+{
+    return buildFieldPathImpl(*this, remainFieldName,
+                              getKeyType(), getValueType());
+}
+
+}  // namespace document
diff --git a/document/src/vespa/document/datatype/mapdatatype.h b/document/src/vespa/document/datatype/mapdatatype.h
new file mode 100644
index 00000000000..d48f2d80f38
--- /dev/null
+++ b/document/src/vespa/document/datatype/mapdatatype.h
@@ -0,0 +1,44 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * \class document::MapDataType
+ * \ingroup datatype
+ *
+ * \brief DataType describing a map
+ */
+#pragma once
+
+#include <vespa/document/datatype/datatype.h>
+
+namespace document {
+
+class MapDataType : public DataType {
+    const DataType *_keyType;
+    const DataType *_valueType;
+
+public:
+    MapDataType() : _keyType(0), _valueType(0) {}
+    MapDataType(const DataType &keyType, const DataType &valueType);
+    MapDataType(const DataType &keyType, const DataType &valueType, int id);
+
+    const DataType& getKeyType() const { return *_keyType; }
+    const DataType& getValueType() const { return *_valueType; }
+
+    virtual std::unique_ptr<FieldValue> createFieldValue() const;
+    virtual void print(std::ostream&, bool verbose,
+                       const std::string& indent) const;
+    virtual bool operator==(const DataType& other) const;
+    virtual MapDataType* clone() const { return new MapDataType(*this); }
+
+    FieldPath::UP onBuildFieldPath(
+            const vespalib::stringref &remainFieldName) const;
+    static FieldPath::UP buildFieldPathImpl(
+            const DataType& dataType,
+            const vespalib::stringref &remainFieldName,
+            const DataType &keyType,
+            const DataType &valueType);
+
+    DECLARE_IDENTIFIABLE(MapDataType);
+};
+
+} // document
+
diff --git a/document/src/vespa/document/datatype/numericdatatype.cpp b/document/src/vespa/document/datatype/numericdatatype.cpp
new file mode 100644
index 00000000000..46a2a4e6b7b
--- /dev/null
+++ b/document/src/vespa/document/datatype/numericdatatype.cpp
@@ -0,0 +1,19 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/document/datatype/numericdatatype.h>
+
+namespace document {
+
+IMPLEMENT_IDENTIFIABLE_ABSTRACT(NumericDataType, PrimitiveDataType);
+
+NumericDataType::NumericDataType(Type type)
+    : PrimitiveDataType(type)
+{
+}
+
+void NumericDataType::print(std::ostream& out, bool, const std::string&) const
+{
+    out << "NumericDataType(" << getName() << ", id " << getId() << ")";
+}
+}  // namespace document
diff --git a/document/src/vespa/document/datatype/numericdatatype.h b/document/src/vespa/document/datatype/numericdatatype.h
new file mode 100644
index 00000000000..cde5ac108ba
--- /dev/null
+++ b/document/src/vespa/document/datatype/numericdatatype.h
@@ -0,0 +1,33 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * \class document::NumericDataType
+ * \ingroup datatype
+ *
+ * \brief Data type holding numbers of various types.
+ *
+ * Data type object allowing you to store a number. This is typically only
+ * created when initializing the global primitive datatypes in the DataType
+ * class.
+ */
+#pragma once
+
+#include <vespa/document/datatype/primitivedatatype.h>
+
+namespace document {
+
+class NumericDataType : public PrimitiveDataType {
+public:
+    NumericDataType(Type type);
+
+        // Implementation of PrimitiveDataType
+    virtual NumericDataType* clone() const
+        { return new NumericDataType(*this); }
+    virtual void print(std::ostream&, bool verbose,
+                       const std::string& indent) const;
+
+    DECLARE_IDENTIFIABLE_ABSTRACT(NumericDataType);
+};
+
+}
+
+
diff --git a/document/src/vespa/document/datatype/positiondatatype.cpp b/document/src/vespa/document/datatype/positiondatatype.cpp
new file mode 100644
index 00000000000..28307476774
--- /dev/null
+++ b/document/src/vespa/document/datatype/positiondatatype.cpp
@@ -0,0 +1,62 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/document/datatype/positiondatatype.h>
+
+namespace document {
+
+namespace {
+
+const vespalib::string ZCURVE("_zcurve");
+
+}
+
+StructDataType::UP PositionDataType::_instance;
+vespalib::Lock     PositionDataType::_lock;
+
+const vespalib::string PositionDataType::STRUCT_NAME("position");
+const vespalib::string PositionDataType::FIELD_X("x");
+const vespalib::string PositionDataType::FIELD_Y("y");
+
+StructDataType::UP
+PositionDataType::createInstance()
+{
+    StructDataType::UP type(new StructDataType(PositionDataType::STRUCT_NAME));
+    type->addField(Field(PositionDataType::FIELD_X, *DataType::INT, true));
+    type->addField(Field(PositionDataType::FIELD_Y, *DataType::INT, true));
+    return type;
+}
+
+const StructDataType &
+PositionDataType::getInstance()
+{
+    if (_instance.get() == NULL) {
+        vespalib::LockGuard guard(_lock);
+        if (_instance.get() == NULL) {
+            _instance = createInstance();
+        }
+    }
+    return *_instance;
+}
+
+vespalib::string
+PositionDataType::getZCurveFieldName(const vespalib::string & fieldName)
+{
+    return fieldName + ZCURVE;
+}
+
+vespalib::stringref
+PositionDataType::cutZCurveFieldName(vespalib::stringref name)
+{
+    return name.substr(0, name.size() - 7);
+}
+
+bool
+PositionDataType::isZCurveFieldName(vespalib::stringref name)
+{
+    if (name.size() > ZCURVE.size()) {
+        return ZCURVE == name.substr(name.size() - ZCURVE.size());
+    }
+    return false;
+}
+
+} // document
diff --git a/document/src/vespa/document/datatype/positiondatatype.h b/document/src/vespa/document/datatype/positiondatatype.h
new file mode 100644
index 00000000000..a9f0079f0dc
--- /dev/null
+++ b/document/src/vespa/document/datatype/positiondatatype.h
@@ -0,0 +1,30 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/document/datatype/structdatatype.h>
+#include <vespa/vespalib/util/sync.h>
+
+namespace document {
+
+class PositionDataType {
+private:
+    static StructDataType::UP _instance;
+    static vespalib::Lock     _lock;
+
+    PositionDataType();
+    static StructDataType::UP createInstance();
+
+public:
+    static const vespalib::string STRUCT_NAME;
+    static const int              STRUCT_VERSION;
+    static const vespalib::string FIELD_X;
+    static const vespalib::string FIELD_Y;
+
+    static const StructDataType &getInstance();
+    static vespalib::string getZCurveFieldName(const vespalib::string &name);
+    static vespalib::stringref cutZCurveFieldName(vespalib::stringref name);
+    static bool isZCurveFieldName(vespalib::stringref name);
+};
+
+} // document
+
diff --git a/document/src/vespa/document/datatype/primitivedatatype.cpp b/document/src/vespa/document/datatype/primitivedatatype.cpp
new file mode 100644
index 00000000000..b335f3c63e9
--- /dev/null
+++ b/document/src/vespa/document/datatype/primitivedatatype.cpp
@@ -0,0 +1,93 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/document/datatype/primitivedatatype.h>
+
+#include <vespa/document/fieldvalue/fieldvalues.h>
+
+namespace document {
+
+IMPLEMENT_IDENTIFIABLE_ABSTRACT(PrimitiveDataType, DataType);
+
+namespace {
+    const char *Int = "Int";
+    const char *Short = "Short";
+    const char *Float = "Float";
+    const char *String = "String";
+    const char *Raw = "Raw";
+    const char *Long = "Long";
+    const char *Double = "Double";
+    const char *Uri = "Uri";
+    const char *Byte = "Byte";
+    const char *Predicate = "Predicate";
+    const char *Tensor = "Tensor";
+
+    const vespalib::stringref getTypeName(DataType::Type type) {
+        switch (type) {
+            case DataType::T_INT: return Int;
+            case DataType::T_SHORT: return Short;
+            case DataType::T_FLOAT: return Float;
+            case DataType::T_STRING: return String;
+            case DataType::T_RAW: return Raw;
+            case DataType::T_LONG: return Long;
+            case DataType::T_DOUBLE: return Double;
+            case DataType::T_URI: return Uri;
+            case DataType::T_BYTE: return Byte;
+            case DataType::T_PREDICATE: return Predicate;
+            case DataType::T_TENSOR: return Tensor;
+            default:
+                throw vespalib::IllegalArgumentException(vespalib::make_string(
+                        "Type %i is not a primitive type", type), VESPA_STRLOC);
+        }
+    }
+
+}
+
+PrimitiveDataType::PrimitiveDataType(Type type)
+    : DataType(getTypeName(type), type)
+{
+}
+
+FieldValue::UP
+PrimitiveDataType::createFieldValue() const
+{
+    switch (getId()) {
+        case T_INT: return FieldValue::UP(new IntFieldValue);
+        case T_SHORT: return FieldValue::UP(new ShortFieldValue);
+        case T_FLOAT: return FieldValue::UP(new FloatFieldValue);
+        case T_URI: return FieldValue::UP(new StringFieldValue);
+        case T_STRING: return FieldValue::UP(new StringFieldValue);
+        case T_RAW: return FieldValue::UP(new RawFieldValue);
+        case T_LONG: return FieldValue::UP(new LongFieldValue);
+        case T_DOUBLE: return FieldValue::UP(new DoubleFieldValue);
+        case T_BYTE: return FieldValue::UP(new ByteFieldValue);
+        case T_PREDICATE: return FieldValue::UP(new PredicateFieldValue);
+        case T_TENSOR: return std::make_unique<TensorFieldValue>();
+            abort();
+    }
+    assert(!"getId() returned value out of range");
+    abort();
+}
+
+void
+PrimitiveDataType::print(std::ostream& out, bool verbose,
+                         const std::string& indent) const
+{
+    (void) verbose; (void) indent;
+    out << "PrimitiveDataType(" << getName() << ", id " << getId() << ")";
+}
+
+FieldPath::UP
+PrimitiveDataType::onBuildFieldPath(const vespalib::stringref & rest) const
+{
+    if (rest.length()) {
+        std::ostringstream ost;
+        ost << "Datatype " << *this << " does not support further recursive structure: " << rest;
+        throw vespalib::IllegalArgumentException(ost.str());
+    }
+
+    return FieldPath::UP(new FieldPath());
+}
+
+
+} // document
diff --git a/document/src/vespa/document/datatype/primitivedatatype.h b/document/src/vespa/document/datatype/primitivedatatype.h
new file mode 100644
index 00000000000..e3b105a8597
--- /dev/null
+++ b/document/src/vespa/document/datatype/primitivedatatype.h
@@ -0,0 +1,39 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * \class document::PrimitiveDataType
+ * \ingroup datatype
+ *
+ * \brief Data type describing a primitive.
+ *
+ * This class describes a primitive data type. Normally you will not access
+ * this class directly, you'll use the global datatypes created in DataType,
+ * such as DataType::STRING and DataType::INT
+ *
+ * \todo Add a LiteralDataType subclass, such that this class can become
+ *       abstract. Right now you can create a PrimitiveDataType object with
+ *       a numeric type id, which is just plain wrong.
+ */
+#pragma once
+
+#include <vespa/document/datatype/datatype.h>
+
+namespace document {
+
+class PrimitiveDataType : public DataType {
+    virtual FieldPath::UP onBuildFieldPath(
+            const vespalib::stringref & remainFieldName) const;
+public:
+    PrimitiveDataType(Type _type);
+
+        // Implementation of DataType
+    virtual std::unique_ptr<FieldValue> createFieldValue() const;
+    virtual PrimitiveDataType* clone() const
+        { return new PrimitiveDataType(*this); }
+    virtual void print(std::ostream&, bool verbose, const std::string& indent) const;
+
+    DECLARE_IDENTIFIABLE_ABSTRACT(PrimitiveDataType);
+};
+
+}
+
+
diff --git a/document/src/vespa/document/datatype/structdatatype.cpp b/document/src/vespa/document/datatype/structdatatype.cpp
new file mode 100644
index 00000000000..659309714ba
--- /dev/null
+++ b/document/src/vespa/document/datatype/structdatatype.cpp
@@ -0,0 +1,232 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/document/datatype/structdatatype.h>
+
+#include <iomanip>
+#include <vespa/log/log.h>
+#include <vespa/document/base/exceptions.h>
+#include <vespa/document/fieldvalue/structfieldvalue.h>
+#include <vespa/document/fieldvalue/document.h>
+
+LOG_SETUP(".document.datatype.struct");
+
+namespace document {
+
+IMPLEMENT_IDENTIFIABLE(StructDataType, StructuredDataType);
+
+StructDataType::StructDataType() :
+    StructuredDataType(),
+    _nameFieldMap(),
+    _idFieldMap(),
+    _idFieldMapV6(),
+    _compressionConfig()
+{
+}
+
+StructDataType::StructDataType(const vespalib::stringref &name)
+    : StructuredDataType(name),
+      _nameFieldMap(),
+      _idFieldMap(),
+      _idFieldMapV6()
+{
+}
+
+StructDataType::StructDataType(const vespalib::stringref & name,
+                               int32_t dataTypeId)
+    : StructuredDataType(name, dataTypeId),
+      _nameFieldMap(),
+      _idFieldMap(),
+      _idFieldMapV6()
+{
+}
+
+void
+StructDataType::print(std::ostream& out, bool verbose,
+                      const std::string& indent) const
+{
+    out << "StructDataType(" << getName();
+    if (verbose) {
+        out << ", id " << getId();
+        if (_compressionConfig.type != CompressionConfig::NONE) {
+            out << ", Compression(" << _compressionConfig.type << ","
+                << int(_compressionConfig.compressionLevel) << ","
+                << int(_compressionConfig.threshold) << ")";
+        }
+    }
+    out << ")";
+    if (verbose) {
+        out << " {";
+        assert(_idFieldMap.size() == _nameFieldMap.size());
+        assert(_idFieldMapV6.size() == _nameFieldMap.size());
+        if (_nameFieldMap.size() > 0) {
+                // Use fieldset to print even though inefficient. Don't need
+                // efficient print, and this gets fields in order
+            Field::Set fields(getFieldSet());
+            for (Field::Set::const_iterator it = fields.begin();
+                 it != fields.end(); ++it)
+            {
+                out << "\n" << indent << "  " << (*it)->toString(verbose);
+            }
+            out << "\n" << indent;
+        }
+        out << "}";
+    }
+}
+
+void
+StructDataType::addField(const Field& field)
+{
+    vespalib::string error = containsConflictingField(field);
+    if (error != "") {
+        throw vespalib::IllegalArgumentException(vespalib::make_string(
+                "Failed to add field '%s' to struct '%s': %s",
+                field.getName().c_str(), getName().c_str(), error.c_str()), VESPA_STRLOC);
+    }
+    if (hasField(field.getName())) {
+        return;
+    }
+    std::shared_ptr<Field> newF(new Field(field));
+    _nameFieldMap[field.getName()] = newF;
+    _idFieldMap[field.getId(Document::getNewestSerializationVersion())] = newF;
+    _idFieldMapV6[field.getId(6)] = newF;
+}
+
+void
+StructDataType::addInheritedField(const Field& field)
+{
+    vespalib::string error = containsConflictingField(field);
+    if (error != "") {
+            // Deploy application should fail if overwriting a field with field
+            // of different type. Java version of document sees to this. C++
+            // just accepts what it gets, as to make it easier to alter the
+            // restrictions.
+        LOG(warning, "Inherited field %s conflicts with existing field. Field "
+                     "not added to struct %s: %s",
+            field.toString().c_str(), getName().c_str(), error.c_str());
+        return;
+    }
+    if (hasField(field.getName())) {
+        return;
+    }
+    std::shared_ptr<Field> newF(new Field(field));
+    _nameFieldMap[field.getName()] = newF;
+    _idFieldMap[field.getId(Document::getNewestSerializationVersion())] = newF;
+    _idFieldMapV6[field.getId(6)] = newF;
+}
+
+FieldValue::UP
+StructDataType::createFieldValue() const
+{
+    return FieldValue::UP(new StructFieldValue(*this));
+}
+
+const Field&
+StructDataType::getField(const vespalib::stringref & name) const
+{
+    StringFieldMap::const_iterator it(
+            _nameFieldMap.find(name));
+    if (it == _nameFieldMap.end()) {
+        throw FieldNotFoundException(name, VESPA_STRLOC);
+    } else {
+        return *it->second;
+    }
+}
+
+namespace {
+
+void throwFieldNotFound(int32_t fieldId, int version) __attribute__((noinline));
+
+void throwFieldNotFound(int32_t fieldId, int version)
+{
+    throw FieldNotFoundException(fieldId, version, VESPA_STRLOC);
+}
+
+}
+
+const Field&
+StructDataType::getField(int32_t fieldId, int version) const
+{
+    if (__builtin_expect(version > 6, true)) {
+        IntFieldMap::const_iterator it(_idFieldMap.find(fieldId));
+        if (__builtin_expect(it == _idFieldMap.end(), false)) {
+            throwFieldNotFound(fieldId, version);
+        }
+        return *it->second;
+    } else {
+        return getFieldV6(fieldId);
+    }
+}
+
+const Field&
+StructDataType::getFieldV6(int32_t fieldId) const
+{
+    IntFieldMap::const_iterator it(_idFieldMapV6.find(fieldId));
+    if (it == _idFieldMapV6.end()) {
+        throwFieldNotFound(fieldId, 6);
+    }
+    return *it->second;
+}
+
+bool StructDataType::hasField(const vespalib::stringref &name) const {
+    return _nameFieldMap.find(name) != _nameFieldMap.end();
+}
+
+bool StructDataType::hasField(int32_t fieldId, int version) const {
+    if (version > 6) {
+        return _idFieldMap.find(fieldId) != _idFieldMap.end();
+    } else {
+        return _idFieldMapV6.find(fieldId) != _idFieldMapV6.end();
+    }
+}
+
+Field::Set
+StructDataType::getFieldSet() const
+{
+    Field::Set fields;
+    for (IntFieldMap::const_iterator it = _idFieldMap.begin();
+         it != _idFieldMap.end(); ++it)
+    {
+        fields.insert(it->second.get());
+    }
+    return fields;
+}
+
+namespace {
+// We cannot use Field::operator==(), since that only compares id.
+bool differs(const Field &field1, const Field &field2) {
+    return field1.getId() != field2.getId()
+        || field1.getId(6) != field2.getId(6)
+        || field1.getName() != field2.getName();
+}
+}  // namespace
+
+vespalib::string
+StructDataType::containsConflictingField(const Field& field) const
+{
+    StringFieldMap::const_iterator it1( _nameFieldMap.find(field.getName()));
+    IntFieldMap::const_iterator it2(
+            _idFieldMap.find(field.getId(
+                    Document::getNewestSerializationVersion())));
+    IntFieldMap::const_iterator it3( _idFieldMapV6.find(field.getId(6)));
+
+    if (it1 != _nameFieldMap.end() && differs(field, *it1->second)) {
+        return vespalib::make_string(
+                "Name in use by field with different id %s.",
+                it1->second->toString().c_str());
+    }
+    if (it2 != _idFieldMap.end() && differs(field, *it2->second)) {
+        return vespalib::make_string(
+                "Field id in use by field %s.",
+                it2->second->toString().c_str());
+    }
+    if (it3 != _idFieldMapV6.end() && differs(field, *it3->second)) {
+        return vespalib::make_string(
+                "Version 6 document field id in use by field %s.",
+                it3->second->toString().c_str());
+    }
+
+    return "";
+}
+
+} // document
diff --git a/document/src/vespa/document/datatype/structdatatype.h b/document/src/vespa/document/datatype/structdatatype.h
new file mode 100644
index 00000000000..915dfcd19a4
--- /dev/null
+++ b/document/src/vespa/document/datatype/structdatatype.h
@@ -0,0 +1,87 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * \class document::StructDataType
+ * \ingroup datatype
+ *
+ * \brief A data type describing what can be contained in a struct field value.
+ *
+ * Describes what can be stored in a struct.
+ */
+#pragma once
+
+#include <vespa/document/datatype/structureddatatype.h>
+#include <vespa/vespalib/stllike/hash_map.h>
+#include <vespa/document/util/compressionconfig.h>
+#include <memory>
+
+namespace document {
+
+class StructDataType : public StructuredDataType {
+public:
+    typedef std::unique_ptr<StructDataType> UP;
+    typedef std::shared_ptr<StructDataType> SP;
+
+    StructDataType();
+    StructDataType(const vespalib::stringref &name);
+    StructDataType(const vespalib::stringref &name, int32_t id);
+
+    /**
+     * @throws vespalib::IllegalArgumentException if field conflicts with
+     *                                            already existing field.
+     */
+    void addField(const Field& field);
+
+    /**
+     * Similar to addField(field), but does not throw exceptions on errors.
+     * Fields that can be added are, and the other ones are skipped. Skipped
+     * fields will logs a warning informing about the conflict.
+     *
+     * This is typically called from DocumentType::inherit() to add the fields
+     * that does not conflict with existing fields.
+     */
+    void addInheritedField(const Field& field);
+
+        // Implementation of StructuredDataType
+    virtual FieldValue::UP createFieldValue() const;
+    virtual void print(std::ostream&, bool verbose, const std::string& indent) const;
+    virtual uint32_t getFieldCount() const { return _idFieldMap.size(); }
+
+    virtual const Field& getField(const vespalib::stringref & name) const;
+
+    /**
+     * Retrieves a field based on its ID. To determine which ID to use, we also
+     * need the document serialization version.
+     */
+    virtual const Field& getField(int32_t fieldId, int version) const;
+
+    virtual bool hasField(const vespalib::stringref &name) const;
+    virtual bool hasField(int32_t fieldId, int version) const;
+    bool hasField(const Field& f) const {
+        return hasField(f.getId(7), 7) || hasField(f.getId(6), 6);
+    }
+
+    virtual Field::Set getFieldSet() const;
+    virtual StructDataType* clone() const { return new StructDataType(*this); }
+
+    void setCompressionConfig(const CompressionConfig& cfg) { _compressionConfig = cfg; };
+    const CompressionConfig& getCompressionConfig() const { return _compressionConfig; }
+
+    DECLARE_IDENTIFIABLE(StructDataType);
+
+private:
+    typedef vespalib::hash_map<vespalib::string, Field::SP> StringFieldMap;
+    typedef vespalib::hash_map<int32_t, Field::SP> IntFieldMap;
+    StringFieldMap _nameFieldMap;
+    IntFieldMap _idFieldMap;
+    IntFieldMap _idFieldMapV6;
+
+    CompressionConfig _compressionConfig;
+
+    /** @return "" if not conflicting. Error message otherwise. */
+    vespalib::string containsConflictingField(const Field& field) const;
+    const Field& getFieldV6(int32_t fieldId) const __attribute__((noinline));
+};
+
+}
+
+
diff --git a/document/src/vespa/document/datatype/structureddatatype.cpp b/document/src/vespa/document/datatype/structureddatatype.cpp
new file mode 100644
index 00000000000..0fc55dddd1e
--- /dev/null
+++ b/document/src/vespa/document/datatype/structureddatatype.cpp
@@ -0,0 +1,94 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/document/datatype/structureddatatype.h>
+#include <stdexcept>
+#include <vespa/document/base/exceptions.h>
+#include <vespa/vespalib/stllike/asciistream.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".document.datatype.structured");
+
+namespace document {
+
+IMPLEMENT_IDENTIFIABLE_ABSTRACT(StructuredDataType, DataType);
+
+StructuredDataType::StructuredDataType() :
+    DataType()
+{
+}
+
+StructuredDataType::StructuredDataType(const vespalib::stringref &name)
+    : DataType(name, createId(name))
+{
+}
+
+StructuredDataType::StructuredDataType(const vespalib::stringref &name,
+                                       int dataTypeId)
+    : DataType(name, dataTypeId)
+{
+}
+
+bool StructuredDataType::operator==(const DataType& type) const
+{
+    if (!DataType::operator==(type)) return false;
+    return (dynamic_cast<const StructuredDataType*>(&type) != 0);
+}
+
+namespace {
+uint32_t crappyJavaStringHash(const vespalib::stringref &value) {
+    uint32_t h = 0;
+    for (uint32_t i = 0; i < value.size(); ++i) {
+        h = 31 * h + value[i];
+    }
+    return h;
+}
+}  // namespace
+
+int32_t StructuredDataType::createId(const vespalib::stringref &name)
+{
+    if (name == "document") {
+        return 8;
+    }
+    // This should be equal to java implementation if name only has 7-bit
+    // ASCII characters. Probably screwed up otherwise, but generated ids
+    // should only be used in testing anyways. In production this will be
+    // set from the document manager config.
+    vespalib::asciistream ost;
+    ost << name << ".0";  // Hardcode version 0 (version is not supported).
+    return crappyJavaStringHash(ost.str());
+}
+
+FieldPath::UP
+StructuredDataType::onBuildFieldPath(const vespalib::stringref & remainFieldName) const
+{
+    vespalib::stringref currFieldName(remainFieldName);
+    vespalib::stringref subFieldName;
+
+    for (uint32_t i = 0; i < remainFieldName.size(); i++) {
+        if (remainFieldName[i] == '.') {
+            currFieldName = remainFieldName.substr(0, i);
+            subFieldName = remainFieldName.substr(i + 1);
+            break;
+        } else if (remainFieldName[i] == '{' || remainFieldName[i] == '[') {
+            currFieldName = remainFieldName.substr(0, i);
+            subFieldName = remainFieldName.substr(i);
+            break;
+        }
+    }
+
+    // LOG(debug, "Field %s of datatype %s split into %s and %s", remainFieldName.c_str(), getName().c_str(), currFieldName.c_str(), subFieldName.c_str());
+    if (hasField(currFieldName)) {
+        const document::Field &fp = getField(currFieldName);
+        FieldPath::UP fieldPath = fp.getDataType().buildFieldPath(subFieldName);
+        if (!fieldPath.get()) {
+            return FieldPath::UP();
+        }
+        fieldPath->insert(fieldPath->begin(), FieldPathEntry(fp));
+        return fieldPath;
+    } else {
+        LOG(debug, "Could not find field %s", currFieldName.c_str());
+        return FieldPath::UP();
+    }
+}
+
+} // document
diff --git a/document/src/vespa/document/datatype/structureddatatype.h b/document/src/vespa/document/datatype/structureddatatype.h
new file mode 100644
index 00000000000..4ac0a310d2b
--- /dev/null
+++ b/document/src/vespa/document/datatype/structureddatatype.h
@@ -0,0 +1,55 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * \class document::StructuredDataType
+ * \ingroup datatype
+ *
+ * \brief Data type describing common parts for structured datatypes.
+ *
+ * This class contains common functionality for structured data types, like
+ * structs and documents.
+ */
+#pragma once
+
+#include <vespa/document/base/field.h>
+#include <vespa/document/datatype/datatype.h>
+#include <vespa/fastos/fastos.h> // Get uint16_t type on linux
+#include <set>
+#include <vespa/vespalib/util/exception.h>
+
+namespace document {
+
+class StructuredDataType : public DataType {
+    virtual FieldPath::UP onBuildFieldPath(const vespalib::stringref & remainFieldName) const;
+
+protected:
+    StructuredDataType();
+    StructuredDataType(const vespalib::stringref & name);
+    StructuredDataType(const vespalib::stringref & name, int32_t dataTypeId);
+
+
+public:
+    virtual uint32_t getFieldCount() const = 0;
+
+    /** @throws FieldNotFoundException if field does not exist. */
+    virtual const Field& getField(const vespalib::stringref & name) const = 0;
+
+    /** @throws FieldNotFoundException if field does not exist. */
+    virtual const Field& getField(int fieldId, int version) const = 0;
+
+    virtual bool hasField(const vespalib::stringref & name) const = 0;
+    virtual bool hasField(int32_t fieldId, int version) const = 0;
+
+    virtual Field::Set getFieldSet() const = 0;
+
+        // Implementation of DataType
+    virtual StructuredDataType* clone() const = 0;
+    virtual bool operator==(const DataType& type) const;
+
+    static int32_t createId(const vespalib::stringref &name);
+
+    DECLARE_IDENTIFIABLE_ABSTRACT(StructuredDataType);
+
+};
+
+}
+
diff --git a/document/src/vespa/document/datatype/urldatatype.cpp b/document/src/vespa/document/datatype/urldatatype.cpp
new file mode 100644
index 00000000000..aada45d0e23
--- /dev/null
+++ b/document/src/vespa/document/datatype/urldatatype.cpp
@@ -0,0 +1,45 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/document/datatype/urldatatype.h>
+
+namespace document {
+
+StructDataType::UP UrlDataType::_instance;
+vespalib::Lock     UrlDataType::_lock;
+
+const vespalib::string UrlDataType::STRUCT_NAME("url");
+const vespalib::string UrlDataType::FIELD_ALL("all");
+const vespalib::string UrlDataType::FIELD_SCHEME("scheme");
+const vespalib::string UrlDataType::FIELD_HOST("host");
+const vespalib::string UrlDataType::FIELD_PORT("port");
+const vespalib::string UrlDataType::FIELD_PATH("path");
+const vespalib::string UrlDataType::FIELD_QUERY("query");
+const vespalib::string UrlDataType::FIELD_FRAGMENT("fragment");
+
+StructDataType::UP
+UrlDataType::createInstance()
+{
+    StructDataType::UP type(new StructDataType(UrlDataType::STRUCT_NAME));
+    type->addField(Field(UrlDataType::FIELD_ALL,     *DataType::STRING, true));
+    type->addField(Field(UrlDataType::FIELD_SCHEME,  *DataType::STRING, true));
+    type->addField(Field(UrlDataType::FIELD_HOST,    *DataType::STRING, true));
+    type->addField(Field(UrlDataType::FIELD_PORT,    *DataType::STRING, true));
+    type->addField(Field(UrlDataType::FIELD_PATH,    *DataType::STRING, true));
+    type->addField(Field(UrlDataType::FIELD_QUERY,   *DataType::STRING, true));
+    type->addField(Field(UrlDataType::FIELD_FRAGMENT,*DataType::STRING, true));
+    return type;
+}
+
+const StructDataType &
+UrlDataType::getInstance()
+{
+    if (_instance.get() == NULL) {
+        vespalib::LockGuard guard(_lock);
+        if (_instance.get() == NULL) {
+            _instance = createInstance();
+        }
+    }
+    return *_instance;
+}
+
+} // document
diff --git a/document/src/vespa/document/datatype/urldatatype.h b/document/src/vespa/document/datatype/urldatatype.h
new file mode 100644
index 00000000000..2c704d8e322
--- /dev/null
+++ b/document/src/vespa/document/datatype/urldatatype.h
@@ -0,0 +1,32 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/document/datatype/structdatatype.h>
+#include <vespa/vespalib/util/sync.h>
+
+namespace document {
+
+class UrlDataType {
+private:
+    static StructDataType::UP _instance;
+    static vespalib::Lock     _lock;
+
+    UrlDataType() { /* hide */ }
+    static StructDataType::UP createInstance();
+
+public:
+    static const vespalib::string STRUCT_NAME;
+    static const int              STRUCT_VERSION;
+    static const vespalib::string FIELD_ALL;
+    static const vespalib::string FIELD_SCHEME;
+    static const vespalib::string FIELD_HOST;
+    static const vespalib::string FIELD_PORT;
+    static const vespalib::string FIELD_PATH;
+    static const vespalib::string FIELD_QUERY;
+    static const vespalib::string FIELD_FRAGMENT;
+
+    static const StructDataType &getInstance();
+};
+
+} // document
+
diff --git a/document/src/vespa/document/datatype/weightedsetdatatype.cpp b/document/src/vespa/document/datatype/weightedsetdatatype.cpp
new file mode 100644
index 00000000000..87714221735
--- /dev/null
+++ b/document/src/vespa/document/datatype/weightedsetdatatype.cpp
@@ -0,0 +1,94 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/document/datatype/weightedsetdatatype.h>
+
+#include <vespa/document/datatype/primitivedatatype.h>
+#include <vespa/document/fieldvalue/weightedsetfieldvalue.h>
+#include <vespa/vespalib/stllike/asciistream.h>
+
+namespace document {
+
+IMPLEMENT_IDENTIFIABLE(WeightedSetDataType, CollectionDataType);
+
+namespace {
+    vespalib::string createName(const DataType& nestedType, bool create, bool remove)
+    {
+        if (nestedType.getId() == DataType::T_STRING && create && remove) {
+            return "Tag";
+        }
+        vespalib::asciistream ost;
+        ost << "WeightedSet<" << nestedType.getName() << ">";
+        if (create) {
+            ost << ";Add";
+        }
+        if (remove) {
+            ost << ";Remove";
+        }
+        return ost.str();
+    }
+}
+
+WeightedSetDataType::WeightedSetDataType(
+        const DataType& nested, bool createIfNon, bool remove)
+    : CollectionDataType(createName(nested, createIfNon, remove), nested),
+      _createIfNonExistent(createIfNon),
+      _removeIfZero(remove)
+{
+}
+
+WeightedSetDataType::WeightedSetDataType(
+        const DataType& nested, bool createIfNon, bool remove, int id)
+    : CollectionDataType(createName(nested, createIfNon, remove), nested, id),
+      _createIfNonExistent(createIfNon),
+      _removeIfZero(remove)
+{
+}
+
+FieldValue::UP
+WeightedSetDataType::createFieldValue() const
+{
+    return FieldValue::UP(new WeightedSetFieldValue(*this));
+
+}
+
+void
+WeightedSetDataType::print(std::ostream& out, bool verbose,
+                           const std::string& indent) const
+{
+    if (getNestedType() == *DataType::STRING &&
+        _createIfNonExistent && _removeIfZero)
+    {
+        out << "Tag()";
+    } else {
+        out << "WeightedSetDataType(";
+        getNestedType().print(out, verbose, indent + "    ");
+        if (_createIfNonExistent) {
+            out << ", autoIfNonExistent";
+        }
+        if (_removeIfZero) {
+            out << ", removeIfZero";
+        }
+        out << ", id " << getId() << ")";
+    }
+}
+
+bool
+WeightedSetDataType::operator==(const DataType& other) const
+{
+    if (this == &other) return true;
+    if (!CollectionDataType::operator==(other)) return false;
+    const WeightedSetDataType* w(
+            dynamic_cast<const WeightedSetDataType*>(&other));
+    return (w != 0 && _createIfNonExistent == w->_createIfNonExistent
+                   && _removeIfZero == w->_removeIfZero);
+}
+
+FieldPath::UP
+WeightedSetDataType::onBuildFieldPath(const vespalib::stringref & remainFieldName) const
+{
+    return MapDataType::buildFieldPathImpl(
+            *this, remainFieldName, getNestedType(), *DataType::INT);
+}
+
+} // document
diff --git a/document/src/vespa/document/datatype/weightedsetdatatype.h b/document/src/vespa/document/datatype/weightedsetdatatype.h
new file mode 100644
index 00000000000..fcb8b982463
--- /dev/null
+++ b/document/src/vespa/document/datatype/weightedsetdatatype.h
@@ -0,0 +1,55 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * \class document::WeightedSetDataType
+ * \ingroup datatype
+ *
+ * \brief DataType describing a weighted set.
+ *
+ * Describes what can be stored and behaviour of weighted sets with this type.
+ * The create if non-existing and remove if zero weight functionality, as used
+ * in tagging, is a part of the type.
+ */
+#pragma once
+
+#include <vespa/document/datatype/collectiondatatype.h>
+
+namespace document {
+
+class WeightedSetDataType : public CollectionDataType {
+    bool _createIfNonExistent;
+    bool _removeIfZero;
+
+public:
+    WeightedSetDataType() {}
+    WeightedSetDataType(const DataType& nestedType,
+                        bool createIfNonExistent, bool removeIfZero);
+    WeightedSetDataType(const DataType& nestedType,
+                        bool createIfNonExistent, bool removeIfZero, int id);
+
+    /**
+     * @return Whether values of this datatype will autogenerate entries if
+     * operations that require an existing entries operates on non-existing
+     * ones.
+     */
+    bool createIfNonExistent() const { return _createIfNonExistent; };
+    /**
+     * @return Whether values of this datatype will automatically
+     *         remove entries with zero weight.
+     */
+    bool removeIfZero() const { return _removeIfZero; };
+
+        // CollectionDataType implementation
+    virtual std::unique_ptr<FieldValue> createFieldValue() const;
+    virtual void print(std::ostream&, bool verbose,
+                       const std::string& indent) const;
+    virtual bool operator==(const DataType& other) const;
+    virtual WeightedSetDataType* clone() const
+        { return new WeightedSetDataType(*this); }
+
+    FieldPath::UP onBuildFieldPath(const vespalib::stringref &remainFieldName) const;
+
+    DECLARE_IDENTIFIABLE(WeightedSetDataType);
+};
+
+} // document
+
diff --git a/document/src/vespa/document/document.h b/document/src/vespa/document/document.h
new file mode 100644
index 00000000000..f775b21d355
--- /dev/null
+++ b/document/src/vespa/document/document.h
@@ -0,0 +1,22 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ *
+ * \file document.h
+ *
+ * Include common client parts of document, such that clients can easily
+ * just include this file to get what they need.
+ *
+ * This should pull in all code needed for handling:
+ *    - Datatypes
+ *    - Fieldvalues
+ *    - Updates
+ *    - Selection language
+ */
+
+#pragma once
+
+#include <vespa/document/datatype/datatypes.h>
+#include <vespa/document/fieldvalue/fieldvalues.h>
+#include <vespa/document/update/updates.h>
+#include <vespa/document/select/parser.h>
+
diff --git a/document/src/vespa/document/fieldset/CMakeLists.txt b/document/src/vespa/document/fieldset/CMakeLists.txt
new file mode 100644
index 00000000000..a20a1bb22b2
--- /dev/null
+++ b/document/src/vespa/document/fieldset/CMakeLists.txt
@@ -0,0 +1,9 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_library(document_fieldset OBJECT
+    SOURCES
+    fieldsetrepo.cpp
+    fieldsets.cpp
+    DEPENDS
+    AFTER
+    document_documentconfig
+)
diff --git a/document/src/vespa/document/fieldset/fieldset.h b/document/src/vespa/document/fieldset/fieldset.h
new file mode 100644
index 00000000000..a206bdf64fa
--- /dev/null
+++ b/document/src/vespa/document/fieldset/fieldset.h
@@ -0,0 +1,80 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <memory>
+
+namespace document {
+
+class Document;
+
+/**
+ * FieldSet class. Represents a subset of fields in a document type.
+ * Note that the document id is counted as a field in this context,
+ * but referenced by the special name "[id]"
+ */
+class FieldSet
+{
+public:
+    enum Type {
+        FIELD,
+        SET,
+        ALL,
+        NONE,
+        HEADER, // Special collection used by VDS
+        BODY,   // Special collection used by VDS
+        DOCID
+    };
+
+    typedef std::unique_ptr<FieldSet> UP;
+
+    virtual ~FieldSet() {};
+
+    /**
+     * @return Return true if all the fields in "fields" are contained in
+     * this field set.
+     */
+    virtual bool contains(const FieldSet& fields) const = 0;
+
+    /**
+     * @return Returns the type of field set this is.
+     */
+    virtual Type getType() const = 0;
+
+    /**
+     * @return Returns a copy of this object.
+     */
+    virtual FieldSet* clone() const = 0;
+
+    /**
+     * Copy all fields from src into dest that are contained within the
+     * given field set. If any copied field pre-exists in dest, it will
+     * be overwritten.
+     * NOTE: causes each field to be explicitly copied so thus not very
+     * efficient. Prefer using stripFields for cases where a document
+     * needs to only contain fields matching a given field set and can
+     * readily be modified in-place.
+     */
+    static void copyFields(Document& dest,
+                           const Document& src,
+                           const FieldSet& fields);
+
+    /**
+     * Creates a copy of document src containing only the fields given by
+     * the fieldset. Document type and identifier remain the same.
+     * See comment for copyFields() for performance notes.
+     * @return The new, (partially) copied document instance.
+     */
+    static std::unique_ptr<Document> createDocumentSubsetCopy(
+            const Document& src,
+            const FieldSet& fields);
+
+    /**
+     * Strip all fields _except_ the ones that are contained within the
+     * fieldsToKeep fieldset. Modifies original document in-place.
+     */
+    static void stripFields(Document& doc,
+                            const FieldSet& fieldsToKeep);
+};
+
+}
+
diff --git a/document/src/vespa/document/fieldset/fieldsetrepo.cpp b/document/src/vespa/document/fieldset/fieldsetrepo.cpp
new file mode 100644
index 00000000000..860364207bd
--- /dev/null
+++ b/document/src/vespa/document/fieldset/fieldsetrepo.cpp
@@ -0,0 +1,127 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/document/fieldset/fieldsetrepo.h>
+#include <vespa/vespalib/text/stringtokenizer.h>
+#include <vespa/vespalib/util/exceptions.h>
+#include <vespa/vespalib/stllike/asciistream.h>
+
+using vespalib::StringTokenizer;
+
+namespace document {
+
+namespace {
+
+FieldSet::UP
+parseSpecialValues(const vespalib::stringref & name)
+{
+    FieldSet::UP fs;
+    if ((name.size() == 4) && (name[1] == 'i') && (name[2] == 'd') && (name[3] == ']')) {
+        fs.reset(new DocIdOnly());
+    } else if ((name.size() == 5) && (name[1] == 'a') && (name[2] == 'l') && (name[3] == 'l') && (name[4] == ']')) {
+        fs.reset(new AllFields());
+    } else if ((name.size() == 6) && (name[1] == 'n') && (name[2] == 'o') && (name[3] == 'n') && (name[4] == 'e') && (name[5] == ']')) {
+        fs.reset(new NoFields());
+    } else if ((name.size() == 8) && (name[1] == 'h') && (name[2] == 'e') && (name[3] == 'a') && (name[4] == 'd') && (name[5] == 'e') && (name[6] == 'r') && (name[7] == ']')) {
+        fs.reset(new HeaderFields());
+    } else if ((name.size() == 7) && (name[1] == 'd') && (name[2] == 'o') && (name[3] == 'c') && (name[4] == 'i') && (name[5] == 'd') && (name[6] == ']')) {
+        fs.reset(new DocIdOnly());
+    } else if ((name.size() == 6) && (name[1] == 'b') && (name[2] == 'o') && (name[3] == 'd') && (name[4] == 'y') && (name[5] == ']')) {
+        fs.reset(new BodyFields());
+    } else {
+        throw vespalib::IllegalArgumentException(
+                "The only special names (enclosed in '[]') allowed are "
+                "id, all, none, header, body, not '" + name + "'.");
+    }
+    return fs;
+}
+
+FieldSet::UP
+parseFieldCollection(const DocumentTypeRepo& repo,
+                     const vespalib::stringref & docType,
+                     const vespalib::stringref & fieldNames)
+{
+    const DocumentType* typePtr = repo.getDocumentType(docType);
+    if (!typePtr) {
+        throw vespalib::IllegalArgumentException(
+                "Unknown document type " + docType);
+    }
+    const DocumentType& type(*typePtr);
+
+    StringTokenizer tokenizer(fieldNames, ",");
+    FieldCollection::UP collection(new FieldCollection(type));
+
+    for (uint32_t i = 0; i < tokenizer.size(); ++i) {
+        const DocumentType::FieldSet * fs = type.getFieldSet(tokenizer[i]);
+        if (fs) {
+            for (DocumentType::FieldSet::Fields::const_iterator it(fs->getFields().begin()), mt(fs->getFields().end()); it != mt; it++) {
+                collection->insert(type.getField(*it));
+            }
+        } else {
+            collection->insert(type.getField(tokenizer[i]));
+        }
+    }
+
+    return FieldSet::UP(collection.release());
+}
+
+}
+
+FieldSet::UP
+FieldSetRepo::parse(const DocumentTypeRepo& repo, const vespalib::stringref & str)
+{
+    if (str[0] == '[') {
+        return parseSpecialValues(str);
+    } else {
+        StringTokenizer tokenizer(str, ":");
+        if (tokenizer.size() != 2) {
+            throw vespalib::IllegalArgumentException(
+                    "The field set list must consist of a document type, "
+                    "then a colon (:), then a comma-separated list of field names");
+        }
+
+        return parseFieldCollection(repo, tokenizer[0], tokenizer[1]);
+    }
+}
+
+vespalib::string
+FieldSetRepo::serialize(const FieldSet& fieldSet)
+{
+    switch (fieldSet.getType()) {
+    case FieldSet::FIELD:
+        return static_cast<const Field&>(fieldSet).getName();
+        break;
+    case FieldSet::SET:
+    {
+        const FieldCollection& collection = static_cast<const FieldCollection&>(fieldSet);
+
+        vespalib::asciistream stream;
+        stream << collection.getDocumentType().getName() << ":";
+        for (Field::Set::const_iterator iter = collection.getFields().begin();
+             iter != collection.getFields().end();
+             ++iter) {
+            if (iter != collection.getFields().begin()) {
+                stream << ",";
+            }
+
+            stream << (*iter)->getName();
+        }
+
+        return stream.str();
+    }
+    case FieldSet::ALL:
+        return "[all]";
+    case FieldSet::NONE:
+        return "[none]";
+    case FieldSet::HEADER:
+        return "[header]";
+    case FieldSet::BODY:
+        return "[body]";
+    case FieldSet::DOCID:
+        return "[docid]";
+    default:
+        return "";
+    }
+}
+
+}
+
diff --git a/document/src/vespa/document/fieldset/fieldsetrepo.h b/document/src/vespa/document/fieldset/fieldsetrepo.h
new file mode 100644
index 00000000000..d85fb49f9d1
--- /dev/null
+++ b/document/src/vespa/document/fieldset/fieldsetrepo.h
@@ -0,0 +1,26 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/document/fieldset/fieldset.h>
+#include <vespa/document/fieldset/fieldsets.h>
+#include <vespa/document/datatype/documenttype.h>
+#include <vespa/document/repo/documenttyperepo.h>
+
+namespace document {
+
+/**
+ * Class that has configuration for all document field sets.
+ * Responsible for parsing field set strings using a documenttype.
+ */
+class FieldSetRepo
+{
+public:
+    static FieldSet::UP parse(const DocumentTypeRepo& repo,
+                       const vespalib::stringref & fieldSetString);
+
+    static vespalib::string serialize(const FieldSet& fs);
+};
+
+}
+
+
diff --git a/document/src/vespa/document/fieldset/fieldsets.cpp b/document/src/vespa/document/fieldset/fieldsets.cpp
new file mode 100644
index 00000000000..9a87cb41e85
--- /dev/null
+++ b/document/src/vespa/document/fieldset/fieldsets.cpp
@@ -0,0 +1,182 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/document/fieldset/fieldsets.h>
+#include <vespa/document/fieldvalue/document.h>
+namespace document {
+
+bool
+HeaderFields::contains(const FieldSet& fields) const
+{
+    switch (fields.getType()) {
+    case FIELD:
+        return static_cast<const Field&>(fields).isHeaderField();
+    case SET:
+    {
+        const FieldCollection& coll = static_cast<const FieldCollection&>(fields);
+        for (Field::Set::const_iterator iter = coll.getFields().begin();
+             iter != coll.getFields().end();
+             ++iter) {
+            if (!(*iter)->isHeaderField()) {
+                return false;
+            }
+        }
+
+        return true;
+    }
+    case NONE:
+    case DOCID:
+    case HEADER:
+        return true;
+    case BODY:
+    case ALL:
+        return false;
+    }
+
+    return false;
+}
+
+bool
+BodyFields::contains(const FieldSet& fields) const
+{
+    switch (fields.getType()) {
+    case FIELD:
+        return !static_cast<const Field&>(fields).isHeaderField();
+    case SET:
+    {
+        const FieldCollection& coll = static_cast<const FieldCollection&>(fields);
+        for (Field::Set::const_iterator iter = coll.getFields().begin();
+             iter != coll.getFields().end();
+             ++iter) {
+            if ((*iter)->isHeaderField()) {
+                return false;
+            }
+        }
+
+        return true;
+    }
+    case NONE:
+    case DOCID:
+    case BODY:
+        return true;
+    case HEADER:
+    case ALL:
+        return false;
+    }
+
+    return false;
+}
+
+FieldCollection::FieldCollection(const DocumentType& type,
+                                 const Field::Set& s)
+    : _set(s),
+      _docType(type)
+{
+}
+
+bool
+FieldCollection::contains(const FieldSet& fields) const
+{
+    switch (fields.getType()) {
+    case FIELD:
+        return _set.find(static_cast<const Field*>(&fields)) != _set.end();
+    case SET:
+    {
+        const FieldCollection& coll = static_cast<const FieldCollection&>(fields);
+
+        if (_set.size() < coll._set.size()) {
+            return false;
+        }
+
+        Field::Set::const_iterator iter = coll.getFields().begin();
+
+        while (iter != coll.getFields().end()) {
+            if (_set.find(*iter) == _set.end()) {
+                return false;
+            }
+
+            ++iter;
+        }
+
+        return true;
+    }
+    case NONE:
+    case DOCID:
+        return true;
+    case BODY:
+    case HEADER:
+    case ALL:
+        return false;
+    }
+
+    return false;
+}
+
+void
+FieldCollection::insert(const Field& f)
+{
+    _set.insert(&f);
+}
+
+void
+FieldCollection::insert(const Field::Set& c)
+{
+    _set.insert(c.begin(), c.end());
+}
+
+void
+FieldSet::copyFields(Document& dest,
+                     const Document& src,
+                     const FieldSet& fields)
+{
+    if (fields.getType() == ALL) {
+        dest.getFields() = src.getFields();
+        return;
+    }
+    for (Document::const_iterator it(src.begin()), e(src.end());
+         it != e; ++it)
+    {
+        const Field& f(it.field());
+        if (!fields.contains(f)) {
+            continue;
+        }
+        dest.setValue(f, *src.getValue(f));
+    }
+}
+
+Document::UP
+FieldSet::createDocumentSubsetCopy(const Document& src,
+                                   const FieldSet& fields)
+{
+    Document::UP ret(new Document(src.getType(), src.getId()));
+    copyFields(*ret, src, fields);
+    return ret;
+}
+
+void
+FieldSet::stripFields(Document& doc,
+                      const FieldSet& fieldsToKeep)
+{
+    if (fieldsToKeep.getType() == ALL) {
+        return;
+    } else if (fieldsToKeep.getType() == DOCID
+               || fieldsToKeep.getType() == NONE)
+    {
+        doc.clear();
+        return;
+    }
+    std::vector<const Field*> fieldsToRemove;
+    for (Document::const_iterator it(doc.begin()), e(doc.end());
+         it != e; ++it)
+    {
+        const Field& f(it.field());
+        if (!fieldsToKeep.contains(f)) {
+            fieldsToRemove.push_back(&f);
+        }
+    }
+    for (size_t i = 0; i < fieldsToRemove.size(); ++i) {
+        doc.remove(*fieldsToRemove[i]);
+    }
+}
+
+} // namespace document
+
diff --git a/document/src/vespa/document/fieldset/fieldsets.h b/document/src/vespa/document/fieldset/fieldsets.h
new file mode 100644
index 00000000000..000268dad99
--- /dev/null
+++ b/document/src/vespa/document/fieldset/fieldsets.h
@@ -0,0 +1,154 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <map>
+#include <vespa/document/base/field.h>
+
+namespace document {
+
+class AllFields : public FieldSet
+{
+public:
+    virtual bool contains(const FieldSet&) const {
+        return true;
+    }
+
+    /**
+     * @return Returns the type of field set this is.
+     */
+    virtual Type getType() const {
+        return ALL;
+    }
+
+    virtual FieldSet* clone() const {
+        return new AllFields();
+    }
+};
+
+class NoFields : public FieldSet
+{
+public:
+    virtual bool contains(const FieldSet& f) const {
+        return f.getType() == NONE;
+    }
+
+    /**
+     * @return Returns the type of field set this is.
+     */
+    virtual Type getType() const {
+        return NONE;
+    }
+
+    virtual FieldSet* clone() const {
+        return new NoFields();
+    }
+};
+
+class DocIdOnly : public FieldSet
+{
+public:
+    virtual bool contains(const FieldSet& fields) const {
+        return fields.getType() == DOCID || fields.getType() == NONE;
+    }
+
+    /**
+     * @return Returns the type of field set this is.
+     */
+    virtual Type getType() const {
+        return DOCID;
+    }
+
+    virtual FieldSet* clone() const {
+        return new DocIdOnly();
+    }
+
+};
+
+class HeaderFields : public FieldSet
+{
+public:
+    virtual bool contains(const FieldSet& fields) const;
+
+    /**
+     * @return Returns the type of field set this is.
+     */
+    virtual Type getType() const {
+        return HEADER;
+    }
+
+    virtual FieldSet* clone() const {
+        return new HeaderFields();
+    }
+
+};
+
+class BodyFields : public FieldSet
+{
+public:
+    virtual bool contains(const FieldSet& fields) const;
+
+    /**
+     * @return Returns the type of field set this is.
+     */
+    virtual Type getType() const {
+        return BODY;
+    }
+
+    virtual FieldSet* clone() const {
+        return new BodyFields();
+    }
+};
+
+class FieldCollection : public FieldSet
+{
+public:
+    typedef std::unique_ptr<FieldCollection> UP;
+
+    FieldCollection(const DocumentType& docType)
+        : _docType(docType) {};
+
+    FieldCollection(const DocumentType& docType, const Field::Set& set);
+
+    virtual bool contains(const FieldSet& fields) const;
+
+    /**
+     * @return Returns the type of field set this is.
+     */
+    virtual Type getType() const {
+        return SET;
+    }
+
+    /**
+     * @return Returns the document type the collection is associated with.
+     */
+    const DocumentType& getDocumentType() const {
+        return _docType;
+    }
+
+    /**
+     * Inserts the given field into the collection.
+     */
+    void insert(const Field& f);
+
+    /**
+     * Inserts all the field in the given collection into this collection.
+     */
+    void insert(const Field::Set& f);
+
+    /**
+     * Returns all the fields contained in this collection.
+     */
+    const Field::Set& getFields() const { return _set; }
+
+    virtual FieldSet* clone() const {
+        return new FieldCollection(*this);
+    }
+
+private:
+    Field::Set _set;
+    const DocumentType& _docType;
+};
+
+
+}
+
diff --git a/document/src/vespa/document/fieldvalue/.gitignore b/document/src/vespa/document/fieldvalue/.gitignore
new file mode 100644
index 00000000000..dfa09296ddb
--- /dev/null
+++ b/document/src/vespa/document/fieldvalue/.gitignore
@@ -0,0 +1,4 @@
+*.So
+.*.swp
+.depend
+Makefile
diff --git a/document/src/vespa/document/fieldvalue/CMakeLists.txt b/document/src/vespa/document/fieldvalue/CMakeLists.txt
new file mode 100644
index 00000000000..759d4d42b40
--- /dev/null
+++ b/document/src/vespa/document/fieldvalue/CMakeLists.txt
@@ -0,0 +1,29 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_library(document_fieldvalues OBJECT
+    SOURCES
+    annotationreferencefieldvalue.cpp
+    arrayfieldvalue.cpp
+    bytefieldvalue.cpp
+    collectionfieldvalue.cpp
+    document.cpp
+    doublefieldvalue.cpp
+    fieldvalue.cpp
+    floatfieldvalue.cpp
+    intfieldvalue.cpp
+    literalfieldvalue.cpp
+    longfieldvalue.cpp
+    mapfieldvalue.cpp
+    numericfieldvalue.cpp
+    predicatefieldvalue.cpp
+    rawfieldvalue.cpp
+    serializablearray.cpp
+    shortfieldvalue.cpp
+    stringfieldvalue.cpp
+    structfieldvalue.cpp
+    structuredfieldvalue.cpp
+    tensorfieldvalue.cpp
+    weightedsetfieldvalue.cpp
+    DEPENDS
+    AFTER
+    document_documentconfig
+)
diff --git a/document/src/vespa/document/fieldvalue/annotationreferencefieldvalue.cpp b/document/src/vespa/document/fieldvalue/annotationreferencefieldvalue.cpp
new file mode 100644
index 00000000000..fb4b57b2cce
--- /dev/null
+++ b/document/src/vespa/document/fieldvalue/annotationreferencefieldvalue.cpp
@@ -0,0 +1,46 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".annotationreferencefieldvalue");
+
+#include "annotationreferencefieldvalue.h"
+
+using std::ostream;
+using std::string;
+
+namespace document {
+
+AnnotationReferenceFieldValue::AnnotationReferenceFieldValue(
+        const DataType &type, int32_t annotation_index)
+    : _type(&type),
+      _annotation_index(annotation_index) {
+}
+
+int AnnotationReferenceFieldValue::compare(const FieldValue &other) const {
+    if (*getDataType() == *other.getDataType()) {
+        const AnnotationReferenceFieldValue &val(
+                static_cast<const AnnotationReferenceFieldValue &>(other));
+        return _annotation_index - val._annotation_index;
+    }
+    return (getDataType()->getId() - other.getDataType()->getId());
+}
+
+void AnnotationReferenceFieldValue::print(ostream &out, bool,
+                                          const string &) const {
+    out << "AnnotationReferenceFieldValue(n)";
+}
+
+AnnotationReferenceFieldValue *AnnotationReferenceFieldValue::clone() const {
+    return new AnnotationReferenceFieldValue(*this);
+}
+
+void AnnotationReferenceFieldValue::printXml(XmlOutputStream &out) const {
+    out << _annotation_index;
+}
+
+bool AnnotationReferenceFieldValue::hasChanged() const {
+    return false;
+}
+
+}  // namespace document
diff --git a/document/src/vespa/document/fieldvalue/annotationreferencefieldvalue.h b/document/src/vespa/document/fieldvalue/annotationreferencefieldvalue.h
new file mode 100644
index 00000000000..e6c9e329792
--- /dev/null
+++ b/document/src/vespa/document/fieldvalue/annotationreferencefieldvalue.h
@@ -0,0 +1,38 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "fieldvalue.h"
+#include <vespa/document/datatype/datatype.h>
+
+namespace document {
+class Annotation;
+class AnnotationReferenceDataType;
+
+class AnnotationReferenceFieldValue : public FieldValue {
+    const DataType *_type;
+    int32_t _annotation_index;
+
+public:
+    AnnotationReferenceFieldValue(const DataType &type)
+        : _type(&type), _annotation_index(0) {}
+    AnnotationReferenceFieldValue(const DataType &type,
+                                  int32_t annotation_index);
+    void setAnnotationIndex(int32_t index) { _annotation_index = index; }
+
+    void accept(FieldValueVisitor &visitor) override { visitor.visit(*this); }
+    void accept(ConstFieldValueVisitor &visitor) const override { visitor.visit(*this); }
+
+    int32_t getAnnotationIndex() const { return _annotation_index; }
+
+    virtual int compare(const FieldValue& other) const;
+    virtual void print(std::ostream &out, bool verbose,
+                       const std::string &indent) const;
+    virtual AnnotationReferenceFieldValue *clone() const;
+    virtual const DataType *getDataType() const { return _type; }
+    virtual void printXml(XmlOutputStream &out) const;
+    virtual bool hasChanged() const;
+};
+
+}  // namespace document
+
diff --git a/document/src/vespa/document/fieldvalue/arrayfieldvalue.cpp b/document/src/vespa/document/fieldvalue/arrayfieldvalue.cpp
new file mode 100644
index 00000000000..fa872a31870
--- /dev/null
+++ b/document/src/vespa/document/fieldvalue/arrayfieldvalue.cpp
@@ -0,0 +1,307 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/document/fieldvalue/arrayfieldvalue.h>
+#include <vespa/document/fieldvalue/intfieldvalue.h>
+#include <vespa/document/fieldvalue/floatfieldvalue.h>
+#include <vespa/document/fieldvalue/stringfieldvalue.h>
+#include <vespa/document/fieldvalue/rawfieldvalue.h>
+#include <vespa/document/fieldvalue/longfieldvalue.h>
+#include <vespa/document/fieldvalue/doublefieldvalue.h>
+#include <vespa/document/fieldvalue/bytefieldvalue.h>
+#include <vespa/document/fieldvalue/predicatefieldvalue.h>
+
+#include <vespa/document/util/bytebuffer.h>
+#include <vespa/document/util/serializable.h>
+#include <vespa/log/log.h>
+
+LOG_SETUP(".document.fieldvalue.array");
+
+namespace document {
+
+using vespalib::IllegalArgumentException;
+
+IMPLEMENT_IDENTIFIABLE_ABSTRACT(ArrayFieldValue, CollectionFieldValue);
+
+ArrayFieldValue::ArrayFieldValue(const DataType &type)
+    : CollectionFieldValue(type),
+      _array()
+{
+    if (!type.inherits(ArrayDataType::classId)) {
+        throw IllegalArgumentException(
+                "Cannot generate an array value with non-array type "
+                + type.toString() + ".", VESPA_STRLOC);
+    }
+    _array = createArray(getNestedType());
+}
+
+ArrayFieldValue::ArrayFieldValue(const ArrayFieldValue& other)
+    : CollectionFieldValue(other),
+      _array(other._array->clone())
+{
+}
+
+ArrayFieldValue::~ArrayFieldValue()
+{
+}
+
+ArrayFieldValue&
+ArrayFieldValue::operator=(const ArrayFieldValue& other)
+{
+    if (this != &other) {
+        verifyType(other);
+        ArrayFieldValue copy(other);
+        swap(copy);
+    }
+    return *this;
+}
+
+void
+ArrayFieldValue::remove(uint32_t index)
+{
+    if (_array->size() <= index) {
+        throw IllegalArgumentException(vespalib::make_string(
+                "Cannot remove index %u from an array of size %lu.",
+                index, (unsigned long)_array->size()), VESPA_STRLOC);
+    }
+    _array->erase(array().begin() + index);
+}
+
+bool
+ArrayFieldValue::addValue(const FieldValue& value)
+{
+    if (getNestedType().isValueType(value)) {
+        _array->push_back(value);
+    } else {
+        throw IllegalArgumentException(vespalib::make_string(
+                "Cannot add value of type %s to array containing type %s.",
+                value.getDataType()->toString().c_str(),
+                getNestedType().toString().c_str()), VESPA_STRLOC);
+    }
+    return true;
+}
+
+bool
+ArrayFieldValue::containsValue(const FieldValue& value) const
+{
+    if (getNestedType().isValueType(value)) {
+        for (IArray::const_iterator it(array().begin()), mt(array().end()); it != mt; ++it) {
+            if (*it == value) {
+                return true;
+            }
+        }
+        return false;
+    } else {
+        throw IllegalArgumentException(vespalib::make_string(
+                "Value of type %s can't possibly be in array of type %s.",
+                value.getDataType()->toString().c_str(),
+                getDataType()->toString().c_str()), VESPA_STRLOC);
+    }
+}
+
+bool
+ArrayFieldValue::removeValue(const FieldValue& value)
+{
+    if (getNestedType().isValueType(value)) {
+        size_t oldSize = _array->size();
+        IArray::iterator it = array().begin();
+        while (it != array().end()) {
+            if (*it == value) {
+                it = _array->erase(it);
+            } else {
+                ++it;
+            }
+        }
+        return (oldSize != _array->size());
+    } else {
+        throw IllegalArgumentException(vespalib::make_string(
+                "Value of type %s can't possibly be in array of type %s.",
+                value.getDataType()->toString().c_str(),
+                getDataType()->toString().c_str()), VESPA_STRLOC);
+    }
+}
+
+FieldValue&
+ArrayFieldValue::assign(const FieldValue& value)
+{
+    if (*value.getDataType() == *getDataType()) {
+        const ArrayFieldValue& val(static_cast<const ArrayFieldValue&>(value));
+        operator=(val);
+        return *this;
+    } else {
+        return FieldValue::assign(value);
+    }
+}
+
+int
+ArrayFieldValue::compare(const FieldValue& o) const
+{
+    int diff = CollectionFieldValue::compare(o);
+    if (diff != 0) return diff;
+
+    const ArrayFieldValue& other(static_cast<const ArrayFieldValue&>(o));
+
+    if (size() != other.size()) return (size() - other.size());
+    for (uint32_t i=0, n=size(); i<n; ++i) {
+        diff = array()[i].compare(other.array()[i]);
+        if (diff != 0) return diff;
+    }
+    return 0;
+}
+
+void
+ArrayFieldValue::printXml(XmlOutputStream& xos) const
+{
+    for (uint32_t i=0, n=_array->size(); i<n; ++i) {
+        xos << XmlTag("item");
+        array()[i].printXml(xos);
+        xos << XmlEndTag();
+    }
+}
+
+void
+ArrayFieldValue::print(std::ostream& out, bool verbose,
+                       const std::string& indent) const
+{
+    out << "Array(size: " << _array->size();
+    try {
+        for (uint32_t i=0, n=_array->size(); i<n; ++i) {
+            out << ",\n" << indent << "  ";
+            array()[i].print(out, verbose, indent + "  ");
+        }
+    } catch (const DeserializeException & e) {
+        out << ",\n" << indent << "(Deserialization failed)";
+    }
+    out << "\n" << indent << ")";
+}
+
+bool
+ArrayFieldValue::hasChanged() const
+{
+    for (uint32_t i=0, n=_array->size(); i<n; ++i) {
+        if (array()[i].hasChanged()) return true;
+    }
+    return false;
+}
+
+FieldValue::IteratorHandler::ModificationStatus
+ArrayFieldValue::iterateSubset(int startPos,
+                               int endPos,
+                               const vespalib::stringref & variable,
+                               FieldPath::const_iterator nextPos,
+                               FieldPath::const_iterator end_,
+                               IteratorHandler& handler) const
+{
+    FieldValue::IteratorHandler::ModificationStatus
+        retVal = FieldValue::IteratorHandler::NOT_MODIFIED;
+
+    LOG(spam, "iterateSubset(start=%d, end=%d, variable='%s')",
+        startPos, endPos, variable.c_str());
+
+    std::vector<int> indicesToRemove;
+
+    for (int i = startPos; i <= endPos && i < static_cast<int>(_array->size()); ++i) {
+        if (!variable.empty()) {
+            handler.getVariables()[variable] = IteratorHandler::IndexValue(i);
+        }
+
+        FieldValue::IteratorHandler::ModificationStatus
+            status = array()[i].iterateNested(nextPos, end_, handler);
+
+        if (status == FieldValue::IteratorHandler::REMOVED) {
+            indicesToRemove.push_back(i);
+            retVal = FieldValue::IteratorHandler::MODIFIED;
+        } else if (status == FieldValue::IteratorHandler::MODIFIED) {
+            retVal = status;
+        }
+    }
+
+    if (!variable.empty()) {
+        handler.getVariables().erase(variable);
+    }
+
+    for (std::vector<int>::reverse_iterator i = indicesToRemove.rbegin();
+         i != indicesToRemove.rend(); ++i)
+    {
+        const_cast<ArrayFieldValue&>(*this).remove(*i);
+    }
+
+    return retVal;
+}
+
+FieldValue::IteratorHandler::ModificationStatus
+ArrayFieldValue::onIterateNested(FieldPath::const_iterator start, FieldPath::const_iterator end_, IteratorHandler & handler) const
+{
+    IteratorHandler::CollectionScope autoScope(handler, *this);
+    LOG(spam, "iterating over ArrayFieldValue %s", toString().c_str());
+
+    if (start != end_) {
+        switch (start->getType()) {
+        case FieldPathEntry::ARRAY_INDEX:
+            LOG(spam, "ARRAY_INDEX");
+            return iterateSubset(start->getIndex(), start->getIndex(),
+                                 "", start + 1, end_, handler);
+        case FieldPathEntry::VARIABLE:
+        {
+            LOG(spam, "VARIABLE");
+            IteratorHandler::VariableMap::iterator
+                iter = handler.getVariables().find(start->getVariableName());
+            if (iter != handler.getVariables().end()) {
+                int idx = iter->second.index;
+
+                if (idx == -1) {
+                    throw IllegalArgumentException(
+                            "Mismatch between variables - trying to iterate through map "
+                            "and array with the same variable.");
+                }
+
+                if (idx < (int)_array->size()) {
+                    return iterateSubset(idx, idx, "", start + 1, end_, handler);
+                }
+            } else {
+                return iterateSubset(0, static_cast<int>(_array->size()) - 1,
+                                     start->getVariableName(), start + 1, end_, handler);
+            }
+            break;
+        }
+        default:
+            break;
+        }
+        return iterateSubset(0, static_cast<int>(_array->size()) - 1, "",
+                             start, end_, handler);
+    } else {
+        IteratorHandler::ModificationStatus
+            status = handler.modify(const_cast<ArrayFieldValue&>(*this));
+
+        if (status == FieldValue::IteratorHandler::REMOVED) {
+            return status;
+        }
+
+        if (handler.handleComplex(*this)) {
+            if (iterateSubset(0, static_cast<int>(_array->size()) - 1, "",
+                              start, end_, handler) != FieldValue::IteratorHandler::NOT_MODIFIED)
+            {
+                status = FieldValue::IteratorHandler::MODIFIED;
+            }
+        }
+
+        return status;
+    }
+}
+
+using vespalib::ComplexArrayT;
+using vespalib::PrimitiveArrayT;
+
+namespace {
+class FieldValueFactory : public ComplexArrayT<FieldValue>::Factory
+{
+public:
+    FieldValueFactory(DataType::UP dataType) : _dataType(dataType.release()) { }
+    FieldValue * create() { return _dataType->createFieldValue().release(); }
+    virtual FieldValueFactory * clone() const { return new FieldValueFactory(*this); }
+private:
+    DataType::CP _dataType;
+};
+
+}
+
+} // document
diff --git a/document/src/vespa/document/fieldvalue/arrayfieldvalue.h b/document/src/vespa/document/fieldvalue/arrayfieldvalue.h
new file mode 100644
index 00000000000..203b053812f
--- /dev/null
+++ b/document/src/vespa/document/fieldvalue/arrayfieldvalue.h
@@ -0,0 +1,92 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * \class document::ArrayFieldValue
+ * \ingroup fieldvalue
+ *
+ * \brief A representation of an array of fieldvalues of specific types.
+ *
+ * A field value representing an array of other fieldvalues of a given type.
+ *
+ * \see datatype.h
+ * \see field.h
+ */
+#pragma once
+
+#include <vespa/document/datatype/arraydatatype.h>
+#include <vespa/document/fieldvalue/collectionfieldvalue.h>
+
+namespace document {
+
+class ArrayFieldValue : public CollectionFieldValue {
+private:
+    IArray::UP _array;
+
+    virtual bool addValue(const FieldValue&);
+    virtual bool containsValue(const FieldValue& val) const;
+    virtual bool removeValue(const FieldValue& val);
+    IteratorHandler::ModificationStatus iterateSubset(
+            int startPos, int endPos, const vespalib::stringref & variable,
+            FieldPath::const_iterator nextPos,
+            FieldPath::const_iterator end_,
+            IteratorHandler& handler) const;
+    virtual IteratorHandler::ModificationStatus onIterateNested(
+            FieldPath::const_iterator start, FieldPath::const_iterator end,
+            IteratorHandler & handler) const;
+public:
+    typedef IArray::const_iterator const_iterator;
+    typedef IArray::iterator iterator;
+    typedef std::unique_ptr<ArrayFieldValue> UP;
+
+    /**
+     * @param arrayType Type of the array. Must be an ArrayDataType, but does
+     *                  not enforce type compile time so it will be easier to
+     *                  create instances using field's getDataType().
+     */
+    ArrayFieldValue(const DataType &arrayType);
+    ArrayFieldValue(const ArrayFieldValue&);
+    virtual ~ArrayFieldValue();
+
+    ArrayFieldValue& operator=(const ArrayFieldValue&);
+
+    const FieldValue& operator[](uint32_t index) const { return array()[index]; }
+    FieldValue& operator[](uint32_t index) { return array()[index]; }
+
+    void accept(FieldValueVisitor &visitor) override { visitor.visit(*this); }
+    void accept(ConstFieldValueVisitor &visitor) const override { visitor.visit(*this); }
+
+    void append(FieldValue::UP value) { _array->push_back(*value); }
+    void remove(uint32_t index);
+    bool remove(const FieldValue& val) { return removeValue(val); }
+
+        // CollectionFieldValue implementation
+    virtual bool isEmpty() const { return _array->empty(); }
+    virtual size_t size() const { return _array->size(); }
+    virtual void clear() { _array->clear(); }
+    void reserve(size_t sz) { _array->reserve(sz); }
+    void resize(size_t sz) { _array->resize(sz); }
+
+        // FieldValue implementation
+    virtual FieldValue& assign(const FieldValue&);
+    virtual ArrayFieldValue* clone() const
+        { return new ArrayFieldValue(*this); }
+    virtual int compare(const FieldValue&) const;
+    virtual void printXml(XmlOutputStream& out) const;
+    virtual void print(std::ostream& out, bool verbose,
+                       const std::string& indent) const;
+    virtual bool hasChanged() const;
+    void swap(ArrayFieldValue & other) { _array.swap(other._array); }
+
+        // Iterator functionality
+    const_iterator begin() const { return array().begin(); }
+    const_iterator end() const { return array().end(); }
+    iterator begin() { return array().begin(); }
+    iterator end() { return array().end(); }
+
+    DECLARE_IDENTIFIABLE_ABSTRACT(ArrayFieldValue);
+private:
+    const IArray & array() const { return *_array; }
+    IArray & array() { return *_array; }
+};
+
+} // document
+
diff --git a/document/src/vespa/document/fieldvalue/bytefieldvalue.cpp b/document/src/vespa/document/fieldvalue/bytefieldvalue.cpp
new file mode 100644
index 00000000000..c0d6fee9f3b
--- /dev/null
+++ b/document/src/vespa/document/fieldvalue/bytefieldvalue.cpp
@@ -0,0 +1,10 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/document/fieldvalue/bytefieldvalue.h>
+
+namespace document {
+
+IMPLEMENT_IDENTIFIABLE(ByteFieldValue, NumericFieldValueBase);
+
+} // document
diff --git a/document/src/vespa/document/fieldvalue/bytefieldvalue.h b/document/src/vespa/document/fieldvalue/bytefieldvalue.h
new file mode 100644
index 00000000000..84102ab1350
--- /dev/null
+++ b/document/src/vespa/document/fieldvalue/bytefieldvalue.h
@@ -0,0 +1,37 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * \class document::ByteFieldValue
+ * \ingroup fieldvalue
+ *
+ * \brief Wrapper for field values of datatype BYTE.
+ */
+#pragma once
+
+#include <vespa/document/datatype/numericdatatype.h>
+#include <vespa/document/fieldvalue/numericfieldvalue.h>
+
+namespace document {
+
+class ByteFieldValue : public NumericFieldValue<int8_t> {
+public:
+    typedef std::unique_ptr<ByteFieldValue> UP;
+    typedef int8_t Number;
+
+    ByteFieldValue(Number value = 0)
+        : NumericFieldValue<Number>(value) {}
+
+    void accept(FieldValueVisitor &visitor) override { visitor.visit(*this); }
+    void accept(ConstFieldValueVisitor &visitor) const override { visitor.visit(*this); }
+
+    virtual const DataType *getDataType() const { return DataType::BYTE; }
+
+    virtual ByteFieldValue* clone() const { return new ByteFieldValue(*this); }
+
+    using NumericFieldValue<Number>::operator=;
+
+    DECLARE_IDENTIFIABLE(ByteFieldValue);
+
+};
+
+} // document
+
diff --git a/document/src/vespa/document/fieldvalue/collectionfieldvalue.cpp b/document/src/vespa/document/fieldvalue/collectionfieldvalue.cpp
new file mode 100644
index 00000000000..8d00540856b
--- /dev/null
+++ b/document/src/vespa/document/fieldvalue/collectionfieldvalue.cpp
@@ -0,0 +1,26 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/document/fieldvalue/collectionfieldvalue.h>
+
+namespace document {
+
+IMPLEMENT_IDENTIFIABLE_ABSTRACT(CollectionFieldValue, FieldValue);
+
+CollectionFieldValue::CollectionFieldValue(const CollectionFieldValue& other)
+    : FieldValue(other),
+      _type(other._type)
+{
+}
+
+void CollectionFieldValue::verifyType(const CollectionFieldValue& other) const
+{
+    if (*_type != *other._type) {
+        throw vespalib::IllegalArgumentException(
+                "Cannot assign value of type " + other.getDataType()->toString()
+                + " to value of type " + getDataType()->toString() + ".",
+                VESPA_STRLOC);
+    }
+}
+
+}
diff --git a/document/src/vespa/document/fieldvalue/collectionfieldvalue.h b/document/src/vespa/document/fieldvalue/collectionfieldvalue.h
new file mode 100644
index 00000000000..3524b67afab
--- /dev/null
+++ b/document/src/vespa/document/fieldvalue/collectionfieldvalue.h
@@ -0,0 +1,104 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * \class document::CollectionFieldValue
+ * \ingroup fieldvalue
+ *
+ * \brief A field value containing a collection of other equally typed values.
+ *
+ * This class is the superclass of array and weighted set field values. It
+ * contains some common functionality.
+ */
+#pragma once
+
+#include <vespa/document/datatype/collectiondatatype.h>
+#include <vespa/document/fieldvalue/fieldvalue.h>
+
+namespace document {
+
+class CollectionFieldValue : public FieldValue {
+protected:
+    const DataType *_type;
+
+        // As overloading doesn't work with polymorphy, have protected functions
+        // doing the functionality, such that we can make utility functions here
+    virtual bool addValue(const FieldValue&) = 0;
+    virtual bool containsValue(const FieldValue&) const = 0;
+    virtual bool removeValue(const FieldValue&) = 0;
+    void verifyType(const CollectionFieldValue& other) const;
+
+public:
+    CollectionFieldValue(const DataType &type)
+        : FieldValue(),
+          _type(&type)
+    {}
+
+    CollectionFieldValue(const CollectionFieldValue& other);
+    CollectionFieldValue& operator=(const CollectionFieldValue& other) {
+        verifyType(other);
+        return *this;
+    }
+
+    virtual const DataType *getDataType() const { return _type; }
+
+    FieldValue::UP createNested() const {
+        return getNestedType().createFieldValue();
+    }
+
+    const DataType &getNestedType() const {
+        return static_cast<const CollectionDataType&>(*_type).getNestedType();
+    }
+
+    /**
+     * @return True if element was added. False if element was overwritten.
+     * @throws InvalidDataTypeException If fieldvalue of wrong type is attempted
+     *                                  added.
+     */
+    bool add(const FieldValue& val) { return addValue(val); }
+    bool contains(const FieldValue& val) const { return containsValue(val); }
+    /** @return True if element was found and removed. False if not found. */
+    bool remove(const FieldValue& val) { return removeValue(val); }
+
+    virtual bool isEmpty() const = 0;
+    virtual size_t size() const = 0;
+    virtual void clear() = 0;
+
+    // Convenience functions for using primitives directly
+
+    bool add(const vespalib::stringref & val)
+        { return addValue(*createNested() = val); }
+    bool add(int32_t val)
+        { return addValue(*createNested() = val); }
+    bool add(int64_t val)
+        { return addValue(*createNested() = val); }
+    bool add(float val)
+        { return addValue(*createNested() = val); }
+    bool add(double val)
+        { return addValue(*createNested() = val); }
+
+    bool contains(const vespalib::stringref & val)
+        { return containsValue(*createNested() = val); }
+    bool contains(int32_t val)
+        { return containsValue(*createNested() = val); }
+    bool contains(int64_t val)
+        { return containsValue(*createNested() = val); }
+    bool contains(float val)
+        { return containsValue(*createNested() = val); }
+    bool contains(double val)
+        { return containsValue(*createNested() = val); }
+
+    bool remove(const vespalib::stringref & val)
+        { return removeValue(*createNested() = val); }
+    bool remove(int32_t val)
+        { return removeValue(*createNested() = val); }
+    bool remove(int64_t val)
+        { return removeValue(*createNested() = val); }
+    bool remove(float val)
+        { return removeValue(*createNested() = val); }
+    bool remove(double val)
+        { return removeValue(*createNested() = val); }
+
+    DECLARE_IDENTIFIABLE_ABSTRACT(CollectionFieldValue);
+};
+
+}
+
diff --git a/document/src/vespa/document/fieldvalue/document.cpp b/document/src/vespa/document/fieldvalue/document.cpp
new file mode 100644
index 00000000000..79278a2e1ba
--- /dev/null
+++ b/document/src/vespa/document/fieldvalue/document.cpp
@@ -0,0 +1,482 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/document/fieldvalue/document.h>
+
+#include <memory>
+#include <boost/assign.hpp>
+#include <vespa/vespalib/util/crc.h>
+#include <vespa/document/base/documentid.h>
+#include <vespa/document/base/field.h>
+#include <vespa/document/fieldvalue/fieldvalue.h>
+#include <fstream>
+
+#include <vespa/document/repo/fixedtyperepo.h>
+#include <vespa/document/serialization/vespadocumentdeserializer.h>
+#include <vespa/document/serialization/vespadocumentserializer.h>
+#include <vespa/log/log.h>
+#include <vespa/vespalib/objects/nbostream.h>
+
+using vespalib::nbostream;
+
+LOG_SETUP(".document.fieldvalue.document");
+
+namespace document {
+namespace {
+std::set<uint16_t> getAllowedVersions() {
+    std::set<uint16_t> allowed;
+    using namespace boost::assign;
+    allowed += 6, 7, 8;
+    return allowed;
+}
+
+const std::set<uint16_t> ALLOWED_VERSIONS(getAllowedVersions());
+
+void documentTypeError(const vespalib::stringref & name)
+    __attribute__((noinline));
+void documentTypeError(const vespalib::stringref & name) {
+    throw vespalib::IllegalArgumentException(
+            vespalib::make_string(
+                    "Cannot generate a document with non-document type %s.",
+                    name.c_str()), VESPA_STRLOC);
+}
+
+const DataType &verifyDocumentType(const DataType *type) {
+    if (!type) {
+        documentTypeError("null");
+    } else if ( ! type->getClass().inherits(DocumentType::classId)) {
+        documentTypeError(type->toString());
+    }
+    return *type;
+}
+}  // namespace
+
+IMPLEMENT_IDENTIFIABLE_ABSTRACT(Document, StructuredFieldValue);
+
+Document::Document()
+    : StructuredFieldValue(*DataType::DOCUMENT),
+      _id(),
+      _fields(getType().getFieldsType()),
+      _lastModified(0)
+{
+    _fields.setDocumentType(getType());
+}
+
+Document::Document(const Document& other)
+    : StructuredFieldValue(other),
+      _id(other._id),
+      _fields(other._fields),
+      _lastModified(other._lastModified)
+{
+}
+
+Document::Document(const DataType &type, const DocumentId& documentId)
+    : StructuredFieldValue(verifyDocumentType(&type)),
+      _id(documentId),
+      _fields(getType().getFieldsType()),
+      _lastModified(0)
+{
+    _fields.setDocumentType(getType());
+    if (documentId.hasDocType() && documentId.getDocType() != type.getName()) {
+        throw vespalib::IllegalArgumentException(
+                vespalib::make_string(
+                        "Trying to create a document with type %s that "
+                        "don't match the id (type %s).",
+                        type.getName().c_str(),
+                        documentId.getDocType().c_str()));
+    }
+}
+
+Document::Document(const DataType &type, DocumentId& documentId, bool iWillAllowSwap)
+    : StructuredFieldValue(verifyDocumentType(&type)),
+      _id(),
+      _fields(getType().getFieldsType()),
+      _lastModified(0)
+{
+    (void) iWillAllowSwap;
+    _fields.setDocumentType(getType());
+    if (documentId.hasDocType() && documentId.getDocType() != type.getName()) {
+        throw vespalib::IllegalArgumentException(
+                vespalib::make_string(
+                        "Trying to create a document with type %s that "
+                        "don't match the id (type %s).",
+                        type.getName().c_str(),
+                        documentId.getDocType().c_str()));
+    }
+    _id.swap(documentId);
+}
+
+Document::Document(const DocumentTypeRepo& repo,
+                   ByteBuffer& buffer,
+                   const DataType *anticipatedType)
+    : StructuredFieldValue(anticipatedType ?
+                           verifyDocumentType(anticipatedType) :
+                           *DataType::DOCUMENT),
+      _id(),
+      _fields(static_cast<const DocumentType &>(getType()).getFieldsType()),
+      _lastModified(0)
+{
+    deserialize(repo, buffer);
+}
+
+void Document::setRepo(const DocumentTypeRepo& repo)
+{
+    _fields.setRepo(repo);
+}
+
+Document::Document(const DocumentTypeRepo& repo,
+                   vespalib::nbostream & is,
+                   const DataType *anticipatedType)
+    : StructuredFieldValue(anticipatedType ?
+                           verifyDocumentType(anticipatedType) :
+                           *DataType::DOCUMENT),
+      _id(),
+      _fields(static_cast<const DocumentType &>(getType()).getFieldsType()),
+      _lastModified(0)
+{
+    deserialize(repo, is);
+}
+
+Document::Document(const DocumentTypeRepo& repo,
+                   ByteBuffer& buffer,
+                   bool includeContent,
+                   const DataType *anticipatedType)
+    : StructuredFieldValue(anticipatedType ?
+                           verifyDocumentType(anticipatedType) :
+                           *DataType::DOCUMENT),
+      _id(),
+      _fields(static_cast<const DocumentType &>(getType()).getFieldsType()),
+      _lastModified(0)
+{
+    if (!includeContent) {
+        const DocumentType *newDocType = deserializeDocHeaderAndType(
+                repo, buffer, _id,
+                static_cast<const DocumentType*>(anticipatedType));
+        if (newDocType) {
+            setType(*newDocType);
+        }
+    } else {
+        deserialize(repo, buffer);
+    }
+}
+
+
+Document::Document(const DocumentTypeRepo& repo,
+                   ByteBuffer& header,
+                   ByteBuffer& body,
+                   const DataType *anticipatedType)
+    : StructuredFieldValue(anticipatedType ?
+                           verifyDocumentType(anticipatedType) :
+                           *DataType::DOCUMENT),
+      _id(),
+      _fields(static_cast<const DocumentType &>(getType()).getFieldsType()),
+      _lastModified(0)
+{
+    deserializeHeader(repo, header);
+    deserializeBody(repo, body);
+}
+
+void
+Document::swap(Document & rhs)
+{
+    StructuredFieldValue::swap(rhs);
+    _fields.swap(rhs._fields);
+    _id.swap(rhs._id);
+    std::swap(_lastModified, rhs._lastModified);
+}
+
+Document& Document::operator=(const Document& doc)
+{
+    StructuredFieldValue::operator=(doc);
+    _id = doc._id;
+    _fields = doc._fields;
+    _lastModified = doc._lastModified;
+    return *this;
+}
+
+void
+Document::clear()
+{
+    _fields.clear();
+}
+
+void
+Document::setFieldValue(const Field& field, FieldValue::UP data)
+{
+    _fields.setFieldValue(field, std::move(data));
+}
+
+bool
+Document::hasChanged() const
+{
+    return _fields.hasChanged();
+}
+
+DocumentId
+Document::getIdFromSerialized(ByteBuffer& buf)
+{
+    int position = buf.getPos();
+    DocumentId retVal;
+
+    deserializeDocHeader(buf, retVal);
+    buf.setPos(position);
+
+    return retVal;
+}
+
+const DocumentType *
+Document::getDocTypeFromSerialized(const DocumentTypeRepo& repo,
+                                   ByteBuffer& buf)
+{
+    int position = buf.getPos();
+    DocumentId retVal;
+
+    const DocumentType *docType(deserializeDocHeaderAndType(
+                    repo, buf, retVal, NULL));
+    buf.setPos(position);
+
+    return docType;
+}
+
+FieldValue&
+Document::assign(const FieldValue& value)
+{
+    /// \todo TODO (was warning):  This type checking doesnt work with the way assign is used.
+//    if (*value.getDataType() == *_type) {
+    const Document& other(dynamic_cast<const Document&>(value));
+    return operator=(other);
+//    }
+//    return FieldValue::assign(value); // Generates exception
+}
+
+int
+Document::compare(const FieldValue& other) const
+{
+    int diff = StructuredFieldValue::compare(other);
+    if (diff != 0) {
+        return diff;
+    }
+    const Document& doc(static_cast<const Document&>(other));
+    vespalib::string id1 = _id.toString();
+    vespalib::string id2 = doc._id.toString();
+    if (id1 != id2) {
+        return (id1 < id2 ? -1 : 1);
+    }
+    return _fields.compare(doc._fields);
+}
+
+void
+Document::print(std::ostream& out, bool verbose,
+                const std::string& indent) const
+{
+    if (!verbose) {
+        out << "Document(" << getId() << ", " << getType() << ")";
+    } else {
+        out << "Document(" << getId() << "\n" << indent << "  ";
+        getType().print(out, true, indent + "  ");
+        for (const_iterator it = begin(); it != end(); ++it) {
+            out << "\n" << indent << "  " << it.field().getName() << ": ";
+            getValue(it.field())->print(out, true, indent + "  ");
+        }
+        out << "\n" << indent << ")";
+    }
+}
+
+void
+Document::printXml(XmlOutputStream& xos) const
+{
+    xos << XmlTag("document")
+        << XmlAttribute("documenttype", getType().getName())
+        << XmlAttribute("documentid", getId().toString());
+    if (_lastModified != 0) {
+        xos << XmlAttribute("lastmodifiedtime", _lastModified);
+    }
+    _fields.printXml(xos);
+    xos << XmlEndTag();
+}
+
+std::string
+Document::toXml(const std::string& indent) const
+{
+    std::ostringstream ost;
+    XmlOutputStream xos(ost, indent);
+    printXml(xos);
+    return ost.str();
+}
+
+uint32_t
+Document::calculateChecksum() const
+{
+    vespalib::string docId(_id.toString());
+    const vespalib::string & typeName(getType().getName());
+    uint16_t typeVersion(0);  // Hardcode version 0 (version not supported)
+
+    vespalib::crc_32_type calculator;
+    calculator.process_bytes(docId.c_str(), docId.size());
+    calculator.process_bytes(typeName.c_str(), typeName.size());
+    calculator.process_bytes(&typeVersion, sizeof(typeVersion));
+    return calculator.checksum() ^ _fields.calculateChecksum();
+}
+
+const DocumentType *
+Document::deserializeDocHeaderAndType(
+        const DocumentTypeRepo& repo, ByteBuffer& buffer, DocumentId& id,
+        const DocumentType * docType)
+{
+    deserializeDocHeader(buffer, id);
+
+    vespalib::stringref docTypeName(buffer.getBufferAtPos());
+    buffer.incPos(docTypeName.size() + 1); // Skip 0-byte too
+    {
+        int16_t docTypeVersion;  // version not supported anymore
+        buffer.getShortNetwork(docTypeVersion);
+    }
+    const DocumentType *docTypeNew = 0;
+
+    if (! ((docType != NULL) && (docType->getName() == docTypeName))) {
+        docTypeNew = repo.getDocumentType(docTypeName);
+        if (!docTypeNew) {
+            throw DocumentTypeNotFoundException(docTypeName, VESPA_STRLOC);
+        }
+    }
+    return docTypeNew;
+}
+
+namespace {
+void versionError(uint16_t version) __attribute__((noinline));
+void mainDocumentError(int64_t len) __attribute__((noinline));
+void notEnoughDocumentError(int32_t len, int64_t remaining) __attribute__((noinline));
+
+void versionError(uint16_t version) {
+    throw DeserializeException(vespalib::make_string(
+            "Unrecognized serialization version %d", version),
+            VESPA_STRLOC);
+}
+
+void mainDocumentError(int64_t len) {
+    throw DeserializeException(vespalib::make_string(
+            "Document lengths past %i is not supported. Corrupt data "
+            "said length is %" PRId64 " bytes",
+            std::numeric_limits<int>::max(), len), VESPA_STRLOC);
+}
+
+void notEnoughDocumentError(int32_t len, int64_t remaining) {
+    throw DeserializeException(vespalib::make_string(
+                "Buffer said document length is %i bytes, but only %li "
+                "bytes remain in buffer",
+                len, remaining));
+}
+
+}
+
+void
+Document::deserializeDocHeader(ByteBuffer& buffer, DocumentId& id) {
+    int16_t version;
+    int32_t len;
+    buffer.getShortNetwork(version);
+
+    if (ALLOWED_VERSIONS.find(version) == ALLOWED_VERSIONS.end()) {
+        versionError(version);
+    } else if (version < 7) {
+        int64_t tmpLen = 0;
+        buffer.getInt2_4_8Bytes(tmpLen);
+        if (tmpLen > std::numeric_limits<int>::max()) {
+            mainDocumentError(tmpLen);
+        } else {
+            len = static_cast<int32_t>(tmpLen)
+                - ByteBuffer::getSerializedSize2_4_8Bytes(tmpLen)
+                - sizeof(uint16_t);
+        }
+    } else {
+        buffer.getIntNetwork(len);
+    }
+
+    if (len > (long)buffer.getRemaining()) {
+        notEnoughDocumentError(len, buffer.getRemaining());
+    } else {
+        nbostream stream(buffer.getBufferAtPos(), buffer.getRemaining(),
+                         false);
+        id = DocumentId(stream);
+        buffer.incPos(stream.rp());
+        unsigned char contentByte;
+        buffer.getByte(contentByte);
+    }
+}
+
+void Document::serializeHeader(ByteBuffer& buffer) const {
+    nbostream stream;
+    serializeHeader(stream);
+    buffer.putBytes(stream.peek(), stream.size());
+}
+
+void Document::serializeHeader(nbostream& stream) const {
+    VespaDocumentSerializer serializer(stream);
+    serializer.write(*this, WITHOUT_BODY);
+}
+
+void Document::serializeBody(ByteBuffer& buffer) const {
+    nbostream stream;
+    serializeBody(stream);
+    buffer.putBytes(stream.peek(), stream.size());
+}
+
+void Document::serializeBody(nbostream& stream) const {
+    VespaDocumentSerializer serializer(stream);
+    serializer.write(_fields, BodyFields());
+}
+
+void Document::deserialize(const DocumentTypeRepo& repo,
+                           vespalib::nbostream & os) {
+    VespaDocumentDeserializer deserializer(repo, os, 0);
+    try {
+        deserializer.read(*this);
+    } catch (const vespalib::IllegalStateException &e) {
+        throw DeserializeException(
+                std::string("Buffer out of bounds: ") + e.what());
+    }
+}
+
+void Document::deserialize(const DocumentTypeRepo& repo, ByteBuffer& data,
+                           bool longLivedBuffer) {
+    nbostream stream(data.getBufferAtPos(), data.getRemaining(), longLivedBuffer);
+    deserialize(repo, stream);
+    data.incPos(data.getRemaining() - stream.size());
+}
+
+void Document::deserialize(const DocumentTypeRepo& repo, ByteBuffer& header,
+                           ByteBuffer& body, bool longLivedBuffer) {
+    deserializeHeader(repo, header, longLivedBuffer);
+    deserializeBody(repo, body, longLivedBuffer);
+}
+
+void Document::deserializeHeader(const DocumentTypeRepo& repo,
+                           ByteBuffer& header, bool longLivedBuffer) {
+    nbostream stream(header.getBufferAtPos(), header.getRemaining(), longLivedBuffer);
+    VespaDocumentDeserializer deserializer(repo, stream, 0);
+    deserializer.read(*this);
+    header.incPos(header.getRemaining() - stream.size());
+}
+
+void Document::deserializeBody(const DocumentTypeRepo& repo,
+                           ByteBuffer& body, bool longLivedBuffer) {
+    nbostream body_stream(body.getBufferAtPos(), body.getRemaining(), longLivedBuffer);
+    VespaDocumentDeserializer
+        body_deserializer(repo, body_stream, getFields().getVersion());
+    body_deserializer.readStructNoReset(getFields());
+    body.incPos(body.getRemaining() - body_stream.size());
+}
+
+size_t
+Document::getSerializedSize() const
+{
+    // Temporary non-optimal (but guaranteed correct) implementation.
+    return serialize()->getLength();
+}
+
+StructuredFieldValue::StructuredIterator::UP
+Document::getIterator(const Field* first) const
+{
+    return _fields.getIterator(first);
+}
+
+} // document
diff --git a/document/src/vespa/document/fieldvalue/document.h b/document/src/vespa/document/fieldvalue/document.h
new file mode 100644
index 00000000000..893d1b3fc9c
--- /dev/null
+++ b/document/src/vespa/document/fieldvalue/document.h
@@ -0,0 +1,169 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * \class document::Document
+ * \ingroup fieldvalue
+ *
+ * \brief A class for storing structured data.
+ *
+ * The document is typically the field value handled by clients. It is documents
+ * that are sent around within Vespa. The other fieldvalues are used within
+ * documents, and documents are field values themselves, as we can store
+ * documents within documents, and also to ensure the user interface for
+ * documents is equal to that of other structured types.
+ *
+ * @see documentmanager.h
+ */
+#pragma once
+
+#include <vespa/document/base/documentid.h>
+#include <vespa/document/base/field.h>
+#include <vespa/document/datatype/documenttype.h>
+#include <vespa/document/fieldvalue/structfieldvalue.h>
+
+namespace document {
+
+class Document : public StructuredFieldValue
+{
+private:
+    DocumentId _id;
+    StructFieldValue _fields;
+
+        // To avoid having to return another container object out of docblocks
+        // the meta data has been added to document. This will not be serialized
+        // with the document and really doesn't belong here!
+    int64_t _lastModified;
+public:
+    typedef std::unique_ptr<Document> UP;
+    typedef std::shared_ptr<Document> SP;
+
+    static uint16_t getNewestSerializationVersion() { return 8; };
+
+    Document();
+    Document(const Document&);
+    Document(const DataType &, const DocumentId&);
+    Document(const DataType &, DocumentId &, bool iWillAllowSwap);
+    Document(const DocumentTypeRepo& repo,
+             ByteBuffer& buffer,
+             const DataType *anticipatedType = 0);
+    Document(const DocumentTypeRepo& repo,
+             vespalib::nbostream& stream,
+             const DataType *anticipatedType = 0);
+    /**
+       Constructor to deserialize only document and type from a buffer. Only relevant if includeContent is false.
+    */
+    Document(const DocumentTypeRepo& repo,
+             ByteBuffer& buffer,
+             bool includeContent,
+             const DataType *anticipatedType);
+    Document(const DocumentTypeRepo& repo,
+             ByteBuffer& header,
+             ByteBuffer& body,
+             const DataType *anticipatedType = 0);
+
+    void setRepo(const DocumentTypeRepo & repo);
+    const DocumentTypeRepo * getRepo() const { return _fields.getRepo(); }
+
+    Document& operator=(const Document&);
+
+    void swap(Document & rhs);
+
+    void accept(FieldValueVisitor &visitor) override { visitor.visit(*this); }
+    void accept(ConstFieldValueVisitor &visitor) const override { visitor.visit(*this); }
+
+    const DocumentType& getType() const {
+        return static_cast<const DocumentType &>(StructuredFieldValue::getType());
+    }
+
+    const DocumentId& getId() const { return _id; }
+    DocumentId & getId() { return _id; }
+
+    /**
+     * Get the last modified timestamp of a document if this is a document
+     * you have retrieved from a docblock.
+     */
+    int64_t getLastModified() const { return _lastModified; }
+
+    /**
+     * Set the last modified timestamp that will be retrieved with
+     * getLastModifiedTime().
+     */
+    void setLastModified(const int64_t lastModified) { _lastModified = lastModified; }
+
+    const StructFieldValue& getFields() const { return _fields; }
+    StructFieldValue& getFields() { return _fields; }
+
+    const Field& getField(const vespalib::stringref & name) const { return _fields.getField(name); }
+    bool hasField(const vespalib::stringref & name) const { return _fields.hasField(name); }
+
+    void clear() override;
+
+    bool hasChanged() const override;
+
+    /**
+     * Returns a pointer to the Id of a serialized document, without performing
+     * the deserialization. buffer must point to the start position of the
+     * serialization.  If the buffer doesn't have enough data remaining to have
+     * a legal Id in it, method returns NULL.
+     */
+    static DocumentId getIdFromSerialized(ByteBuffer&);
+
+    /**
+     * Returns a pointer to the document type of a serialized header, without
+     * performing the deserialization. Buffer must point to the start position
+     * of the serialization.
+     */
+    static const DocumentType *getDocTypeFromSerialized(const DocumentTypeRepo&, ByteBuffer&);
+
+    // FieldValue implementation.
+    FieldValue& assign(const FieldValue&) override;
+    int compare(const FieldValue& other) const override;
+    Document* clone() const override { return new Document(*this); }
+    void printXml(XmlOutputStream& out) const override;
+    void print(std::ostream& out, bool verbose, const std::string& indent) const override;
+
+    // Specialized serialization functions
+    void serializeHeader(ByteBuffer& buffer) const;
+    void serializeHeader(vespalib::nbostream& stream) const;
+
+    void serializeBody(ByteBuffer& buffer) const;
+    void serializeBody(vespalib::nbostream& stream) const;
+
+    /** Deserialize document contained in given bytebuffer. */
+    void deserialize(const DocumentTypeRepo& repo, ByteBuffer& data, bool longLivedBuffer=false);
+    void deserialize(const DocumentTypeRepo& repo, vespalib::nbostream & os);
+    /** Deserialize document contained in given bytebuffers. */
+    void deserialize(const DocumentTypeRepo& repo, ByteBuffer& body, ByteBuffer& header, bool longLivedBuffer=false);
+    void deserializeHeader(const DocumentTypeRepo& repo, ByteBuffer& header, bool longLivedBuffer=false);
+    void deserializeBody(const DocumentTypeRepo& repo, ByteBuffer& body, bool longLivedBuffer=false);
+
+    size_t getSerializedSize() const;
+
+    /** Undo fieldvalue's toXml override for document. */
+    std::string toXml(const std::string& indent = "") const override;
+
+    bool empty() const { return _fields.empty(); }
+
+    uint32_t calculateChecksum() const;
+
+    DECLARE_IDENTIFIABLE_ABSTRACT(Document);
+
+    void setFieldValue(const Field& field, FieldValue::UP data) override;
+private:
+    bool hasFieldValue(const Field& field) const override { return _fields.hasValue(field); }
+    void removeFieldValue(const Field& field) override { _fields.remove(field); }
+    FieldValue::UP getFieldValue(const Field& field) const override { return _fields.getValue(field); }
+    bool getFieldValue(const Field& field, FieldValue& value) const override { return _fields.getValue(field, value); }
+
+    // Iterator implementation
+    class FieldIterator;
+    friend class FieldIterator;
+
+    StructuredIterator::UP getIterator(const Field* first) const override;
+
+    static void deserializeDocHeader(ByteBuffer& buffer, DocumentId& id);
+    static const DocumentType *deserializeDocHeaderAndType(
+            const DocumentTypeRepo& repo, ByteBuffer& buffer,
+            DocumentId& id, const DocumentType * docType);
+};
+}  // document
+
diff --git a/document/src/vespa/document/fieldvalue/doublefieldvalue.cpp b/document/src/vespa/document/fieldvalue/doublefieldvalue.cpp
new file mode 100644
index 00000000000..cc5adb763cf
--- /dev/null
+++ b/document/src/vespa/document/fieldvalue/doublefieldvalue.cpp
@@ -0,0 +1,10 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/document/fieldvalue/doublefieldvalue.h>
+
+namespace document {
+
+IMPLEMENT_IDENTIFIABLE(DoubleFieldValue, NumericFieldValueBase);
+
+} // document
diff --git a/document/src/vespa/document/fieldvalue/doublefieldvalue.h b/document/src/vespa/document/fieldvalue/doublefieldvalue.h
new file mode 100644
index 00000000000..d76144858ef
--- /dev/null
+++ b/document/src/vespa/document/fieldvalue/doublefieldvalue.h
@@ -0,0 +1,35 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * \class document::DoubleFieldValue
+ * \ingroup fieldvalue
+ *
+ * \brief Wrapper for field values of datatype DOUBLE.
+ */
+#pragma once
+
+#include <vespa/document/datatype/numericdatatype.h>
+#include <vespa/document/fieldvalue/numericfieldvalue.h>
+
+namespace document {
+
+class DoubleFieldValue : public NumericFieldValue<double> {
+public:
+    typedef std::unique_ptr<DoubleFieldValue> UP;
+    typedef double Number;
+
+    DoubleFieldValue(Number value = 0) : NumericFieldValue<Number>(value) {}
+
+    void accept(FieldValueVisitor &visitor) override { visitor.visit(*this); }
+    void accept(ConstFieldValueVisitor &visitor) const override { visitor.visit(*this); }
+
+    const DataType *getDataType() const override { return DataType::DOUBLE; }
+    DoubleFieldValue* clone() const override { return new DoubleFieldValue(*this); }
+
+    using NumericFieldValue<Number>::operator=;
+
+    DECLARE_IDENTIFIABLE(DoubleFieldValue);
+
+};
+
+} // document
+
diff --git a/document/src/vespa/document/fieldvalue/fieldvalue.cpp b/document/src/vespa/document/fieldvalue/fieldvalue.cpp
new file mode 100644
index 00000000000..8a9b332709b
--- /dev/null
+++ b/document/src/vespa/document/fieldvalue/fieldvalue.cpp
@@ -0,0 +1,274 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/document/fieldvalue/fieldvalue.h>
+
+#include <vespa/document/base/exceptions.h>
+#include <vespa/document/datatype/arraydatatype.h>
+#include <vespa/document/datatype/numericdatatype.h>
+#include <vespa/document/datatype/primitivedatatype.h>
+#include <vespa/document/fieldvalue/arrayfieldvalue.h>
+#include <vespa/document/fieldvalue/intfieldvalue.h>
+#include <vespa/document/fieldvalue/floatfieldvalue.h>
+#include <vespa/document/fieldvalue/stringfieldvalue.h>
+#include <vespa/document/fieldvalue/rawfieldvalue.h>
+#include <vespa/document/fieldvalue/longfieldvalue.h>
+#include <vespa/document/fieldvalue/doublefieldvalue.h>
+#include <vespa/document/fieldvalue/bytefieldvalue.h>
+#include <vespa/document/fieldvalue/predicatefieldvalue.h>
+#include <vespa/document/serialization/vespadocumentserializer.h>
+#include <vespa/document/util/bytebuffer.h>
+#include <vespa/vespalib/objects/fieldbase.h>
+#include <vespa/vespalib/objects/nbostream.h>
+
+using vespalib::FieldBase;
+using vespalib::nbostream;
+using vespalib::DefaultAlloc;
+
+namespace document {
+
+IMPLEMENT_IDENTIFIABLE_ABSTRACT(FieldValue, vespalib::Identifiable);
+
+void FieldValue::serialize(nbostream &stream) const {
+    VespaDocumentSerializer serializer(stream);
+    serializer.write(*this);
+}
+
+void FieldValue::serialize(ByteBuffer& buffer) const {
+    nbostream stream;
+    serialize(stream);
+    buffer.putBytes(stream.peek(), stream.size());
+}
+
+std::unique_ptr<ByteBuffer> FieldValue::serialize() const {
+    nbostream stream;
+    serialize(stream);
+
+    std::unique_ptr<ByteBuffer> retVal(new ByteBuffer(stream.size()));
+    retVal->putBytes(stream.peek(), stream.size());
+    return retVal;
+}
+
+size_t
+FieldValue::hash() const
+{
+    vespalib::nbostream os;
+    serialize(os);
+    return vespalib::hashValue(os.c_str(), os.size()) ;
+}
+
+FieldValue&
+FieldValue::assign(const FieldValue& value)
+{
+    throw vespalib::IllegalArgumentException(
+            "Cannot assign value of type " + value.getDataType()->toString()
+            + " to value of type " + value.getDataType()->toString(), VESPA_STRLOC);
+}
+
+/**
+ * Normally, tag names are decided by the parent node, so children will not
+ * print their parents. For toXML to work on non-root (non-document) nodes
+ * though, we've overwritten toXml to write a start tag for leaf nodes, if
+ * they are used as root nodes in toXml call.
+ */
+std::string
+FieldValue::toXml(const std::string& indent) const
+{
+    std::ostringstream ost;
+    XmlOutputStream xos(ost, indent);
+    xos << XmlTag("value");
+    printXml(xos);
+    xos << XmlEndTag();
+    return ost.str();
+}
+
+// Subtypes should implement the conversion functions that make sense
+
+FieldValue& FieldValue::operator=(const vespalib::stringref &)
+{
+    throw vespalib::IllegalArgumentException(
+            "Cannot assign string to datatype " + getDataType()->toString(),
+            VESPA_STRLOC);
+}
+
+FieldValue& FieldValue::operator=(int32_t)
+{
+    throw vespalib::IllegalArgumentException(
+            "Cannot assign int to datatype " + getDataType()->toString(),
+            VESPA_STRLOC);
+}
+
+FieldValue& FieldValue::operator=(int64_t)
+{
+    throw vespalib::IllegalArgumentException(
+            "Cannot assign long to datatype " + getDataType()->toString(),
+            VESPA_STRLOC);
+}
+
+FieldValue& FieldValue::operator=(float)
+{
+    throw vespalib::IllegalArgumentException(
+            "Cannot assign float to datatype " + getDataType()->toString(),
+            VESPA_STRLOC);
+}
+
+FieldValue& FieldValue::operator=(double)
+{
+    throw vespalib::IllegalArgumentException(
+            "Cannot assign double to datatype " + getDataType()->toString(),
+            VESPA_STRLOC);
+}
+
+char FieldValue::getAsByte() const
+{
+    throw InvalidDataTypeConversionException(
+            *getDataType(), *DataType::BYTE, VESPA_STRLOC);
+}
+
+int32_t FieldValue::getAsInt() const
+{
+    throw InvalidDataTypeConversionException(
+            *getDataType(), *DataType::INT, VESPA_STRLOC);
+}
+
+int64_t FieldValue::getAsLong() const
+{
+    throw InvalidDataTypeConversionException(
+            *getDataType(), *DataType::LONG, VESPA_STRLOC);
+}
+
+float FieldValue::getAsFloat() const
+{
+    throw InvalidDataTypeConversionException(
+            *getDataType(), *DataType::FLOAT, VESPA_STRLOC);
+}
+
+double FieldValue::getAsDouble() const
+{
+    throw InvalidDataTypeConversionException(
+            *getDataType(), *DataType::DOUBLE, VESPA_STRLOC);
+}
+
+vespalib::string FieldValue::getAsString() const
+{
+    throw InvalidDataTypeConversionException(
+            *getDataType(), *DataType::STRING, VESPA_STRLOC);
+}
+
+std::pair<const char*, size_t> FieldValue::getAsRaw() const
+{
+    throw InvalidDataTypeConversionException(
+            *getDataType(), *DataType::RAW, VESPA_STRLOC);
+}
+
+FieldValue::UP FieldValue::getNestedFieldValue(FieldPath::const_iterator start, FieldPath::const_iterator end) const
+{
+    return (start != end) ? onGetNestedFieldValue(start, end) : FieldValue::UP();
+}
+
+FieldValue::UP FieldValue::onGetNestedFieldValue(FieldPath::const_iterator start, FieldPath::const_iterator end) const
+{
+    (void) start;
+    (void) end;
+    return FieldValue::UP();
+}
+
+FieldValue::IteratorHandler::ModificationStatus
+FieldValue::iterateNested(FieldPath::const_iterator start, FieldPath::const_iterator end, IteratorHandler & handler) const
+{
+    return onIterateNested(start, end, handler);
+}
+
+FieldValue::IteratorHandler::ModificationStatus
+FieldValue::onIterateNested(FieldPath::const_iterator start, FieldPath::const_iterator end, IteratorHandler & handler) const
+{
+    if (start == end) {
+        handler.handlePrimitive(*this);
+        return handler.modify(const_cast<FieldValue&>(*this));
+    } else {
+        throw vespalib::IllegalArgumentException("Primitive types can't be iterated through");
+    }
+}
+
+bool
+FieldValue::IteratorHandler::IndexValue::operator==(const FieldValue::IteratorHandler::IndexValue& other) const {
+    if (key.get() != NULL) {
+        if (other.key.get() != NULL && *key == *other.key) {
+            return true;
+        }
+        return false;
+    }
+
+    return index == other.index;
+}
+
+void FieldValue::IteratorHandler::handlePrimitive(const FieldValue & fv) { onPrimitive(Content(fv, getWeight())); }
+bool FieldValue::IteratorHandler::handleComplex(const FieldValue & fv) { return onComplex(Content(fv, getWeight())); }
+void FieldValue::IteratorHandler::handleCollectionStart(const FieldValue & fv) { onCollectionStart(Content(fv, getWeight())); }
+void FieldValue::IteratorHandler::handleCollectionEnd(const FieldValue & fv) { onCollectionEnd(Content(fv, getWeight())); }
+void FieldValue::IteratorHandler::handleStructStart(const FieldValue & fv) { onStructStart(Content(fv, getWeight())); }
+void FieldValue::IteratorHandler::handleStructEnd(const FieldValue & fv) { onStructEnd(Content(fv, getWeight())); }
+
+std::string
+FieldValue::IteratorHandler::toString(const VariableMap& vars) {
+    std::ostringstream out;
+    out << "[ ";
+    for (FieldValue::IteratorHandler::VariableMap::const_iterator iter = vars.begin();
+         iter != vars.end();
+         iter++) {
+        out << iter->first << "=" << iter->second.toString() << " ";
+    }
+    out << "]";
+    return out.str();
+}
+
+std::string
+FieldValue::toString(bool verbose, const std::string& indent) const
+{
+    std::ostringstream o;
+    print(o, verbose, indent);
+    return o.str();
+}
+
+using vespalib::ComplexArrayT;
+using vespalib::PrimitiveArrayT;
+
+namespace {
+class FieldValueFactory : public ComplexArrayT<FieldValue>::Factory
+{
+public:
+    FieldValueFactory(DataType::UP dataType) : _dataType(dataType.release()) { }
+    FieldValue * create() { return _dataType->createFieldValue().release(); }
+    virtual FieldValueFactory * clone() const { return new FieldValueFactory(*this); }
+private:
+    DataType::CP _dataType;
+};
+}
+
+FieldValue::IArray::UP
+FieldValue::createArray(const DataType & baseType)
+{
+    switch(baseType.getId()) {
+    case DataType::T_INT:
+        return IArray::UP(new PrimitiveArrayT<IntFieldValue, FieldValue>());
+    case DataType::T_FLOAT:
+        return IArray::UP(new PrimitiveArrayT<FloatFieldValue, FieldValue>());
+    case DataType::T_STRING:
+        return IArray::UP(new PrimitiveArrayT<StringFieldValue, FieldValue>());
+    case DataType::T_RAW:
+        return IArray::UP(new PrimitiveArrayT<RawFieldValue, FieldValue>());
+    case DataType::T_LONG:
+        return IArray::UP(new PrimitiveArrayT<LongFieldValue, FieldValue>());
+    case DataType::T_DOUBLE:
+        return IArray::UP(new PrimitiveArrayT<DoubleFieldValue, FieldValue>());
+    case DataType::T_BYTE:
+        return IArray::UP(new PrimitiveArrayT<ByteFieldValue, FieldValue>());
+    default:
+        return IArray::UP(new ComplexArrayT<FieldValue>(FieldValueFactory::UP(new FieldValueFactory(DataType::UP(baseType.clone())))));
+    }
+}
+
+
+
+
+} // document
diff --git a/document/src/vespa/document/fieldvalue/fieldvalue.h b/document/src/vespa/document/fieldvalue/fieldvalue.h
new file mode 100644
index 00000000000..a3292a5cd36
--- /dev/null
+++ b/document/src/vespa/document/fieldvalue/fieldvalue.h
@@ -0,0 +1,336 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * \class document::FieldValue
+ * \ingroup fieldvalue
+ *
+ * \brief Wraps values stored in documents.
+ *
+ * This class is a superclass for all values that can be stored within a
+ * document. A field value stores data as defined by the datatype belonging
+ * to the value.
+ */
+#pragma once
+
+#include <vespa/vespalib/objects/cloneable.h>
+
+#include "fieldvaluevisitor.h"
+#include <vespa/document/datatype/datatype.h>
+#include <vespa/document/util/xmlserializable.h>
+#include <vespa/vespalib/util/polymorphicarrays.h>
+#include <vespa/document/util/bytebuffer.h>
+#include <vespa/vespalib/objects/nbostream.h>
+#include <map>
+
+namespace document {
+
+class FieldValue : public vespalib::Identifiable
+{
+protected:
+    FieldValue(const FieldValue&) { }
+    FieldValue& operator=(const FieldValue&) { return *this; }
+    typedef vespalib::IArrayT<FieldValue> IArray;
+    static IArray::UP createArray(const DataType & baseType);
+
+public:
+    typedef std::unique_ptr<FieldValue> UP;
+    typedef std::shared_ptr<FieldValue> SP;
+    typedef vespalib::LinkedPtr<FieldValue> LP;
+    typedef vespalib::CloneablePtr<FieldValue> CP;
+
+    class IteratorHandler {
+    public:
+        class CollectionScope {
+        public:
+            CollectionScope(IteratorHandler& handler, const FieldValue& value)
+                : _handler(handler), _value(value)
+            {
+                _handler.handleCollectionStart(_value);
+            }
+
+            ~CollectionScope() {
+                _handler.handleCollectionEnd(_value);
+            }
+        private:
+            IteratorHandler& _handler;
+            const FieldValue& _value;
+        };
+
+        class StructScope {
+        public:
+            StructScope(IteratorHandler& handler, const FieldValue& value)
+                : _handler(handler), _value(value)
+            {
+                _handler.handleStructStart(_value);
+            }
+
+            ~StructScope() {
+                _handler.handleStructEnd(_value);
+            }
+        private:
+            IteratorHandler& _handler;
+            const FieldValue& _value;
+        };
+
+        class IndexValue {
+        public:
+            IndexValue() : index(-1), key() {}
+            IndexValue(int index_) : index(index_), key() {}
+            IndexValue(const FieldValue& key_)
+                : index(-1),
+                  key(FieldValue::CP(key_.clone()))
+            {}
+
+            std::string toString() const {
+                if (key.get() != NULL) {
+                    return key->toString();
+                } else {
+                    return vespalib::make_string("%d", index);
+                }
+            }
+
+            bool operator==(const IndexValue& other) const;
+
+            int index; // For array
+            FieldValue::CP key; // For map/wset
+        };
+
+        enum ModificationStatus {
+            MODIFIED,
+            REMOVED,
+            NOT_MODIFIED
+        };
+
+        typedef std::map<vespalib::string, IndexValue> VariableMap;
+    protected:
+        class Content {
+        public:
+            Content(const FieldValue & fv, int weight=1) : _fieldValue(fv), _weight(weight) { }
+            int getWeight()               const { return _weight; }
+            const FieldValue & getValue() const { return _fieldValue; }
+        private:
+            const FieldValue & _fieldValue;
+            int                _weight;
+        };
+        IteratorHandler() : _weight(1) { }
+    public:
+        virtual ~IteratorHandler() { }
+
+        void handlePrimitive(const FieldValue & fv);
+
+        /**
+           Handles a complex type (struct/array/map etc) that is at the end of the
+           field path.
+           @return Return true if you want to recurse into the members.
+        */
+        bool handleComplex(const FieldValue& fv);
+        void handleCollectionStart(const FieldValue & fv);
+        void handleCollectionEnd(const FieldValue & fv);
+        void handleStructStart(const FieldValue & fv);
+        void handleStructEnd(const FieldValue & fv);
+        void setWeight(int weight) { _weight = weight; }
+        ModificationStatus modify(FieldValue& fv) { return doModify(fv); }
+
+        VariableMap& getVariables() { return _variables; }
+        void setVariables(const VariableMap& vars) { _variables = vars; }
+        static std::string toString(const VariableMap& vars);
+        virtual bool createMissingPath() const { return false; }
+    private:
+        virtual bool onComplex(const Content& fv) { (void) fv; return true; }
+        virtual void onPrimitive(const Content & fv) { (void) fv; }
+        virtual void onCollectionStart(const Content & fv) { (void) fv; }
+        virtual void onCollectionEnd(const Content & fv) { (void) fv; }
+        virtual void onStructStart(const Content & fv) { (void) fv; }
+        virtual void onStructEnd(const Content & fv) { (void) fv; }
+        virtual ModificationStatus doModify(FieldValue&) { return NOT_MODIFIED; };
+
+        // Scratchpad to store pass on weight.
+        int getWeight() const  { return _weight; }
+        int _weight;
+        VariableMap _variables;
+    };
+
+    DECLARE_IDENTIFIABLE_ABSTRACT(FieldValue);
+
+    FieldValue() {}
+
+    /**
+     * Visit this fieldvalue for double dispatch.
+     */
+    virtual void accept(FieldValueVisitor &visitor) = 0;
+    virtual void accept(ConstFieldValueVisitor &visitor) const = 0;
+
+    /**
+     * operator= is only implemented for leaf types, such that they are
+     * checked at compile time. assign() here can be used to assign potentially
+     * any value to this field value. It will check whether type is supported
+     * at runtime. Note that operator= has some overloaded functions allowing
+     * to assign primitives to fieldvalues too.
+     *
+     * @throw vespalib::IllegalArgumentException If type of value not compatible
+     */
+    virtual FieldValue& assign(const FieldValue&);
+
+    /** Get the datatype describing what can be stored in this fieldvalue. */
+    virtual const DataType *getDataType() const = 0;
+
+    /** Wrapper for datatypes isA() function. See DataType. */
+    virtual bool isA(const FieldValue& other) const
+        { return (getDataType()->isA(*other.getDataType())); }
+
+    void serialize(vespalib::nbostream &stream) const;
+    void serialize(ByteBuffer& buffer) const;
+    ByteBuffer::UP serialize() const;
+
+    /**
+     * Compares this fieldvalue with another fieldvalue.
+     * Should return 0 if the two are equal, <0 if this object is "less" than
+     * the other, and >0 if this object is more than the other.
+     */
+    virtual int compare(const FieldValue& other) const {
+        const DataType & a = *getDataType();
+        const DataType & b = *other.getDataType();
+        return (a < b)
+                   ? -1
+                   : (b < a)
+                       ? 1
+                       : 0;
+    }
+
+    /**
+     * Returns true if this object have been altered since last
+     * serialization/deserialization. If hasChanged() is false, then cached
+     * information from last serialization effort is still valid.
+     */
+    virtual bool hasChanged() const = 0;
+
+    /** Cloneable implementation */
+    virtual FieldValue* clone() const = 0;
+
+      // Utility methods to be able to compare values easily
+    bool operator>(const FieldValue& v)  const { return (compare(v) > 0); }
+    bool operator>=(const FieldValue& v) const { return (compare(v) >= 0); }
+    bool operator==(const FieldValue& v) const { return (compare(v) == 0); }
+    bool operator<=(const FieldValue& v) const { return (compare(v) <= 0); }
+    bool operator<(const FieldValue& v)  const { return (compare(v) < 0); }
+    bool operator!=(const FieldValue& v) const { return (compare(v) != 0); }
+    virtual size_t hash() const;
+
+    /** Override toXml from XmlSerializable to add start/stop tags. */
+    virtual std::string toXml(const std::string& indent = "") const;
+
+    // Utility functions to set commonly used value types.
+    virtual FieldValue& operator=(const vespalib::stringref &);
+    virtual FieldValue& operator=(int32_t);
+    virtual FieldValue& operator=(int64_t);
+    virtual FieldValue& operator=(float);
+    virtual FieldValue& operator=(double);
+
+    // Utility functions to unwrap field values if you know the type.
+
+    /**
+     * @return Returns the wrapped value if it is a byte or compatible type.
+     * @throws document::InvalidDataTypeConversionException
+     */
+    virtual char getAsByte() const;
+
+    /**
+     * @return Returns the wrapped value if it is an int or compatible type.
+     * @throws document::InvalidDataTypeConversionException
+     */
+    virtual int32_t getAsInt() const;
+
+    /**
+     * @return Returns the wrapped value if it is a long or compatible type.
+     * @throws document::InvalidDataTypeConversionException
+     */
+    virtual int64_t getAsLong() const;
+
+    /**
+     * @return Returns the wrapped value if it is a float or compatible type.
+     * @throws document::InvalidDataTypeConversionException
+     */
+    virtual float getAsFloat() const;
+
+    /**
+     * @return Returns the wrapped value if it is a double or compatible type.
+     * @throws document::InvalidDataTypeConversionException
+     */
+    virtual double getAsDouble() const;
+
+    /**
+     * @return Returns the wrapped value if it is a string or compatible type.
+     * @throws document::InvalidDataTypeConversionException
+     */
+    virtual vespalib::string getAsString() const;
+
+    /**
+     * @return Returns the wrapped value if it is a raw or compatible type.
+     * @throws document::InvalidDataTypeConversionException
+     */
+    virtual std::pair<const char*, size_t> getAsRaw() const;
+
+    /**
+     * Will give you the leaf fieldvalue you are looking for in your fieldPath.
+     * If the path does not lead anywhere an empty UP will be returned.
+     */
+    FieldValue::UP getNestedFieldValue(
+            FieldPath::const_iterator start,
+            FieldPath::const_iterator end) const;
+
+    /**
+     * Will iterate the possibly nested fieldvalue depth first.
+     * It will follow the specifed path with proper invocations of
+     * onXXXStart/onXXXEnd.  At end it will iterate the rest below with
+     * invocations of the before mentioned methods and the additional
+     * onPrimitive.
+     */
+    IteratorHandler::ModificationStatus iterateNested(
+            FieldPath::const_iterator start,
+            FieldPath::const_iterator end,
+            IteratorHandler & handler) const;
+
+    IteratorHandler::ModificationStatus iterateNested(
+            const FieldPath& fieldPath,
+            IteratorHandler& handler) const
+    {
+        return iterateNested(fieldPath.begin(), fieldPath.end(), handler);
+    }
+
+    virtual void print(std::ostream& out,
+                       bool verbose,
+                       const std::string& indent) const = 0;
+    // Duplication to reduce size of FieldValue
+    void print(std::ostream& out) const { print(out, false, ""); }
+    void print(std::ostream& out, bool verbose) const { print(out, verbose, ""); }
+    void print(std::ostream& out, const std::string& indent) const { print(out, false, indent); }
+    /** Utility function to get this output as a string.  */
+    std::string toString(bool verbose=false, const std::string& indent="") const;
+    virtual void printXml(XmlOutputStream& out) const = 0;
+
+private:
+    virtual FieldValue::UP onGetNestedFieldValue(
+                                 FieldPath::const_iterator start,
+                                 FieldPath::const_iterator end) const;
+
+    virtual IteratorHandler::ModificationStatus onIterateNested(
+            FieldPath::const_iterator start,
+            FieldPath::const_iterator end,
+            IteratorHandler & handler) const;
+};
+
+inline bool operator != (const FieldValue::LP & a, const FieldValue::LP & b) { return *a != *b; }
+inline bool operator  < (const FieldValue::LP & a, const FieldValue::LP & b) { return *a < *b; }
+
+inline std::ostream& operator<<(std::ostream& out, const FieldValue & p) {
+    p.print(out);
+    return out;
+}
+
+inline XmlOutputStream & operator<<(XmlOutputStream & out, const FieldValue & p) {
+    p.printXml(out);
+    return out;
+}
+
+
+} // document
+
diff --git a/document/src/vespa/document/fieldvalue/fieldvalues.h b/document/src/vespa/document/fieldvalue/fieldvalues.h
new file mode 100644
index 00000000000..70b54e2e24d
--- /dev/null
+++ b/document/src/vespa/document/fieldvalue/fieldvalues.h
@@ -0,0 +1,19 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/document/fieldvalue/arrayfieldvalue.h>
+#include <vespa/document/fieldvalue/bytefieldvalue.h>
+#include <vespa/document/fieldvalue/document.h>
+#include <vespa/document/fieldvalue/doublefieldvalue.h>
+#include <vespa/document/fieldvalue/floatfieldvalue.h>
+#include <vespa/document/fieldvalue/intfieldvalue.h>
+#include <vespa/document/fieldvalue/longfieldvalue.h>
+#include <vespa/document/fieldvalue/mapfieldvalue.h>
+#include <vespa/document/fieldvalue/predicatefieldvalue.h>
+#include <vespa/document/fieldvalue/rawfieldvalue.h>
+#include <vespa/document/fieldvalue/shortfieldvalue.h>
+#include <vespa/document/fieldvalue/stringfieldvalue.h>
+#include <vespa/document/fieldvalue/weightedsetfieldvalue.h>
+#include <vespa/document/fieldvalue/tensorfieldvalue.h>
+
diff --git a/document/src/vespa/document/fieldvalue/fieldvaluevisitor.h b/document/src/vespa/document/fieldvalue/fieldvaluevisitor.h
new file mode 100644
index 00000000000..e57b4394491
--- /dev/null
+++ b/document/src/vespa/document/fieldvalue/fieldvaluevisitor.h
@@ -0,0 +1,66 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+namespace document {
+class AnnotationReferenceFieldValue;
+class ArrayFieldValue;
+class ByteFieldValue;
+class Document;
+class DoubleFieldValue;
+class FloatFieldValue;
+class IntFieldValue;
+class LongFieldValue;
+class MapFieldValue;
+class PredicateFieldValue;
+class RawFieldValue;
+class ShortFieldValue;
+class StringFieldValue;
+class StructFieldValue;
+class WeightedSetFieldValue;
+class TensorFieldValue;
+
+struct FieldValueVisitor {
+    virtual ~FieldValueVisitor() {}
+
+    virtual void visit(AnnotationReferenceFieldValue &value) = 0;
+    virtual void visit(ArrayFieldValue &value) = 0;
+    virtual void visit(ByteFieldValue &value) = 0;
+    virtual void visit(Document &value) = 0;
+    virtual void visit(DoubleFieldValue &value) = 0;
+    virtual void visit(FloatFieldValue &value) = 0;
+    virtual void visit(IntFieldValue &value) = 0;
+    virtual void visit(LongFieldValue &value) = 0;
+    virtual void visit(MapFieldValue &value) = 0;
+    virtual void visit(PredicateFieldValue &value) = 0;
+    virtual void visit(RawFieldValue &value) = 0;
+    virtual void visit(ShortFieldValue &value) = 0;
+    virtual void visit(StringFieldValue &value) = 0;
+    virtual void visit(StructFieldValue &value) = 0;
+    virtual void visit(WeightedSetFieldValue &value) = 0;
+    virtual void visit(TensorFieldValue &value) = 0;
+};
+
+struct ConstFieldValueVisitor {
+    virtual ~ConstFieldValueVisitor() {}
+
+    virtual void visit(const AnnotationReferenceFieldValue &value) = 0;
+    virtual void visit(const ArrayFieldValue &value) = 0;
+    virtual void visit(const ByteFieldValue &value) = 0;
+    virtual void visit(const Document &value) = 0;
+    virtual void visit(const DoubleFieldValue &value) = 0;
+    virtual void visit(const FloatFieldValue &value) = 0;
+    virtual void visit(const IntFieldValue &value) = 0;
+    virtual void visit(const LongFieldValue &value) = 0;
+    virtual void visit(const MapFieldValue &value) = 0;
+    virtual void visit(const PredicateFieldValue &value) = 0;
+    virtual void visit(const RawFieldValue &value) = 0;
+    virtual void visit(const ShortFieldValue &value) = 0;
+    virtual void visit(const StringFieldValue &value) = 0;
+    virtual void visit(const StructFieldValue &value) = 0;
+    virtual void visit(const WeightedSetFieldValue &value) = 0;
+    virtual void visit(const TensorFieldValue &value) = 0;
+};
+
+}  // namespace document
+
diff --git a/document/src/vespa/document/fieldvalue/fieldvaluewriter.h b/document/src/vespa/document/fieldvalue/fieldvaluewriter.h
new file mode 100644
index 00000000000..18d39403ff8
--- /dev/null
+++ b/document/src/vespa/document/fieldvalue/fieldvaluewriter.h
@@ -0,0 +1,16 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+namespace document {
+class FieldValue;
+
+struct FieldValueWriter {
+    virtual ~FieldValueWriter() {}
+
+    virtual void writeFieldValue(const FieldValue &value) = 0;
+    virtual void writeSerializedData(const void *buf, size_t length) = 0;
+};
+
+}  // namespace document
+
diff --git a/document/src/vespa/document/fieldvalue/floatfieldvalue.cpp b/document/src/vespa/document/fieldvalue/floatfieldvalue.cpp
new file mode 100644
index 00000000000..a2553dd7f96
--- /dev/null
+++ b/document/src/vespa/document/fieldvalue/floatfieldvalue.cpp
@@ -0,0 +1,10 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/document/fieldvalue/floatfieldvalue.h>
+
+namespace document {
+
+IMPLEMENT_IDENTIFIABLE(FloatFieldValue, NumericFieldValueBase);
+
+} // document
diff --git a/document/src/vespa/document/fieldvalue/floatfieldvalue.h b/document/src/vespa/document/fieldvalue/floatfieldvalue.h
new file mode 100644
index 00000000000..d579b1b097f
--- /dev/null
+++ b/document/src/vespa/document/fieldvalue/floatfieldvalue.h
@@ -0,0 +1,35 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * \class document::FloatFieldValue
+ * \ingroup fieldvalue
+ *
+ * \brief Wrapper for field values of datatype FLOAT.
+ */
+#pragma once
+
+#include <vespa/document/datatype/numericdatatype.h>
+#include <vespa/document/fieldvalue/numericfieldvalue.h>
+
+namespace document {
+
+class FloatFieldValue : public NumericFieldValue<float> {
+public:
+    typedef std::unique_ptr<FloatFieldValue> UP;
+    typedef float Number;
+
+    FloatFieldValue(Number value = 0) : NumericFieldValue<Number>(value) {}
+
+    void accept(FieldValueVisitor &visitor) override { visitor.visit(*this); }
+    void accept(ConstFieldValueVisitor &visitor) const override { visitor.visit(*this); }
+
+    const DataType *getDataType() const override { return DataType::FLOAT; }
+    FloatFieldValue* clone() const override { return new FloatFieldValue(*this); }
+
+    using NumericFieldValue<Number>::operator=;
+
+    DECLARE_IDENTIFIABLE(FloatFieldValue);
+
+};
+
+} // document
+
diff --git a/document/src/vespa/document/fieldvalue/intfieldvalue.cpp b/document/src/vespa/document/fieldvalue/intfieldvalue.cpp
new file mode 100644
index 00000000000..e48d74e1f77
--- /dev/null
+++ b/document/src/vespa/document/fieldvalue/intfieldvalue.cpp
@@ -0,0 +1,10 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/document/fieldvalue/intfieldvalue.h>
+
+namespace document {
+
+IMPLEMENT_IDENTIFIABLE(IntFieldValue, NumericFieldValueBase);
+
+} // document
diff --git a/document/src/vespa/document/fieldvalue/intfieldvalue.h b/document/src/vespa/document/fieldvalue/intfieldvalue.h
new file mode 100644
index 00000000000..007e27dd290
--- /dev/null
+++ b/document/src/vespa/document/fieldvalue/intfieldvalue.h
@@ -0,0 +1,34 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * \class document::IntFieldValue
+ * \ingroup fieldvalue
+ *
+ * \brief Wrapper for field values of datatype INT.
+ */
+#pragma once
+
+#include <vespa/document/datatype/numericdatatype.h>
+#include <vespa/document/fieldvalue/numericfieldvalue.h>
+
+namespace document {
+
+class IntFieldValue : public NumericFieldValue<int32_t> {
+public:
+    typedef std::unique_ptr<IntFieldValue> UP;
+    typedef int32_t Number;
+
+    IntFieldValue(Number value = 0) : NumericFieldValue<Number>(value) {}
+
+    void accept(FieldValueVisitor &visitor) override { visitor.visit(*this); }
+    void accept(ConstFieldValueVisitor &visitor) const override { visitor.visit(*this); }
+
+    const DataType *getDataType() const override { return DataType::INT; }
+    IntFieldValue* clone() const override { return new IntFieldValue(*this); }
+
+    using NumericFieldValue<Number>::operator=;
+    DECLARE_IDENTIFIABLE(IntFieldValue);
+
+};
+
+} // document
+
diff --git a/document/src/vespa/document/fieldvalue/literalfieldvalue.cpp b/document/src/vespa/document/fieldvalue/literalfieldvalue.cpp
new file mode 100644
index 00000000000..3d492e1e1e8
--- /dev/null
+++ b/document/src/vespa/document/fieldvalue/literalfieldvalue.cpp
@@ -0,0 +1,135 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/document/fieldvalue/literalfieldvalue.h>
+
+namespace document {
+
+IMPLEMENT_IDENTIFIABLE_ABSTRACT(LiteralFieldValueB, FieldValue);
+
+LiteralFieldValueB::LiteralFieldValueB(const LiteralFieldValueB& other)
+    : FieldValue(other),
+      _value(),
+      _backing(other.getValueRef()),
+      _altered(other._altered)
+{
+    _value = _backing;
+}
+
+LiteralFieldValueB::LiteralFieldValueB(const string& value)
+    : FieldValue(),
+      _value(),
+      _backing(value),
+      _altered(true)
+{
+    _value = _backing;
+}
+
+LiteralFieldValueB &
+LiteralFieldValueB::operator=(const LiteralFieldValueB& other)
+{
+    FieldValue::operator=(other);
+    _backing = other.getValueRef();
+    _value = _backing;
+    _altered = other._altered;
+    return *this;
+}
+
+FieldValue&
+LiteralFieldValueB::assign(const FieldValue& value)
+{
+    if (value.getDataType() == getDataType()) {
+        return operator=(static_cast<const LiteralFieldValueB&>(value));
+    }
+    return FieldValue::assign(value);
+}
+
+int
+LiteralFieldValueB::compare(const FieldValue& other) const
+{
+    if (*getDataType() == *other.getDataType()) {
+        const LiteralFieldValueB& sval(
+                static_cast<const LiteralFieldValueB&>(other));
+        return getValueRef().compare(sval.getValueRef());
+    }
+    return (getDataType()->getId() - other.getDataType()->getId());
+}
+
+void
+LiteralFieldValueB::printXml(XmlOutputStream& out) const
+{
+    out << XmlContentWrapper(_value.c_str(), _value.size());
+}
+
+void
+LiteralFieldValueB::
+print(std::ostream& out, bool, const std::string&) const
+{
+    vespalib::string escaped;
+    out << StringUtil::escape(getValue(), escaped);
+}
+
+FieldValue&
+LiteralFieldValueB::operator=(const vespalib::stringref & value)
+{
+    setValue(value);
+    return *this;
+}
+
+vespalib::string
+LiteralFieldValueB::getAsString() const
+{
+    return getValue();
+}
+
+std::pair<const char*, size_t>
+LiteralFieldValueB::getAsRaw() const
+{
+    return std::make_pair(_value.c_str(), _value.size());
+}
+
+void
+LiteralFieldValueB::syncBacking() const
+{
+    _backing = _value;
+    _value = _backing;
+}
+
+
+namespace {
+template <typename T>
+std::string valueToString(T value) {
+    std::ostringstream ost;
+    ost << value;
+    return ost.str();
+}
+}  // namespace
+
+FieldValue&
+LiteralFieldValueB::operator=(int32_t value)
+{
+    setValue(valueToString(value));
+    return *this;
+}
+
+FieldValue&
+LiteralFieldValueB::operator=(int64_t value)
+{
+    setValue(valueToString(value));
+    return *this;
+}
+
+FieldValue&
+LiteralFieldValueB::operator=(float value)
+{
+    setValue(valueToString(value));
+    return *this;
+}
+
+FieldValue&
+LiteralFieldValueB::operator=(double value)
+{
+    setValue(valueToString(value));
+    return *this;
+}
+}  // namespace document
diff --git a/document/src/vespa/document/fieldvalue/literalfieldvalue.h b/document/src/vespa/document/fieldvalue/literalfieldvalue.h
new file mode 100644
index 00000000000..14f0d199543
--- /dev/null
+++ b/document/src/vespa/document/fieldvalue/literalfieldvalue.h
@@ -0,0 +1,125 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * \class document::LiteralFieldValue
+ * \ingroup fieldvalue
+ *
+ * \brief Super class for primitive field values not containing numbers.
+ *
+ * Templated superclass to minimalize need for code duplication in such types.
+ * This covers strings, raw, termboost, URI at time of writing.
+ *
+ * Note that raw is a bit different than the rest as it define addZeroTerm
+ * false. Otherwise, the types are functionally equivalent, only difference is
+ * the type id.
+ */
+#pragma once
+
+#include <vespa/document/datatype/primitivedatatype.h>
+#include <vespa/document/fieldvalue/fieldvalue.h>
+#include <vespa/document/util/bytebuffer.h>
+#include <vespa/document/util/stringutil.h>
+
+namespace document {
+
+class LiteralFieldValueB : public FieldValue {
+public:
+    typedef vespalib::string string;
+    typedef vespalib::stringref stringref;
+    DECLARE_IDENTIFIABLE_ABSTRACT(LiteralFieldValueB);
+    typedef std::unique_ptr<LiteralFieldValueB> UP;
+    typedef string value_type;
+
+    LiteralFieldValueB() :
+        FieldValue(),
+        _value(),
+        _backing(),
+        _altered(true)
+    {
+        _value = _backing;
+    }
+
+    LiteralFieldValueB(const LiteralFieldValueB &);
+    LiteralFieldValueB(const string& value);
+
+    const value_type & getValue() const { sync(); return _backing; }
+    /**
+     * Get a ref to the value. If value has recently been deserialized, and
+     * never needed as an std::string before, this method lets you get a hold
+     * of the data without creating the string.
+     */
+    stringref getValueRef() const { return _value; }
+
+    LiteralFieldValueB & operator=(const LiteralFieldValueB &);
+
+    void setValueRef(const stringref & value) {
+        _value = value;
+        _altered = true;
+    }
+
+    void setValue(const stringref & value) {
+        _backing = value;
+        _value = _backing;
+        _altered = true;
+    }
+    virtual size_t hash() const { return vespalib::hashValue(_value.c_str()); }
+    void setValue(const char* val, size_t size) { setValue(stringref(val, size)); }
+
+        // FieldValue implementation.
+    virtual int compare(const FieldValue& other) const;
+
+    virtual vespalib::string getAsString() const;
+    virtual std::pair<const char*, size_t> getAsRaw() const;
+
+    virtual void printXml(XmlOutputStream& out) const;
+    virtual void print(std::ostream& out, bool verbose,
+                       const std::string& indent) const;
+    virtual FieldValue& assign(const FieldValue&);
+    virtual bool hasChanged() const { return _altered; }
+
+    virtual FieldValue& operator=(const vespalib::stringref &);
+    virtual FieldValue& operator=(int32_t);
+    virtual FieldValue& operator=(int64_t);
+    virtual FieldValue& operator=(float);
+    virtual FieldValue& operator=(double);
+protected:
+    void syncBacking() const __attribute__((noinline));
+    void sync() const {
+        if (__builtin_expect(_backing.c_str() != _value.c_str(), false)) {
+            syncBacking();
+        }
+    }
+    mutable stringref _value;
+    mutable string    _backing; // Lazily set when needed
+    mutable bool      _altered; // Set if altered after deserialization
+private:
+    virtual bool getAddZeroTerm() const = 0;
+};
+
+template<typename SubClass, int type, bool addZeroTerm>
+class LiteralFieldValue : public LiteralFieldValueB {
+private:
+    virtual bool getAddZeroTerm() const { return addZeroTerm; }
+public:
+    typedef std::unique_ptr<SubClass> UP;
+
+    LiteralFieldValue() : LiteralFieldValueB() { }
+    LiteralFieldValue(const string& value) : LiteralFieldValueB(value) { }
+    virtual const DataType *getDataType() const;
+};
+
+template<typename SubClass, int type, bool addZeroTerm>
+const DataType *
+LiteralFieldValue<SubClass, type, addZeroTerm>::getDataType() const
+{
+    switch (type) {
+    case DataType::T_URI:    return DataType::URI;
+    case DataType::T_STRING: return DataType::STRING;
+    case DataType::T_RAW:    return DataType::RAW;
+    default:
+        throw vespalib::IllegalStateException(vespalib::make_string(
+                        "Illegal literal type id %i", type), VESPA_STRLOC);
+    }
+}
+
+} // document
+
diff --git a/document/src/vespa/document/fieldvalue/longfieldvalue.cpp b/document/src/vespa/document/fieldvalue/longfieldvalue.cpp
new file mode 100644
index 00000000000..ee0cea2f5a3
--- /dev/null
+++ b/document/src/vespa/document/fieldvalue/longfieldvalue.cpp
@@ -0,0 +1,9 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/document/fieldvalue/longfieldvalue.h>
+
+namespace document {
+
+IMPLEMENT_IDENTIFIABLE(LongFieldValue, NumericFieldValueBase);
+
+} // document
diff --git a/document/src/vespa/document/fieldvalue/longfieldvalue.h b/document/src/vespa/document/fieldvalue/longfieldvalue.h
new file mode 100644
index 00000000000..6ccb54ab8dd
--- /dev/null
+++ b/document/src/vespa/document/fieldvalue/longfieldvalue.h
@@ -0,0 +1,35 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * \class document::LongFieldValue
+ * \ingroup fieldvalue
+ *
+ * \brief Wrapper for field values of datatype LONG.
+ */
+#pragma once
+
+#include <vespa/document/datatype/numericdatatype.h>
+#include <vespa/document/fieldvalue/numericfieldvalue.h>
+
+namespace document {
+
+class LongFieldValue : public NumericFieldValue<int64_t> {
+public:
+    typedef std::unique_ptr<LongFieldValue> UP;
+    typedef int64_t Number;
+
+    LongFieldValue(Number value = 0) : NumericFieldValue<Number>(value) {}
+
+    void accept(FieldValueVisitor &visitor) override { visitor.visit(*this); }
+    void accept(ConstFieldValueVisitor &visitor) const override { visitor.visit(*this); }
+
+    const DataType *getDataType() const override { return DataType::LONG; }
+    LongFieldValue* clone() const override { return new LongFieldValue(*this); }
+
+    using NumericFieldValue<Number>::operator=;
+
+    DECLARE_IDENTIFIABLE(LongFieldValue);
+
+};
+
+} // document
+
diff --git a/document/src/vespa/document/fieldvalue/mapfieldvalue.cpp b/document/src/vespa/document/fieldvalue/mapfieldvalue.cpp
new file mode 100644
index 00000000000..262592588c7
--- /dev/null
+++ b/document/src/vespa/document/fieldvalue/mapfieldvalue.cpp
@@ -0,0 +1,441 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+#include <vespa/document/fieldvalue/mapfieldvalue.h>
+
+#include <vespa/document/base/exceptions.h>
+#include <vespa/document/util/bytebuffer.h>
+//#include <vespa/vespalib/util/exceptions.h>
+#include <vespa/document/datatype/arraydatatype.h>
+#include <vespa/document/fieldvalue/arrayfieldvalue.h>
+#include <vespa/document/fieldvalue/weightedsetfieldvalue.h>
+#include <vespa/vespalib/objects/identifiable.h>
+
+using vespalib::Identifiable;
+
+LOG_SETUP(".document.fieldvalue.map");
+
+/// \todo TODO (was warning):
+// Find a way to search through internal map without
+// duplicating keys to create shared pointers.
+
+namespace document {
+
+IMPLEMENT_IDENTIFIABLE_ABSTRACT(MapFieldValue, FieldValue);
+
+namespace {
+const MapDataType *verifyMapType(const DataType& type) {
+    const MapDataType *ptr(Identifiable::cast<const MapDataType *>(&type));
+    if (!ptr) {
+        throw vespalib::IllegalArgumentException(
+                "Datatype given is not a map type", VESPA_STRLOC);
+    }
+    return ptr;
+}
+}  // namespace
+
+MapFieldValue::MapFieldValue(const DataType &mapType)
+    : FieldValue(),
+      _type(verifyMapType(mapType)),
+      _keys(createArray(getMapType().getKeyType())),
+      _values(createArray(getMapType().getValueType())),
+      _altered(true)
+{
+}
+
+MapFieldValue::~MapFieldValue()
+{
+}
+
+MapFieldValue::MapFieldValue(const MapFieldValue & rhs) :
+    FieldValue(rhs),
+    _type(rhs._type),
+    _keys(rhs._keys ? rhs._keys->clone() : nullptr),
+    _values(rhs._values ? rhs._values->clone() : nullptr),
+    _altered(rhs._altered)
+{
+}
+
+MapFieldValue &
+MapFieldValue::operator = (const MapFieldValue & rhs)
+{
+    if (this != & rhs) {
+        MapFieldValue copy(rhs);
+        swap(copy);
+    }
+    return *this;
+}
+
+void MapFieldValue::verifyKey(const FieldValue & fv) const
+{
+    const DataType &dt = getMapType().getKeyType();
+    if (!dt.isValueType(fv)) {
+        throw InvalidDataTypeException(*fv.getDataType(), dt, VESPA_STRLOC);
+    }
+}
+
+void MapFieldValue::verifyValue(const FieldValue & fv) const
+{
+    const DataType &dt = getMapType().getValueType();
+    if (!dt.isValueType(fv)) {
+        throw InvalidDataTypeException(*fv.getDataType(), dt, VESPA_STRLOC);
+    }
+}
+
+bool
+MapFieldValue::insertVerify(const FieldValue& key, const FieldValue& value)
+{
+    verifyKey(key);
+    verifyValue(value);
+    iterator found = find(key);
+    bool result(false);
+    if (found != end()) {
+        if (!(value == *found->second)) {
+            _altered = true;
+            found->second->assign(value);
+        }
+    } else {
+        push_back(key, value);
+        _altered = true;
+        result = true;
+    }
+    return result;
+}
+
+void
+MapFieldValue::push_back(const FieldValue& key, const FieldValue& value)
+{
+    _keys->push_back(key);
+    _values->push_back(value);
+    _altered = true;
+}
+
+
+void
+MapFieldValue::push_back(FieldValue::UP key, FieldValue::UP value)
+{
+    _keys->push_back(*key);
+    _values->push_back(*value);
+    _altered = true;
+}
+
+bool
+MapFieldValue::insert(FieldValue::UP key, FieldValue::UP value)
+{
+    return insertVerify(*key, *value);
+}
+
+bool
+MapFieldValue::put(FieldValue::UP key, FieldValue::UP value)
+{
+    return insertVerify(*key, *value);
+}
+
+bool
+MapFieldValue::put(const FieldValue& key, const FieldValue& value)
+{
+    return insertVerify(key, value);
+}
+
+bool
+MapFieldValue::addValue(const FieldValue& fv)
+{
+    return put(fv, fv);
+}
+
+FieldValue::UP
+MapFieldValue::get(const FieldValue& key) const
+{
+    const_iterator it = find(key);
+    return FieldValue::UP(it == end() ? nullptr : it->second->clone());
+}
+
+bool
+MapFieldValue::contains(const FieldValue& key) const
+{
+    verifyKey(key);
+    return find(key) != end();
+}
+
+bool
+MapFieldValue::erase(const FieldValue& key)
+{
+    verifyKey(key);
+    iterator found(find(key));
+    bool result(found != end());
+    if (result) {
+        _keys->erase(_keys->begin() + found.offset());
+        _values->erase(_values->begin() + found.offset());
+        _altered = true;
+    }
+    return result;
+}
+FieldValue&
+MapFieldValue::assign(const FieldValue& value)
+{
+    if (getDataType()->isValueType(value)) {
+        return operator=(static_cast<const MapFieldValue&>(value));
+    }
+    return FieldValue::assign(value);
+}
+
+int
+MapFieldValue::compare(const FieldValue& other) const
+{
+    int diff = FieldValue::compare(other);
+    if (diff != 0) return diff;
+
+    const MapFieldValue& o(dynamic_cast<const MapFieldValue&>(other));
+
+    if (_keys->size() != o._keys->size()) {
+        return (_keys->size() - o._keys->size());
+    }
+
+    const_iterator it1 = begin();
+
+    while (it1 != end()) {
+        const_iterator it2 = o.find(*it1->first);
+        if (it2 != o.end()) {
+            diff = it1->second->compare(*it2->second);
+            if (diff != 0) {
+                return diff;
+            }
+        } else {
+            return -1;
+        }
+        ++it1;
+    }
+    return 0;
+}
+
+void
+MapFieldValue::print(std::ostream& out, bool verbose,
+                       const std::string& indent) const
+{
+    out << "Map(";
+
+    int count = 0;
+    for (const auto & item : *this) {
+        if (count++ != 0) {
+            out << ",";
+        }
+        out << "\n" << indent << "  ";
+        item.first->print(out, verbose, indent + "  ");
+        out << " - ";
+        item.second->print(out, verbose, indent + "  ");
+    }
+    if (_keys->size() > 0) out << "\n" << indent;
+    out << ")";
+}
+
+void
+MapFieldValue::printXml(XmlOutputStream& xos) const
+{
+    for (const auto & item : *this) {
+        xos << XmlTag("item");
+        xos << XmlTag("key");
+        item.first->printXml(xos);
+        xos << XmlEndTag();
+        xos << XmlTag("value");
+        item.second->printXml(xos);
+        xos << XmlEndTag();
+        xos << XmlEndTag();
+    }
+}
+
+bool
+MapFieldValue::hasChanged() const
+{
+    // Keys are not allowed to change in a map, so the keys should not be
+    // referred to externally, and should thus not need to be checked.
+    return _altered;
+}
+
+MapFieldValue::const_iterator
+MapFieldValue::find(const FieldValue& key) const
+{
+    for(size_t i(0), m(_keys->size()); i < m; i++) {
+        if ((*_keys)[i] == key) {
+            return const_iterator(*this, i);
+        }
+    }
+    return end();
+}
+
+MapFieldValue::iterator
+MapFieldValue::find(const FieldValue& key)
+{
+    for(size_t i(0), m(_keys->size()); i < m; i++) {
+        if ((*_keys)[i] == key) {
+            return iterator(*this, i);
+        }
+    }
+    return end();
+}
+bool
+MapFieldValue::checkAndRemove(const FieldValue& key,
+                              FieldValue::IteratorHandler::ModificationStatus status,
+                              bool wasModified,
+                              std::vector<const FieldValue*>& keysToRemove) const
+{
+    if (status == FieldValue::IteratorHandler::REMOVED) {
+        LOG(spam, "will remove: %s", key.toString().c_str());
+        keysToRemove.push_back(&key);
+        return true;
+    } else if (status == FieldValue::IteratorHandler::MODIFIED) {
+        return true;
+    }
+
+    return wasModified;
+}
+
+FieldValue::IteratorHandler::ModificationStatus
+MapFieldValue::iterateNestedImpl(FieldPath::const_iterator start,
+                                 FieldPath::const_iterator end_,
+                                 IteratorHandler & handler,
+                                 const FieldValue& complexFieldValue) const
+{
+    IteratorHandler::CollectionScope autoScope(handler, complexFieldValue);
+    std::vector<const FieldValue*> keysToRemove;
+    bool wasModified = false;
+    const bool isWSet(complexFieldValue.inherits(WeightedSetFieldValue::classId));
+
+    if (start != end_) {
+        LOG(spam, "not yet at end of field path");
+        switch (start->getType()) {
+        case FieldPathEntry::MAP_KEY:
+        {
+            LOG(spam, "MAP_KEY");
+            const_iterator iter = find(*start->getLookupKey());
+            if (iter != end()) {
+                wasModified = checkAndRemove(*start->getLookupKey(),
+                        iter->second->iterateNested(start + 1, end_, handler),
+                        wasModified, keysToRemove);
+            } else if (handler.createMissingPath()) {
+                LOG(spam, "creating missing path");
+                FieldValue::UP val =
+                    getMapType().getValueType().createFieldValue();
+                IteratorHandler::ModificationStatus status
+                    = val->iterateNested(start + 1, end_, handler);
+                if (status == IteratorHandler::MODIFIED) {
+                    const_cast<MapFieldValue&>(*this).put(FieldValue::UP(start->getLookupKey()->clone()), std::move(val));
+                    return status;
+                }
+            }
+            break;
+        }
+        case FieldPathEntry::MAP_ALL_KEYS:
+            LOG(spam, "MAP_ALL_KEYS");
+            for (const_iterator it(begin()), mt(end()); it != mt; it++) {
+                if (isWSet) {
+                    handler.setWeight(static_cast<const IntFieldValue &>(*it->second).getValue());
+                }
+                wasModified = checkAndRemove(*it->first,
+                        it->first->iterateNested(start + 1, end_, handler),
+                        wasModified, keysToRemove);
+            }
+            break;
+        case FieldPathEntry::MAP_ALL_VALUES:
+            LOG(spam, "MAP_ALL_VALUES");
+            for (const_iterator it(begin()), mt(end()); it != mt; it++) {
+                wasModified = checkAndRemove(*it->second,
+                        it->second->iterateNested(start + 1, end_, handler),
+                        wasModified, keysToRemove);
+            }
+            break;
+        case FieldPathEntry::VARIABLE:
+        {
+            LOG(spam, "VARIABLE");
+            IteratorHandler::VariableMap::iterator
+                iter = handler.getVariables().find(start->getVariableName());
+            if (iter != handler.getVariables().end()) {
+                LOG(spam, "variable key = %s", iter->second.key->toString().c_str());
+                const_iterator found = find(*iter->second.key);
+                if (found != end()) {
+                    wasModified = checkAndRemove(*iter->second.key,
+                            found->second->iterateNested(start + 1, end_, handler),
+                            wasModified, keysToRemove);
+                }
+            } else {
+                for (const_iterator it(begin()), mt(end()); it != mt; it++) {
+                    LOG(spam, "key is '%s'", it->first->toString().c_str());
+                    handler.getVariables()[start->getVariableName()]
+                        = IteratorHandler::IndexValue(*it->first);
+                    LOG(spam, "vars at this time = %s",
+                        FieldValue::IteratorHandler::toString(handler.getVariables()).c_str());
+                    wasModified = checkAndRemove(*it->first,
+                            it->second->iterateNested(start + 1, end_, handler),
+                            wasModified, keysToRemove);
+                }
+                handler.getVariables().erase(start->getVariableName());
+            }
+            break;
+        }
+        default:
+            LOG(spam, "default");
+            for (const_iterator it(begin()), mt(end()); it != mt; it++) {
+                if (isWSet) {
+                    handler.setWeight(static_cast<const IntFieldValue &>(*it->second).getValue());
+                }
+                wasModified = checkAndRemove(*it->first,
+                        it->first->iterateNested(start, end_, handler),
+                        wasModified, keysToRemove);
+                // Don't iterate over values
+                /*wasModified = checkAndRemove(*it->second,
+                        it->second->iterateNested(start, end_, handler),
+                        wasModified, keysToRemove);*/
+            }
+            break;
+        }
+    } else {
+        LOG(spam, "at end of field path");
+        IteratorHandler::ModificationStatus
+            status = handler.modify(const_cast<FieldValue&>(complexFieldValue));
+
+        if (status == IteratorHandler::REMOVED) {
+            LOG(spam, "status = REMOVED");
+            return status;
+        } else if (status == IteratorHandler::MODIFIED) {
+            LOG(spam, "status = MODIFIED");
+            wasModified = true;
+        }
+
+        if (handler.handleComplex(complexFieldValue)) {
+            LOG(spam, "calling handler.handleComplex for all map keys");
+            for (const_iterator it(begin()), mt(end()); it != mt; it++) {
+                if (isWSet) {
+                    handler.setWeight(static_cast<const IntFieldValue &>(*it->second).getValue());
+                }
+                wasModified = checkAndRemove(*it->first,
+                        it->first->iterateNested(start, end_, handler),
+                        wasModified, keysToRemove);
+                // XXX: Map value iteration is currently disabled since it changes
+                // existing search behavior
+                /*wasModified = checkAndRemove(*it->second,
+                        it->second->iterateNested(start, end_, handler),
+                        wasModified, keysToRemove);*/
+            }
+        }
+    }
+    handler.setWeight(1);
+    for (std::vector<const FieldValue*>::iterator
+             i = keysToRemove.begin(), last = keysToRemove.end();
+         i != last; ++i)
+    {
+        LOG(spam, "erasing map entry with key %s", (*i)->toString().c_str());
+        const_cast<MapFieldValue&>(*this).erase(**i);
+    }
+    return wasModified ? IteratorHandler::MODIFIED : IteratorHandler::NOT_MODIFIED;
+}
+
+FieldValue::IteratorHandler::ModificationStatus
+MapFieldValue::onIterateNested(FieldPath::const_iterator start,
+                               FieldPath::const_iterator end_,
+                               IteratorHandler & handler) const
+{
+    LOG(spam, "iterating over MapFieldValue");
+    return iterateNestedImpl(start, end_, handler, *this);
+}
+
+
+} // document
diff --git a/document/src/vespa/document/fieldvalue/mapfieldvalue.h b/document/src/vespa/document/fieldvalue/mapfieldvalue.h
new file mode 100644
index 00000000000..a137baede6d
--- /dev/null
+++ b/document/src/vespa/document/fieldvalue/mapfieldvalue.h
@@ -0,0 +1,159 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * \class document::MapFieldValue
+ * \ingroup fieldvalue
+ *
+ * \brief A fieldvalue containing fieldvalue <-> weight mappings.
+ */
+#pragma once
+
+#include <vespa/document/fieldvalue/fieldvalue.h>
+#include <vespa/document/datatype/mapdatatype.h>
+#include <vespa/vespalib/util/polymorphicarrays.h>
+
+namespace document {
+
+class MapFieldValue : public FieldValue
+{
+private:
+    typedef vespalib::IArrayT<FieldValue> IArray;
+    const MapDataType *_type;
+    IArray::UP _keys;
+    IArray::UP _values;
+    bool _altered;
+
+    virtual bool addValue(const FieldValue& fv);
+    virtual bool containsValue(const FieldValue& fv) const { return contains(fv); }
+    virtual bool removeValue(const FieldValue& fv) { return erase(fv); }
+    bool checkAndRemove(const FieldValue& key,
+                        FieldValue::IteratorHandler::ModificationStatus status,
+                        bool wasModified,
+                        std::vector<const FieldValue*>& keysToRemove) const;
+    virtual IteratorHandler::ModificationStatus onIterateNested(
+            FieldPath::const_iterator start, FieldPath::const_iterator end,
+            IteratorHandler & handler) const;
+    // Utility method to avoid constant explicit casting
+    const MapDataType& getMapType() const { return *_type; }
+
+    void verifyKey(const FieldValue & key) const __attribute__((noinline));
+    void verifyValue(const FieldValue & value) const __attribute__((noinline));
+public:
+    typedef std::unique_ptr<MapFieldValue> UP;
+    class iterator {
+        typedef std::pair<FieldValue *, FieldValue *> pair;
+    public:
+        iterator(MapFieldValue & map, size_t index) : _map(&map), _index(index) { }
+        bool operator == (const iterator & rhs) const { return _map == rhs._map && _index == rhs._index; }
+        bool operator != (const iterator & rhs) const { return _map != rhs._map || _index != rhs._index; }
+        iterator& operator++() { ++_index; return *this; }
+        iterator operator++(int) { iterator other(*this); ++_index; return other; }
+        const pair & operator * () const { setCurr(); return _current; }
+        const pair * operator -> () const { setCurr(); return &_current; }
+        size_t offset() const { return _index; }
+    private:
+        void setCurr() const {
+             _current.first  = &(*_map->_keys)[_index];
+             _current.second = &(*_map->_values)[_index];
+        }
+        MapFieldValue *_map;
+        size_t         _index;
+        mutable pair   _current;
+    };
+    class const_iterator {
+        typedef std::pair<const FieldValue *, const FieldValue *> pair;
+    public:
+        const_iterator(const MapFieldValue & map, size_t index) : _map(&map), _index(index) { }
+        bool operator == (const const_iterator & rhs) const { return _map == rhs._map && _index == rhs._index; }
+        bool operator != (const const_iterator & rhs) const { return _map != rhs._map || _index != rhs._index; }
+        const_iterator& operator++() { ++_index; return *this; }
+        const_iterator operator++(int) { const_iterator other(*this); ++_index; return other; }
+        const pair & operator * () const { setCurr(); return _current; }
+        const pair * operator -> () const { setCurr(); return &_current; }
+    private:
+        void setCurr() const {
+             _current.first  = &(*_map->_keys)[_index];
+             _current.second = &(*_map->_values)[_index];
+        }
+        const MapFieldValue *_map;
+        size_t               _index;
+        mutable pair   _current;
+    };
+
+    MapFieldValue(const DataType &mapType);
+    virtual ~MapFieldValue();
+
+    MapFieldValue(const MapFieldValue & rhs);
+    MapFieldValue & operator = (const MapFieldValue & rhs);
+    void swap(MapFieldValue & rhs) {
+        std::swap(_keys, rhs._keys);
+        std::swap(_values, rhs._values);
+        std::swap(_altered, rhs._altered);
+        std::swap(_type, rhs._type);
+    }
+
+    void accept(FieldValueVisitor &visitor) override { visitor.visit(*this); }
+    void accept(ConstFieldValueVisitor &visitor) const override { visitor.visit(*this); }
+
+    /**
+     * These methods for insertion will check for uniqueness.
+     **/
+    bool put(FieldValue::UP key, FieldValue::UP value);
+    bool put(const FieldValue& key, const FieldValue& value);
+    bool insertVerify(const FieldValue& key, const FieldValue& value);
+    bool insert(FieldValue::UP key, FieldValue::UP value);
+
+    /**
+     * This will just append the values to the set, assuming that the
+     * new entry is not a duplicate.
+     **/
+    void push_back(FieldValue::UP key, FieldValue::UP value);
+    void push_back(const FieldValue & key, const FieldValue & value);
+
+    FieldValue::UP get(const FieldValue& key) const;
+    bool erase(const FieldValue& key);
+    bool contains(const FieldValue& key) const;
+
+    // CollectionFieldValue methods kept for compatability's sake
+    virtual bool isEmpty() const { return _keys->empty(); }
+    virtual size_t size() const { return _keys->size(); }
+    virtual void clear() { _keys->clear(); _values->clear(); }
+    void reserve(size_t sz) { _keys->reserve(sz); _values->reserve(sz); }
+    void resize(size_t sz) { _keys->resize(sz); _values->resize(sz); }
+
+    IteratorHandler::ModificationStatus iterateNestedImpl(
+            FieldPath::const_iterator start,
+            FieldPath::const_iterator end_,
+            IteratorHandler & handler,
+            const FieldValue& complexFieldValue) const;
+
+    // FieldValue implementation
+    virtual FieldValue& assign(const FieldValue&);
+    virtual MapFieldValue* clone() const { return new MapFieldValue(*this); }
+    virtual int compare(const FieldValue&) const;
+    virtual void print(std::ostream& out, bool verbose, const std::string& indent) const;
+    virtual bool hasChanged() const;
+    virtual const DataType *getDataType() const { return _type; }
+
+    virtual void printXml(XmlOutputStream& out) const;
+
+    const_iterator begin() const { return const_iterator(*this, 0); }
+    iterator begin() { return iterator(*this, 0); }
+
+    const_iterator end() const { return const_iterator(*this, size()); }
+    iterator end() { return iterator(*this, size()); }
+
+    const_iterator find(const FieldValue& fv) const;
+    iterator find(const FieldValue& fv);
+
+    FieldValue::UP createKey() const {
+        return getMapType().getKeyType().createFieldValue();
+    }
+    FieldValue::UP createValue() const {
+        return getMapType().getValueType().createFieldValue();
+    }
+
+    DECLARE_IDENTIFIABLE_ABSTRACT(MapFieldValue);
+};
+
+}  // namespace document
+
diff --git a/document/src/vespa/document/fieldvalue/numericfieldvalue.cpp b/document/src/vespa/document/fieldvalue/numericfieldvalue.cpp
new file mode 100644
index 00000000000..0e38b5c35f7
--- /dev/null
+++ b/document/src/vespa/document/fieldvalue/numericfieldvalue.cpp
@@ -0,0 +1,16 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/document/fieldvalue/numericfieldvalue.h>
+
+namespace document {
+
+IMPLEMENT_IDENTIFIABLE_ABSTRACT(NumericFieldValueBase, FieldValue);
+
+void
+NumericFieldValueBase::printXml(XmlOutputStream& out) const
+{
+    out << XmlContent(getAsString());
+}
+
+
+} // document
diff --git a/document/src/vespa/document/fieldvalue/numericfieldvalue.h b/document/src/vespa/document/fieldvalue/numericfieldvalue.h
new file mode 100644
index 00000000000..8636d015816
--- /dev/null
+++ b/document/src/vespa/document/fieldvalue/numericfieldvalue.h
@@ -0,0 +1,249 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * \class document::NumericFieldValue
+ * \ingroup fieldvalue
+ *
+ * \brief Templated parent class for field values of numbers.
+ *
+ * To prevent code duplication, numeric values are implemented using this
+ * template.
+ */
+#pragma once
+
+#include <boost/cast.hpp>
+#include <boost/lexical_cast.hpp>
+#include <boost/numeric/conversion/cast.hpp>
+#include <vespa/document/fieldvalue/fieldvalue.h>
+#include <vespa/document/util/bytebuffer.h>
+#include <vespa/vespalib/stllike/asciistream.h>
+#include <vespa/vespalib/stllike/lexical_cast.h>
+
+namespace document {
+
+class NumericFieldValueBase : public FieldValue
+{
+public:
+    DECLARE_IDENTIFIABLE_ABSTRACT(NumericFieldValueBase);
+    virtual void printXml(XmlOutputStream& out) const;
+};
+
+template<typename Number>
+class NumericFieldValue : public NumericFieldValueBase {
+protected:
+    Number _value;
+    bool _altered;
+
+public:
+    typedef Number value_type;
+
+    NumericFieldValue(Number value=0) : NumericFieldValueBase(), _value(value) { }
+
+    value_type getValue() const { return _value; }
+    void setValue(Number newValue) { _value = newValue; }
+
+        // FieldValue implementation.
+    virtual FieldValue& assign(const FieldValue&);
+    virtual int compare(const FieldValue& other) const;
+
+    virtual FieldValue& operator=(const vespalib::stringref &);
+    virtual FieldValue& operator=(int32_t);
+    virtual FieldValue& operator=(int64_t);
+    virtual FieldValue& operator=(float);
+    virtual FieldValue& operator=(double);
+    virtual size_t hash() const { return vespalib::hash<Number>()(_value); }
+
+    virtual char getAsByte() const;
+    virtual int32_t getAsInt() const;
+    virtual int64_t getAsLong() const;
+    virtual float getAsFloat() const;
+    virtual double getAsDouble() const;
+    virtual vespalib::string getAsString() const;
+
+    virtual void print(std::ostream& out, bool verbose,
+                       const std::string& indent) const;
+    virtual bool hasChanged() const { return _altered; }
+};
+
+template<typename Number>
+FieldValue&
+NumericFieldValue<Number>::assign(const FieldValue& value)
+{
+    if        (value.getClass().id() == IDENTIFIABLE_CLASSID(ByteFieldValue)) {
+        _value = static_cast<Number>(value.getAsByte());
+    } else if (value.getClass().id() == IDENTIFIABLE_CLASSID(ShortFieldValue)) {
+        _value = static_cast<Number>(value.getAsInt());
+    } else if (value.getClass().id() == IDENTIFIABLE_CLASSID(IntFieldValue)) {
+        _value = static_cast<Number>(value.getAsInt());
+    } else if (value.getClass().id() == IDENTIFIABLE_CLASSID(LongFieldValue)) {
+        _value = static_cast<Number>(value.getAsLong());
+    } else if (value.getClass().id() == IDENTIFIABLE_CLASSID(FloatFieldValue)) {
+        _value = static_cast<Number>(value.getAsFloat());
+    } else if (value.getClass().id() == IDENTIFIABLE_CLASSID(DoubleFieldValue))
+    {
+        _value = static_cast<Number>(value.getAsDouble());
+    } else {
+        return FieldValue::assign(value);
+    }
+    _altered = true;
+    return *this;
+}
+
+template<typename Number>
+int
+NumericFieldValue<Number>::compare( const FieldValue& other) const
+{
+    int diff = FieldValue::compare(other);
+    if (diff != 0) return diff;
+
+    const NumericFieldValue & otherNumber(static_cast<const NumericFieldValue &>(other));
+    return  (_value == otherNumber._value)
+                ? 0
+                : (_value - otherNumber._value > 0)
+                    ? 1
+                    : -1;
+}
+
+template<typename Number>
+void
+NumericFieldValue<Number>::print(
+        std::ostream& out, bool, const std::string&) const
+{
+    if (sizeof(Number) == 1) { // Make sure char's are printed as numbers
+        out << (int) _value;
+    } else {
+        out << _value;
+    }
+}
+
+template<typename Number>
+FieldValue&
+NumericFieldValue<Number>::operator=(const vespalib::stringref & value)
+{
+        // Lexical cast doesn't allow hex syntax we use in XML,
+        // so detect these in front.
+    if ((value.size() > 2) && (value[0] == '0') && ((value[1] | 0x20) == 'x')) {
+        char* endp;
+            // It is safe to assume that all hex numbers can be contained within
+            // 64 bit unsigned value.
+        unsigned long long val = strtoull(value.c_str(), &endp, 16);
+        if (*endp == '\0') {
+                // Allow numbers to be specified in range max signed to max
+                // unsigned. These become negative numbers.
+            _value = static_cast<Number>(val);
+            _altered = true;
+            return *this;
+        }
+    }
+    if (sizeof(Number) == sizeof(int8_t)) {
+        int val = vespalib::lexical_cast<int>(value);
+        if (val < -128 || val > 255) {
+            throw vespalib::IllegalArgumentException(
+                "Value of byte must be in the range -128 to 255", VESPA_STRLOC);
+        }
+        _value = static_cast<Number>(val);
+    } else {
+        try{
+            _value = boost::lexical_cast<Number>(value);
+        } catch (boost::bad_lexical_cast& e) {
+            // If bad cast is thrown due to value being bigger than max positive
+            // signed value, but less than max positive unsigned value,
+            // use this workaround to try to convert it to signed.
+            if (sizeof(Number) == sizeof(uint32_t)) {
+                _value = boost::numeric_cast<Number>(
+                        static_cast<int32_t>(boost::lexical_cast<uint32_t>(value)));
+            } else {
+                _value = boost::numeric_cast<Number>(
+                        static_cast<int64_t>(boost::lexical_cast<uint64_t>(value)));
+            }
+        }
+    }
+    _altered = true;
+    return *this;
+}
+
+template<typename Number>
+FieldValue&
+NumericFieldValue<Number>::operator=(int32_t value)
+{
+    _value = static_cast<Number>(value);
+    _altered = true;
+    return *this;
+}
+
+template<typename Number>
+FieldValue&
+NumericFieldValue<Number>::operator=(int64_t value)
+{
+    _value = static_cast<Number>(value);
+    _altered = true;
+    return *this;
+}
+
+template<typename Number>
+FieldValue&
+NumericFieldValue<Number>::operator=(float value)
+{
+    _value = static_cast<Number>(value);
+    _altered = true;
+    return *this;
+}
+
+template<typename Number>
+FieldValue&
+NumericFieldValue<Number>::operator=(double value)
+{
+    _value = static_cast<Number>(value);
+    _altered = true;
+    return *this;
+}
+
+template<typename Number>
+char
+NumericFieldValue<Number>::getAsByte() const
+{
+    return static_cast<char>(_value);
+}
+
+template<typename Number>
+int32_t
+NumericFieldValue<Number>::getAsInt() const
+{
+    return static_cast<int32_t>(_value);
+}
+
+template<typename Number>
+int64_t
+NumericFieldValue<Number>::getAsLong() const
+{
+    return static_cast<int64_t>(_value);
+}
+
+template<typename Number>
+float
+NumericFieldValue<Number>::getAsFloat() const
+{
+    return static_cast<float>(_value);
+}
+
+template<typename Number>
+double
+NumericFieldValue<Number>::getAsDouble() const
+{
+    return static_cast<double>(_value);
+}
+
+template<typename Number>
+vespalib::string
+NumericFieldValue<Number>::getAsString() const
+{
+    vespalib::asciistream ost;
+    if (sizeof(Number) == sizeof(uint8_t)) {
+        ost << static_cast<uint32_t>(_value);
+    } else {
+        ost << _value;
+    }
+    return ost.str();
+}
+
+} // document
+
diff --git a/document/src/vespa/document/fieldvalue/predicatefieldvalue.cpp b/document/src/vespa/document/fieldvalue/predicatefieldvalue.cpp
new file mode 100644
index 00000000000..1d1d0b03b2f
--- /dev/null
+++ b/document/src/vespa/document/fieldvalue/predicatefieldvalue.cpp
@@ -0,0 +1,71 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/log/log.h>
+LOG_SETUP(".predicatefieldvalue");
+#include <vespa/fastos/fastos.h>
+
+#include "predicatefieldvalue.h"
+#include <vespa/document/predicate/predicate.h>
+#include <vespa/document/predicate/predicate_printer.h>
+#include <vespa/vespalib/data/slime/inserter.h>
+
+using vespalib::Slime;
+using vespalib::slime::SlimeInserter;
+
+namespace document {
+
+IMPLEMENT_IDENTIFIABLE(PredicateFieldValue, FieldValue);
+
+PredicateFieldValue::PredicateFieldValue()
+    : _slime(std::make_unique<Slime>()), _altered(false) {
+}
+
+PredicateFieldValue::PredicateFieldValue(vespalib::Slime::UP s)
+    : FieldValue(),
+      _slime(std::move(s)),
+      _altered(false)
+{ }
+
+PredicateFieldValue::PredicateFieldValue(const PredicateFieldValue &rhs)
+    : FieldValue(rhs),
+      _slime(new Slime),
+      _altered(rhs._altered)
+{
+    inject(rhs._slime->get(), SlimeInserter(*_slime));
+}
+
+FieldValue &PredicateFieldValue::assign(const FieldValue &rhs) {
+    if (rhs.inherits(PredicateFieldValue::classId)) {
+        operator=(static_cast<const PredicateFieldValue &>(rhs));
+        return *this;
+    } else {
+        _slime.reset();
+        _altered = true;
+        return FieldValue::assign(rhs);
+    }
+}
+
+PredicateFieldValue &PredicateFieldValue::operator=(const PredicateFieldValue &rhs)
+{
+    _slime = std::make_unique<Slime>();
+    inject(rhs._slime->get(), SlimeInserter(*_slime));
+    _altered = true;
+    return *this;
+}
+
+int PredicateFieldValue::compare(const FieldValue&rhs) const {
+    int diff = FieldValue::compare(rhs);
+    if (diff != 0) return diff;
+    const PredicateFieldValue &o = static_cast<const PredicateFieldValue &>(rhs);
+    return Predicate::compare(*_slime, *o._slime);
+}
+
+void PredicateFieldValue::printXml(XmlOutputStream& out) const {
+    out << XmlContent(PredicatePrinter::print(*_slime));
+}
+
+void PredicateFieldValue::print(std::ostream& out, bool, const std::string&) const {
+    out << PredicatePrinter::print(*_slime) << "\n";
+}
+
+}  // namespace document
diff --git a/document/src/vespa/document/fieldvalue/predicatefieldvalue.h b/document/src/vespa/document/fieldvalue/predicatefieldvalue.h
new file mode 100644
index 00000000000..0e9bc226ab0
--- /dev/null
+++ b/document/src/vespa/document/fieldvalue/predicatefieldvalue.h
@@ -0,0 +1,43 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <memory>
+#include "fieldvalue.h"
+#include <vespa/vespalib/data/slime/slime.h>
+
+
+namespace document {
+
+class PredicateFieldValue : public FieldValue {
+    vespalib::Slime::UP _slime;
+    bool _altered;
+
+public:
+    PredicateFieldValue();
+    PredicateFieldValue(vespalib::Slime::UP s);
+    PredicateFieldValue(const PredicateFieldValue &rhs);
+
+    PredicateFieldValue &operator=(const PredicateFieldValue &rhs);
+
+    void accept(FieldValueVisitor &visitor) override { visitor.visit(*this); }
+    void accept(ConstFieldValueVisitor &visitor) const override { visitor.visit(*this); }
+
+    virtual FieldValue *clone() const override { return new PredicateFieldValue(*this); }
+    virtual int compare(const FieldValue &rhs) const override;
+
+    virtual void printXml(XmlOutputStream &out) const override;
+    virtual void print(std::ostream &out, bool verbose, const std::string &indent) const override;
+
+    virtual const DataType *getDataType() const override { return DataType::PREDICATE; }
+    virtual bool hasChanged() const override { return _altered; }
+
+    const vespalib::Slime &getSlime() const { return *_slime; }
+
+    virtual FieldValue &assign(const FieldValue &rhs) override;
+
+DECLARE_IDENTIFIABLE(PredicateFieldValue);
+};
+
+}  // namespace document
+
diff --git a/document/src/vespa/document/fieldvalue/rawfieldvalue.cpp b/document/src/vespa/document/fieldvalue/rawfieldvalue.cpp
new file mode 100644
index 00000000000..46731d16cb6
--- /dev/null
+++ b/document/src/vespa/document/fieldvalue/rawfieldvalue.cpp
@@ -0,0 +1,26 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/document/fieldvalue/rawfieldvalue.h>
+
+#include <vespa/document/util/stringutil.h>
+
+namespace document {
+
+IMPLEMENT_IDENTIFIABLE(RawFieldValue, LiteralFieldValueB);
+
+void
+RawFieldValue::printXml(XmlOutputStream& out) const
+{
+    out << XmlBase64Content()
+        << XmlContentWrapper(_value.c_str(), _value.size());
+}
+
+void
+RawFieldValue::print(std::ostream& out, bool, const std::string&) const
+{
+    StringUtil::printAsHex(out, _value.c_str(),
+                                _value.size());
+}
+
+} // document
diff --git a/document/src/vespa/document/fieldvalue/rawfieldvalue.h b/document/src/vespa/document/fieldvalue/rawfieldvalue.h
new file mode 100644
index 00000000000..e98d5a9de91
--- /dev/null
+++ b/document/src/vespa/document/fieldvalue/rawfieldvalue.h
@@ -0,0 +1,48 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * \class document::RawFieldValue
+ * \ingroup fieldvalue
+ *
+ * \brief Wrapper for raw field values.
+ */
+#pragma once
+
+#include <vespa/document/fieldvalue/literalfieldvalue.h>
+#include <memory>
+
+namespace document {
+
+class RawFieldValue
+        : public LiteralFieldValue<RawFieldValue, DataType::T_RAW, false>
+{
+public:
+    typedef LiteralFieldValue<RawFieldValue, DataType::T_RAW, false> Parent;
+
+    RawFieldValue()
+        : Parent() { }
+
+    RawFieldValue(const string& value)
+        : Parent(value) {}
+
+    RawFieldValue(const char* rawVal, int len)
+        : Parent(string(rawVal, len))
+    {
+    }
+
+    void accept(FieldValueVisitor &visitor) override { visitor.visit(*this); }
+    void accept(ConstFieldValueVisitor &visitor) const override { visitor.visit(*this); }
+
+    virtual RawFieldValue* clone() const
+        { return new RawFieldValue(*this); }
+    virtual void printXml(XmlOutputStream& out) const;
+    virtual void print(std::ostream& out, bool verbose,
+                       const std::string& indent) const;
+
+    RawFieldValue& operator=(const string& value)
+        { setValue(value); return *this; }
+
+    DECLARE_IDENTIFIABLE(RawFieldValue);
+};
+
+} // document
+
diff --git a/document/src/vespa/document/fieldvalue/serializablearray.cpp b/document/src/vespa/document/fieldvalue/serializablearray.cpp
new file mode 100644
index 00000000000..a2d92100dde
--- /dev/null
+++ b/document/src/vespa/document/fieldvalue/serializablearray.cpp
@@ -0,0 +1,223 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".document.serializable-array");
+
+#include <vespa/document/fieldvalue/serializablearray.h>
+
+#include <vespa/document/util/bytebuffer.h>
+#include <vespa/document/util/compressor.h>
+#include <vespa/vespalib/util/stringfmt.h>
+#include <algorithm>
+#include <string>
+#include <vector>
+
+using std::vector;
+
+namespace document {
+
+SerializableArray::Statistics SerializableArray::_stats;
+
+SerializableArray::SerializableArray()
+    : _serializedCompression(CompressionConfig::NONE),
+      _uncompressedLength(0)
+{
+}
+
+SerializableArray::SerializableArray(const SerializableArray& other)
+    : Cloneable(),
+      _entries(other._entries),
+      _owned(),
+      _uncompSerData(other._uncompSerData.get() ? new ByteBuffer(*other._uncompSerData) : NULL),
+      _compSerData(other._compSerData.get() ? new ByteBuffer(*other._compSerData) : NULL),
+      _serializedCompression(other._serializedCompression),
+      _uncompressedLength(other._uncompressedLength)
+{
+    for (size_t i(0); i < _entries.size(); i++) {
+        Entry & e(_entries[i]);
+        if (e.hasBuffer()) {
+            // Pointing to a buffer in the _owned structure.
+            ByteBuffer::UP buf(ByteBuffer::copyBuffer(e.getBuffer(_uncompSerData.get()), e.size()));
+            e.setBuffer(buf->getBuffer());
+            _owned[e.id()] = std::move(buf);
+        } else {
+            // If not it is relative to the buffer _uncompSerData, and hence it is valid as is.
+        }
+    }
+    if (_uncompSerData.get()) {
+        LOG_ASSERT(_uncompressedLength == _uncompSerData->getRemaining());
+    }
+}
+
+void
+SerializableArray::swap(SerializableArray& other)
+{
+    _entries.swap(other._entries);
+    _owned.swap(other._owned);
+    std::swap(_uncompSerData, other._uncompSerData);
+    std::swap(_compSerData, other._compSerData);
+    std::swap(_serializedCompression, other._serializedCompression);
+    std::swap(_uncompressedLength, other._uncompressedLength);
+}
+
+void SerializableArray::clear()
+{
+    _entries.clear();
+    _uncompSerData.reset();
+    _compSerData.reset();
+    _serializedCompression = CompressionConfig::NONE;
+    _uncompressedLength = 0;
+}
+
+SerializableArray::~SerializableArray()
+{
+}
+
+void
+SerializableArray::invalidate()
+{
+    _compSerData.reset();
+}
+
+void
+SerializableArray::set(int id, ByteBuffer::UP buffer)
+{
+    maybeDecompress();
+    Entry e(id, buffer->getRemaining(), buffer->getBuffer());
+    _owned[id] = std::move(buffer);
+    EntryMap::iterator it = find(id);
+    if (it == _entries.end()) {
+        _entries.push_back(e);
+    } else {
+        *it = e;
+    }
+    invalidate();
+}
+
+void SerializableArray::set(int id, const char* value, int len)
+{
+    set(id, std::unique_ptr<ByteBuffer>(ByteBuffer::copyBuffer(value,len)));
+}
+
+SerializableArray::EntryMap::const_iterator
+SerializableArray::find(int id) const
+{
+    return std::find_if(_entries.begin(), _entries.end(), [id](const auto& e){ return e.id() == id; });
+}
+
+SerializableArray::EntryMap::iterator
+SerializableArray::find(int id)
+{
+    return std::find_if(_entries.begin(), _entries.end(), [id](const auto& e){ return e.id() == id; });
+}
+
+bool
+SerializableArray::has(int id) const
+{
+    return (find(id) != _entries.end());
+}
+
+vespalib::ConstBufferRef
+SerializableArray::get(int id) const
+{
+    vespalib::ConstBufferRef buf;
+    if ( !maybeDecompressAndCatch() ) {
+        EntryMap::const_iterator found = find(id);
+
+        if (found != _entries.end()) {
+            const Entry& entry = *found;
+            buf = vespalib::ConstBufferRef(entry.getBuffer(_uncompSerData.get()), entry.size());
+        }
+    } else {
+        // should we clear all or what?
+    }
+
+    return buf;
+}
+
+bool
+SerializableArray::deCompressAndCatch() const
+{
+    try {
+        const_cast<SerializableArray *>(this)->deCompress();
+        return false;
+    } catch (const std::exception & e) {
+        LOG(warning, "Deserializing compressed content failed: %s", e.what());
+        return true;
+    }
+}
+
+void
+SerializableArray::clear(int id)
+{
+    maybeDecompress();
+    EntryMap::iterator it = find(id);
+    if (it != _entries.end()) {
+        _entries.erase(it);
+        _owned.erase(id);
+        invalidate();
+    }
+}
+
+void
+SerializableArray::deCompress() // throw (DeserializeException)
+{
+    // will only do this once
+
+    LOG_ASSERT(_compSerData);
+    LOG_ASSERT(!_uncompSerData);
+
+    if (_serializedCompression == CompressionConfig::NONE ||
+        _serializedCompression == CompressionConfig::UNCOMPRESSABLE)
+    {
+        _uncompSerData = std::move(_compSerData);
+        LOG_ASSERT(_uncompressedLength == _uncompSerData->getRemaining());
+    } else {
+        ByteBuffer::UP newSerialization(new ByteBuffer(_uncompressedLength));
+        vespalib::DataBuffer unCompressed(newSerialization->getBuffer(), newSerialization->getLength());
+        unCompressed.clear();
+        try {
+            decompress(_serializedCompression,
+                       _uncompressedLength,
+                       vespalib::ConstBufferRef(_compSerData->getBufferAtPos(), _compSerData->getRemaining()),
+                       unCompressed,
+                       false);
+        } catch (const std::runtime_error & e) {
+            throw DeserializeException(
+                vespalib::make_string( "Document was compressed with code unknown code %d", _serializedCompression),
+                VESPA_STRLOC);
+        }
+
+        if (unCompressed.getDataLen() != (size_t)_uncompressedLength) {
+            throw DeserializeException(
+                    vespalib::make_string(
+                            "Did not decompress to the expected length: had %" PRIu64 ", wanted %" PRId32 ", got %" PRIu64,
+                            _compSerData->getRemaining(), _uncompressedLength, unCompressed.getDataLen()),
+                    VESPA_STRLOC);
+        }
+        assert(newSerialization->getBuffer() == unCompressed.getData());
+        newSerialization->setLimit(_uncompressedLength);
+        _uncompSerData = std::move(newSerialization);
+        LOG_ASSERT(_uncompressedLength == _uncompSerData->getRemaining());
+    }
+}
+
+void SerializableArray::assign(EntryMap & entries,
+                               ByteBuffer::UP buffer,
+                               CompressionConfig::Type comp_type,
+                               uint32_t uncompressed_length)
+{
+    _serializedCompression = comp_type;
+
+    _entries.clear();
+    _entries.swap(entries);
+    if (CompressionConfig::isCompressed(_serializedCompression)) {
+        _compSerData.reset(buffer.release());
+        _uncompressedLength = uncompressed_length;
+    } else {
+        _uncompressedLength = buffer->getRemaining();
+        _uncompSerData.reset(buffer.release());
+    }
+}
+
+} // document
diff --git a/document/src/vespa/document/fieldvalue/serializablearray.h b/document/src/vespa/document/fieldvalue/serializablearray.h
new file mode 100644
index 00000000000..f1872195f74
--- /dev/null
+++ b/document/src/vespa/document/fieldvalue/serializablearray.h
@@ -0,0 +1,222 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * \class document::SerializableArray
+ * \brief key/value array that can be serialized and deserialized efficiently.
+ *
+ * The SerializableArray class is optimized for doing multiple
+ * serialize()/deserialize() without changing attributes. Once
+ * an attribute is changed, serialization is much slower. This makes
+ * sense, since a document travels between a lot of processes and
+ * queues, where nothing happens except serialization and deserialization.
+ *
+ * It also supports multiple deserializations, where serializations
+ * from multiple other arrays are merged into one array.
+ * Attributes that overlap Get the last known value.
+ */
+
+#pragma once
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/document/util/bytebuffer.h>
+#include <vespa/document/util/compressionconfig.h>
+#include <vespa/document/util/compressor.h>
+#include <vespa/document/util/serializable.h>
+#include <map>
+#include <set>
+#include <stdio.h>
+#include <vector>
+#include <vespa/vespalib/objects/cloneable.h>
+#include <vespa/vespalib/stllike/hash_map.h>
+#include <vespa/vespalib/util/buffer.h>
+
+namespace document
+{
+
+class SerializableArrayIterator;
+
+class SerializableArray : public vespalib::Cloneable
+{
+public:
+        // Counts set during serialization, in order to provide metrics for how
+        // often we use cached version, and how often we compress.
+    struct Statistics {
+        uint64_t _usedCachedSerializationCount;
+        uint64_t _compressedDocumentCount;
+        uint64_t _compressionDidntHelpCount;
+        uint64_t _uncompressableCount;
+        uint64_t _serializedUncompressed;
+        uint64_t _inputWronglySerialized;
+
+        Statistics()
+            : _usedCachedSerializationCount(0),
+              _compressedDocumentCount(0),
+              _compressionDidntHelpCount(0),
+              _uncompressableCount(0),
+              _serializedUncompressed(0),
+              _inputWronglySerialized(0) {}
+    };
+
+    /**
+     * Contains the id of a field, the size and a buffer reference that is either
+     * a relative offset to a common buffer, or the buffer itself it it is not.
+     * The most significant bit of the _sz member indicates which of the 2 it is.
+     */
+    class Entry {
+    public:
+        Entry() : _id(0), _sz(0), _data() {}
+        Entry(int i) : _id(i), _sz(0), _data()  {}
+        Entry(uint32_t i, uint32_t sz, uint32_t off) : _id(i), _sz(sz), _data(off) {}
+        Entry(uint32_t i, uint32_t sz, const char * buf) : _id(i), _sz(sz | BUFFER_MASK), _data(buf) {}
+
+        int32_t id() const { return _id; }
+        uint32_t size() const { return _sz & ~BUFFER_MASK; }
+        bool hasBuffer() const { return (_sz & BUFFER_MASK); }
+        bool operator < (const Entry & e) const { return cmp(e) < 0; }
+        int cmp(const Entry & e) const { return _id - e._id; }
+        void setBuffer(const char * buffer) { _data._buffer = buffer; _sz |= BUFFER_MASK; }
+        const char * getBuffer(const ByteBuffer * readOnlyBuffer) const { return hasBuffer() ? _data._buffer : readOnlyBuffer->getBuffer() + getOffset(); }
+    private:
+        uint32_t getOffset() const { return _data._offset; }
+        enum { BUFFER_MASK=0x80000000 };
+        int32_t      _id;
+        uint32_t     _sz;
+        union Data {
+           Data() : _buffer(0) { }
+           Data(const char * buffer) : _buffer(buffer) { }
+           Data(uint32_t offset) : _offset(offset) { }
+           const char * _buffer;
+           uint32_t     _offset;
+        } _data;
+    };
+    class EntryMap : public std::vector<Entry>
+    {
+    private:
+        using V=std::vector<Entry>;
+    public:
+        EntryMap() : V() { }
+    };
+
+    static const uint32_t ReservedId = 100;
+    static const uint32_t ReservedIdUpper = 128;
+
+
+private:
+    static Statistics _stats;
+
+    typedef vespalib::hash_map<int, uint32_t> HashMap;
+
+public:
+    static Statistics& getStatistics() { return _stats; }
+    typedef vespalib::LinkedPtr<SerializableArray> LP;
+    typedef vespalib::CloneablePtr<SerializableArray> CP;
+    typedef std::unique_ptr<SerializableArray> UP;
+
+    SerializableArray();
+    virtual ~SerializableArray();
+
+    void swap(SerializableArray& other);
+
+    /**
+     * Stores a value in the array.
+     *
+     * @param id The ID to associate the value with.
+     * @param value The value to store.
+     * @param len The length of the buffer.
+     */
+    void set(int id, const char* value, int len);
+
+    /** Stores a value in the array. */
+    void set(int id, std::unique_ptr<ByteBuffer> buffer);
+
+    /**
+     * Gets a value from the array. This is the faster version of the above.
+     * It will just give you the pointers needed. No refcounting or anything.
+     *
+     * @param id The ID of the value to Get.
+     *
+     * @return Returns a reference to a buffer. c_str and size will be zero if
+     * none is found.
+     */
+    vespalib::ConstBufferRef get(int id) const;
+
+    /** @return Returns true if the given ID is Set in the array. */
+    bool has(int id) const;
+
+    /** @return Number of elements in array */
+    bool hasAnyElems() const { return !_entries.empty(); }
+
+    /**
+     * clears an attribute.
+     *
+     * @param id The ID of the attribute to remove from the array.
+     */
+    void clear(int id);
+
+    /** Deletes all stored attributes. */
+    void clear();
+
+    CompressionConfig::Type getCompression() const { return _serializedCompression; }
+    CompressionInfo getCompressionInfo() const {
+        return CompressionInfo(_uncompressedLength, _compSerData->getRemaining());
+    }
+
+    /**
+     * Sets the serialized data that is the basis for this object's
+     * content. This is used by deserialization. Any existing entries
+     * are cleared.
+     */
+    VESPA_DLL_LOCAL void assign(EntryMap &entries,
+                                ByteBuffer::UP buffer,
+                                CompressionConfig::Type comp_type,
+                                uint32_t uncompressed_length);
+
+    bool empty() const { return _entries.empty(); }
+
+    const ByteBuffer* getSerializedBuffer() const {
+        return CompressionConfig::isCompressed(_serializedCompression)
+            ? _compSerData.get()
+            : _uncompSerData.get();
+    }
+
+    SerializableArray* clone() const override { return new SerializableArray(*this); }
+    SerializableArray(const SerializableArray&); // Public only for test
+    SerializableArray& operator=(const SerializableArray&) = delete;
+    const EntryMap & getEntries() const { return _entries; }
+private:
+    bool shouldDecompress() const {
+        return _compSerData.get() && !_uncompSerData.get();
+    }
+    bool maybeDecompressAndCatch() const {
+        if ( shouldDecompress() ) {
+            return deCompressAndCatch();
+        }
+        return false;
+    }
+
+    VESPA_DLL_LOCAL bool deCompressAndCatch() const;
+    void maybeDecompress() const {
+        if ( shouldDecompress() ) {
+            const_cast<SerializableArray *>(this)->deCompress();
+        }
+    }
+    VESPA_DLL_LOCAL void deCompress(); // throw (DeserializeException);
+
+    /** Contains the stored attributes, with reference to the real data.. */
+    EntryMap _entries;
+    /** The buffers we own. */
+    vespalib::hash_map<int, ByteBuffer::UP > _owned;
+
+    /** Data we deserialized from, if applicable. */
+    ByteBuffer::UP           _uncompSerData;
+    ByteBuffer::UP           _compSerData;
+    CompressionConfig::Type  _serializedCompression;
+
+    uint32_t     _uncompressedLength;
+
+    VESPA_DLL_LOCAL void invalidate();
+    VESPA_DLL_LOCAL EntryMap::const_iterator find(int id) const;
+    VESPA_DLL_LOCAL EntryMap::iterator find(int id);
+};
+
+} // document
+
diff --git a/document/src/vespa/document/fieldvalue/shortfieldvalue.cpp b/document/src/vespa/document/fieldvalue/shortfieldvalue.cpp
new file mode 100644
index 00000000000..465233719de
--- /dev/null
+++ b/document/src/vespa/document/fieldvalue/shortfieldvalue.cpp
@@ -0,0 +1,10 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/document/fieldvalue/shortfieldvalue.h>
+
+namespace document {
+
+IMPLEMENT_IDENTIFIABLE(ShortFieldValue, NumericFieldValueBase);
+
+} // document
diff --git a/document/src/vespa/document/fieldvalue/shortfieldvalue.h b/document/src/vespa/document/fieldvalue/shortfieldvalue.h
new file mode 100644
index 00000000000..329e33d0189
--- /dev/null
+++ b/document/src/vespa/document/fieldvalue/shortfieldvalue.h
@@ -0,0 +1,37 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * \class document::ShortFieldValue
+ * \ingroup fieldvalue
+ *
+ * \brief Wrapper for field values of datatype SHORT.
+ */
+#pragma once
+
+#include <vespa/document/datatype/numericdatatype.h>
+#include <vespa/document/fieldvalue/numericfieldvalue.h>
+
+namespace document {
+
+class ShortFieldValue : public NumericFieldValue<int16_t> {
+public:
+    typedef std::unique_ptr<ShortFieldValue> UP;
+    typedef int16_t Number;
+
+    ShortFieldValue(Number value = 0)
+        : NumericFieldValue<Number>(value) {}
+
+    void accept(FieldValueVisitor &visitor) override { visitor.visit(*this); }
+    void accept(ConstFieldValueVisitor &visitor) const override { visitor.visit(*this); }
+
+    virtual const DataType *getDataType() const { return DataType::SHORT; }
+
+    virtual ShortFieldValue* clone() const { return new ShortFieldValue(*this); }
+
+    using NumericFieldValue<Number>::operator=;
+
+    DECLARE_IDENTIFIABLE(ShortFieldValue);
+
+};
+
+} // document
+
diff --git a/document/src/vespa/document/fieldvalue/stringfieldvalue.cpp b/document/src/vespa/document/fieldvalue/stringfieldvalue.cpp
new file mode 100644
index 00000000000..cb705def1aa
--- /dev/null
+++ b/document/src/vespa/document/fieldvalue/stringfieldvalue.cpp
@@ -0,0 +1,149 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/document/fieldvalue/stringfieldvalue.h>
+
+#include <vespa/document/annotation/spantree.h>
+#include <vespa/document/serialization/annotationserializer.h>
+#include <vespa/vespalib/objects/nbostream.h>
+#include <vespa/document/serialization/util.h>
+#include <vespa/document/serialization/annotationdeserializer.h>
+#include <vespa/document/repo/fixedtyperepo.h>
+#include <vespa/document/serialization/vespadocumentserializer.h>
+
+using vespalib::nbostream;
+using vespalib::ConstBufferRef;
+using vespalib::stringref;
+
+namespace document {
+
+IMPLEMENT_IDENTIFIABLE(StringFieldValue, LiteralFieldValueB);
+
+StringFieldValue::StringFieldValue(const StringFieldValue & rhs) :
+    Parent(rhs),
+    _annotationData(rhs.copyAnnotationData())
+{
+}
+
+StringFieldValue & StringFieldValue::operator=(const StringFieldValue & rhs)
+{
+    if (&rhs != this) {
+        Parent::operator=(rhs);
+        _annotationData = rhs.copyAnnotationData();
+    }
+    return *this;
+}
+
+int StringFieldValue::compare(const FieldValue& other) const {
+    if (other.inherits(StringFieldValue::classId)) {
+        const StringFieldValue &other_s(static_cast<const StringFieldValue &>(other));
+        return _value.compare(other_s._value);
+    } else {
+        return Parent::compare(other);
+    }
+}
+
+void StringFieldValue::print(std::ostream& out, bool verbose, const std::string& indent) const {
+    if ( ! hasSpanTrees()) {
+        Parent::print(out, verbose, indent);
+    } else {
+        out << "StringFieldValue(\"";
+        Parent::print(out, verbose, indent);
+        ConstBufferRef buf = getSerializedAnnotations();
+        out << "\"\n" << indent << " " << vespalib::HexDump(buf.data(), buf.size());
+        out << ")";
+    }
+}
+
+void StringFieldValue::setSpanTrees(ConstBufferRef serialized, const FixedTypeRepo & repo, uint8_t version, bool isSerializedDataLongLived)
+{
+    _annotationData = std::make_unique<AnnotationData>(serialized, repo, version, isSerializedDataLongLived);
+}
+
+
+void StringFieldValue::setSpanTrees(const SpanTrees & trees, const FixedTypeRepo & repo)
+{
+    nbostream os;
+    putInt1_2_4Bytes(os, trees.size());
+    AnnotationSerializer serializer(os);
+    for (const auto & tree : trees) {
+        serializer.write(*tree);
+    }
+    setSpanTrees(ConstBufferRef(os.peek(), os.size()), repo, VespaDocumentSerializer::getCurrentVersion(), false);
+}
+StringFieldValue::SpanTrees StringFieldValue::getSpanTrees() const {
+    SpanTrees trees;
+    if (hasSpanTrees()) {
+        trees = _annotationData->getSpanTrees();
+    }
+    return trees;
+}
+
+const SpanTree * StringFieldValue::findTree(const SpanTrees & trees, const stringref & name)
+{
+    for(const auto & tree : trees) {
+        if (tree->getName() == name) {
+            return tree.get();
+        }
+    }
+    return nullptr;
+}
+
+StringFieldValue &StringFieldValue::operator=(const stringref& value)
+{
+    setValue(value);
+    _annotationData.reset();
+    return *this;
+}
+
+FieldValue & StringFieldValue::assign(const FieldValue & rhs)
+{
+    if (rhs.inherits(StringFieldValue::classId)) {
+        *this = static_cast<const StringFieldValue &>(rhs);
+    } else {
+        *this = static_cast<const stringref &>(rhs.getAsString());
+    }
+    return *this;
+}
+
+StringFieldValue::AnnotationData::UP
+StringFieldValue::copyAnnotationData() const {
+    return hasSpanTrees()
+           ? std::make_unique<AnnotationData>(*_annotationData)
+           : AnnotationData::UP();
+}
+
+StringFieldValue::AnnotationData::AnnotationData(vespalib::ConstBufferRef serialized, const FixedTypeRepo &repo,
+                                                 uint8_t version, bool isSerializedDataLongLived)
+        : _serialized(serialized),
+          _repo(repo.getDocumentTypeRepo()),
+          _docType(repo.getDocumentType()),
+          _version(version)
+{
+    if ( ! isSerializedDataLongLived) {
+        _backingBlob.assign(serialized.c_str(), serialized.c_str() + serialized.size());
+        _serialized = ConstBufferRef(&_backingBlob[0], _backingBlob.size());
+    }
+}
+
+StringFieldValue::SpanTrees StringFieldValue::AnnotationData::getSpanTrees() const
+{
+    SpanTrees trees;
+    if (hasSpanTrees()) {
+        nbostream is(_serialized.data(), _serialized.size());
+        size_t tree_count = getInt1_2_4Bytes(is);
+        FixedTypeRepo repo(_repo, _docType);
+        AnnotationDeserializer deserializer(repo, is, _version);
+        for (size_t i = 0; i < tree_count; ++i) {
+            trees.emplace_back(deserializer.readSpanTree());
+        }
+    }
+    return trees;
+}
+
+StringFieldValue::AnnotationData::AnnotationData(const StringFieldValue::AnnotationData & rhs) :
+        AnnotationData(rhs._serialized, FixedTypeRepo(rhs._repo, rhs._docType), rhs._version, false)
+{
+}
+
+} // document
diff --git a/document/src/vespa/document/fieldvalue/stringfieldvalue.h b/document/src/vespa/document/fieldvalue/stringfieldvalue.h
new file mode 100644
index 00000000000..6344c0bd5b9
--- /dev/null
+++ b/document/src/vespa/document/fieldvalue/stringfieldvalue.h
@@ -0,0 +1,76 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * \class document::StringFieldValue
+ * \ingroup fieldvalue
+ *
+ * \brief Wrapper for string field values.
+ */
+#pragma once
+
+#include <vespa/document/fieldvalue/literalfieldvalue.h>
+#include <vespa/document/annotation/spantree.h>
+#include <vespa/vespalib/stllike/hash_map.h>
+#include <vespa/document/repo/fixedtyperepo.h>
+
+namespace document {
+class FixedTypeRepo;
+
+class StringFieldValue : public LiteralFieldValue<StringFieldValue, DataType::T_STRING, true> {
+public:
+    typedef LiteralFieldValue<StringFieldValue, DataType::T_STRING, true> Parent;
+    typedef std::vector<SpanTree::UP> SpanTrees;
+
+    StringFieldValue() : Parent(), _annotationData() { }
+    StringFieldValue(const string &value)
+            : Parent(value), _annotationData() { }
+
+    StringFieldValue(const StringFieldValue &rhs);
+
+    StringFieldValue &operator=(const StringFieldValue &rhs);
+    StringFieldValue &operator=(const vespalib::stringref &value) override;
+
+    FieldValue &assign(const FieldValue &) override;
+
+    void accept(FieldValueVisitor &visitor) override { visitor.visit(*this); }
+    void accept(ConstFieldValueVisitor &visitor) const override { visitor.visit(*this); }
+    StringFieldValue *clone() const override { return new StringFieldValue(*this); }
+    int compare(const FieldValue &other) const override;
+    void print(std::ostream &out, bool verbose, const std::string &indent) const override;
+    void setSpanTrees(vespalib::ConstBufferRef serialized, const FixedTypeRepo &repo, uint8_t version,
+                      bool isSerializedDataLongLived);
+    void setSpanTrees(const SpanTrees &trees, const FixedTypeRepo &repo);
+    SpanTrees getSpanTrees() const;
+    vespalib::ConstBufferRef getSerializedAnnotations() const {
+        return _annotationData ? _annotationData->getSerializedAnnotations() : vespalib::ConstBufferRef();
+    }
+    bool hasSpanTrees() const { return _annotationData ? _annotationData->hasSpanTrees() : false; }
+    static const SpanTree *findTree(const SpanTrees &trees, const vespalib::stringref &name);
+
+    using LiteralFieldValueB::operator=;
+    DECLARE_IDENTIFIABLE(StringFieldValue);
+private:
+
+    class AnnotationData {
+    public:
+        typedef vespalib::nbostream::Buffer BackingBlob;
+        typedef std::unique_ptr<AnnotationData> UP;
+        VESPA_DLL_LOCAL AnnotationData(const AnnotationData & rhs);
+        VESPA_DLL_LOCAL AnnotationData(vespalib::ConstBufferRef serialized, const FixedTypeRepo &repo,
+                                       uint8_t version, bool isSerializedDataLongLived);
+
+        bool hasSpanTrees() const { return _serialized.size() > 0u; }
+        vespalib::ConstBufferRef getSerializedAnnotations() const { return _serialized; }
+        VESPA_DLL_LOCAL SpanTrees getSpanTrees() const;
+    private:
+        vespalib::ConstBufferRef _serialized;
+        BackingBlob              _backingBlob;
+        const DocumentTypeRepo  &_repo;
+        const DocumentType      &_docType;
+        uint8_t                  _version;
+    };
+    VESPA_DLL_LOCAL AnnotationData::UP copyAnnotationData() const;
+    AnnotationData::UP _annotationData;
+};
+
+} // document
+
diff --git a/document/src/vespa/document/fieldvalue/structfieldvalue.cpp b/document/src/vespa/document/fieldvalue/structfieldvalue.cpp
new file mode 100644
index 00000000000..4c74c6d7dc7
--- /dev/null
+++ b/document/src/vespa/document/fieldvalue/structfieldvalue.cpp
@@ -0,0 +1,427 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".document.structfieldvalue");
+
+#include <vespa/document/fieldvalue/structfieldvalue.h>
+
+#include "fieldvaluewriter.h"
+#include <vespa/document/fieldvalue/document.h>
+#include <vespa/document/repo/fixedtyperepo.h>
+#include <vespa/document/serialization/vespadocumentdeserializer.h>
+#include <vespa/document/fieldset/fieldsets.h>
+#include <vespa/vespalib/objects/nbostream.h>
+#include <vespa/vespalib/util/crc.h>
+#include <vespa/document/datatype/positiondatatype.h>
+#include <vespa/vespalib/util/vstringfmt.h>
+
+using std::vector;
+using vespalib::nbostream;
+
+namespace document {
+
+IMPLEMENT_IDENTIFIABLE_ABSTRACT(StructFieldValue, StructuredFieldValue);
+
+StructFieldValue::StructFieldValue(const DataType &type)
+    : StructuredFieldValue(type),
+      _repo(NULL),
+      _doc_type(NULL),
+      _version(Document::getNewestSerializationVersion()),
+      _hasChanged(true)
+{
+    if (!type.getClass().inherits(StructDataType::classId)) {
+        throw vespalib::IllegalArgumentException(
+                "Cannot generate a struct value with non-struct type "
+                + type.toString() + ".", VESPA_STRLOC);
+    }
+}
+
+void
+StructFieldValue::swap(StructFieldValue & rhs)
+{
+    StructuredFieldValue::swap(rhs);
+    std::swap(_chunks, rhs._chunks);
+    std::swap(_hasChanged, rhs._hasChanged);
+    std::swap(_repo, rhs._repo);
+    std::swap(_doc_type, rhs._doc_type);
+    std::swap(_version, _version);
+}
+
+void
+StructFieldValue::lazyDeserialize(const FixedTypeRepo &repo,
+                                  uint16_t version,
+                                  SerializableArray::EntryMap && fm,
+                                  ByteBuffer::UP buffer,
+                                  CompressionConfig::Type comp_type,
+                                  int32_t uncompressed_length)
+{
+    _repo = &repo.getDocumentTypeRepo();
+    _doc_type = &repo.getDocumentType();
+    _version = version;
+
+    _chunks.push_back(SerializableArray::UP(new SerializableArray()));
+    _chunks.back().assign(fm, std::move(buffer), comp_type, uncompressed_length);
+    _hasChanged = false;
+}
+
+bool StructFieldValue::serializeField(int field_id, uint16_t version,
+                                      FieldValueWriter &writer) const {
+    if (version == _version) {
+        for (int i = _chunks.size() - 1; i >= 0; --i) {
+            vespalib::ConstBufferRef buf = _chunks[i].get(field_id);
+            if ( buf.size() != 0) {
+                writer.writeSerializedData(buf.data(), buf.size());
+                break;
+            }
+        }
+
+        return true;
+    }
+    try {
+        const Field &f = getStructType().getField(field_id, _version);
+        writer.writeFieldValue(*getFieldValue(f));
+        return true;
+    } catch (FieldNotFoundException &) {
+        LOG(info, "Dropping field %d when serializing to a newer version", field_id);
+        return false;
+    }
+}
+
+int StructFieldValue::fieldIdFromRawId(int raw_id, uint16_t version) const {
+    if (version != _version) {
+        return getStructType().getField(raw_id, _version).getId(version);
+    }
+    return raw_id;
+}
+
+void StructFieldValue::getRawFieldIds(vector<int> &raw_ids) const {
+    raw_ids.clear();
+
+    size_t count(0);
+    for (uint32_t i = 0; i < _chunks.size(); ++i) {
+        count += _chunks[i].getEntries().size();
+    }
+    raw_ids.reserve(count);
+    for (uint32_t i = 0; i < _chunks.size(); ++i) {
+        const SerializableArray::EntryMap & entries = _chunks[i].getEntries();
+        for (const SerializableArray::Entry & entry : entries) {
+            raw_ids.emplace_back(entry.id());
+        }
+    }
+    sort(raw_ids.begin(), raw_ids.end());
+    raw_ids.erase(unique(raw_ids.begin(), raw_ids.end()), raw_ids.end());
+}
+
+void
+StructFieldValue::getRawFieldIds(vector<int> &raw_ids,
+                                 const FieldSet& fieldSet) const {
+    raw_ids.clear();
+
+    for (uint32_t i = 0; i < _chunks.size(); ++i) {
+        const SerializableArray::EntryMap & entries = _chunks[i].getEntries();
+        for (const SerializableArray::Entry & entry : entries) {
+            if (fieldSet.contains(getStructType().getField(entry.id(), _version))) {
+                raw_ids.push_back(entry.id());
+            }
+        }
+    }
+    sort(raw_ids.begin(), raw_ids.end());
+    raw_ids.erase(unique(raw_ids.begin(), raw_ids.end()), raw_ids.end());
+}
+
+bool
+StructFieldValue::hasField(const vespalib::stringref & name) const
+{
+    return getStructType().hasField(name);
+}
+
+const Field&
+StructFieldValue::getField(const vespalib::stringref & name) const
+{
+    return getStructType().getField(name);
+}
+
+namespace {
+
+void
+createFV(FieldValue & value, const DocumentTypeRepo & repo, nbostream & stream, const DocumentType & doc_type, uint32_t version)
+{
+    FixedTypeRepo frepo(repo, doc_type);
+    VespaDocumentDeserializer deserializer(frepo, stream, version);
+    deserializer.read(value);
+}
+
+}
+
+FieldValue::UP
+StructFieldValue::getFieldValue(const Field& field) const
+{
+    int fieldId = field.getId(_version);
+
+    for (int i = _chunks.size() - 1; i >= 0; --i) {
+        vespalib::ConstBufferRef buf = _chunks[i].get(fieldId);
+        if (buf.size() != 0) {
+            nbostream stream(buf.c_str(), buf.size());
+            FieldValue::UP value(field.getDataType().createFieldValue());
+            if ((_repo == NULL) && (_doc_type != NULL)) {
+                DocumentTypeRepo::UP tmpRepo(new DocumentTypeRepo(*_doc_type));
+                createFV(*value, *tmpRepo, stream, *_doc_type, _version);
+            } else {
+                createFV(*value, *_repo, stream, *_doc_type, _version);
+            }
+            return value;
+        }
+    }
+    return FieldValue::UP();
+}
+
+vespalib::ConstBufferRef
+StructFieldValue::getRawField(uint32_t id) const
+{
+    for (int i = _chunks.size() - 1; i >= 0; --i) {
+        vespalib::ConstBufferRef buf = _chunks[i].get(id);
+        if (buf.size() > 0) {
+            return buf;
+        }
+    }
+
+    return vespalib::ConstBufferRef();
+}
+
+bool
+StructFieldValue::getFieldValue(const Field& field, FieldValue& value) const
+{
+    int fieldId = field.getId(_version);
+
+    vespalib::ConstBufferRef buf = getRawField(fieldId);
+    if (buf.size() > 0) {
+        nbostream stream(buf.c_str(), buf.size(), true);
+        if ((_repo == NULL) && (_doc_type != NULL)) {
+            DocumentTypeRepo::UP tmpRepo(new DocumentTypeRepo(*_doc_type));
+            createFV(value, *tmpRepo, stream, *_doc_type, _version);
+        } else {
+            createFV(value, *_repo, stream, *_doc_type, _version);
+        }
+        return true;
+    }
+    return false;
+}
+
+bool
+StructFieldValue::hasFieldValue(const Field& field) const
+{
+    for (int i = _chunks.size() - 1; i >= 0; --i) {
+        if (_chunks[i].has(field.getId(_version))) {
+            return true;
+        }
+    }
+
+    return false;
+}
+
+void
+StructFieldValue::setFieldValue(const Field& field, FieldValue::UP value)
+{
+    int fieldId = field.getId(_version);
+    std::unique_ptr<ByteBuffer> serialized(value->serialize());
+    if (serialized->getLength() >= 0x4000000) { // Max 64 MB fields.
+        throw SerializeException(vespalib::make_string(
+                        "Field value for field %i larger than 64 MB",
+                        fieldId), VESPA_STRLOC);
+    }
+    serialized->flip();
+    if (_chunks.empty()) {
+        _chunks.push_back(SerializableArray::UP(new SerializableArray()));
+    }
+
+    _chunks.back().set(fieldId, std::move(serialized));
+
+    _hasChanged = true;
+}
+
+void
+StructFieldValue::removeFieldValue(const Field& field)
+{
+    for (uint32_t i = 0; i < _chunks.size(); ++i) {
+        _chunks[i].clear(field.getId(_version));
+    }
+    _hasChanged = true;
+}
+
+void
+StructFieldValue::clear()
+{
+    _chunks.clear();
+    _hasChanged = true;
+}
+
+// FieldValue implementation.
+FieldValue&
+StructFieldValue::assign(const FieldValue& value)
+{
+    const StructFieldValue& other(dynamic_cast<const StructFieldValue&>(value));
+    return operator=(other);
+}
+
+int
+StructFieldValue::compare(const FieldValue& otherOrg) const
+{
+    int comp = StructuredFieldValue::compare(otherOrg);
+    if (comp != 0) {
+        return comp;
+    }
+    const StructFieldValue& other = static_cast<const StructFieldValue&>(otherOrg);
+
+    std::vector<int> a;
+    getRawFieldIds(a);
+    std::vector<int> b;
+    other.getRawFieldIds(b);
+
+    for (size_t i(0); i < std::min(a.size(), b.size()); i++) {
+        if (a[i] != b[i]) {
+            return a[i] < b[i] ? -1 : 1;
+        }
+        vespalib::ConstBufferRef ar = getRawField(a[i]);
+        vespalib::ConstBufferRef br = other.getRawField(a[i]);
+        comp = memcmp(ar.c_str(), br.c_str(), std::min(ar.size(), br.size()));
+        if (comp != 0) {
+            return comp;
+        }
+        if ( ar.size() != br.size()) {
+            return ar.size() < br.size() ? -1 : 1;
+        }
+    }
+    if (a.size() != b.size()) {
+        return a.size() < b.size() ? -1 : 1;
+    }
+    return 0;
+}
+
+uint32_t
+StructFieldValue::calculateChecksum() const
+{
+    ByteBuffer::UP buffer(serialize());
+    vespalib::crc_32_type calculator;
+    calculator.process_bytes(buffer->getBuffer(), buffer->getPos());
+    return calculator.checksum();
+}
+
+void
+StructFieldValue::printXml(XmlOutputStream& xos) const
+{
+    if (getType() == PositionDataType::getInstance()
+        && getFieldValue(getField(PositionDataType::FIELD_Y))
+        && getFieldValue(getField(PositionDataType::FIELD_X)))
+    {
+        double ns = getFieldValue(getField(PositionDataType::FIELD_Y))->getAsInt() / 1.0e6;
+        double ew = getFieldValue(getField(PositionDataType::FIELD_X))->getAsInt() / 1.0e6;
+        vespalib::string buf = vespalib::make_vespa_string("%s%.6f;%s%.6f",
+                (ns < 0 ? "S" : "N"),
+                (ns < 0 ? (-ns) : ns),
+                (ew < 0 ? "W" : "E"),
+                (ew < 0 ? (-ew) : ew));
+        xos << buf;
+        return;
+    }
+    for (const_iterator it = begin(); it != end(); ++it) {
+        xos << XmlTag(it.field().getName());
+        getValue(it.field())->printXml(xos);
+        xos << XmlEndTag();
+    }
+}
+
+void
+StructFieldValue::print(std::ostream& out, bool verbose,
+                        const std::string& indent) const
+{
+    out << "Struct " << getDataType()->getName() << "(";
+    int count = 0;
+    for (const_iterator it = begin(); it != end(); ++it) {
+        if (count++ != 0) {
+            out << ",";
+        }
+        out << "\n" << indent << "  " << it.field().getName() << " - ";
+        getValue(it.field())->print(out, verbose, indent + "  ");
+    }
+    if (count > 0) out << "\n" << indent;
+    out << ")";
+}
+
+bool
+StructFieldValue::empty() const
+{
+    for (uint32_t i = 0; i < _chunks.size(); ++i) {
+        if (!_chunks[i].empty()) {
+            return false;
+        }
+    }
+
+    return true;
+}
+
+void
+StructFieldValue::reset()
+{
+    clear();
+    _hasChanged = false;
+}
+
+struct StructFieldValue::FieldIterator : public StructuredIterator {
+    const StructFieldValue& _struct;
+    std::vector<int> _ids;
+    std::vector<int>::iterator _cur;
+    int _version;
+
+    FieldIterator(const StructFieldValue& s, int version)
+        : _struct(s),
+          _ids(),
+          _cur(_ids.begin()),
+          _version(version)
+    {
+        s.getRawFieldIds(_ids);
+        _cur = _ids.begin();
+    }
+
+    void skipTo(int fieldId) {
+        while (_cur != _ids.end() && fieldId != *_cur) {
+            ++_cur;
+        }
+    }
+
+    virtual const Field* getNextField() {
+        while (_cur != _ids.end()) {
+            int id = *_cur++;
+            try {
+                return &_struct.getStructType().getField(id, _version);
+            } catch (FieldNotFoundException& e) {
+                // Should not get this exception until after we've moved the iterator.
+                LOG(debug, "exception for id %d version %d", id, _version);
+                LOG(debug, "struct data type: %s", _struct.getType().toString(true).c_str());
+            }
+        }
+        return 0;
+    }
+};
+
+StructuredFieldValue::StructuredIterator::UP
+StructFieldValue::getIterator(const Field* toFind) const
+{
+    StructuredIterator::UP ret;
+
+    FieldIterator *fi = new FieldIterator(*this, _version);
+    ret.reset(fi);
+
+    if (toFind != NULL) {
+        fi->skipTo(toFind->getId(_version));
+    }
+    return ret;
+}
+
+void
+StructFieldValue::setType(const DataType& type)
+{
+    clear();
+    StructuredFieldValue::setType(type);
+}
+
+} // document
diff --git a/document/src/vespa/document/fieldvalue/structfieldvalue.h b/document/src/vespa/document/fieldvalue/structfieldvalue.h
new file mode 100644
index 00000000000..ce89208b249
--- /dev/null
+++ b/document/src/vespa/document/fieldvalue/structfieldvalue.h
@@ -0,0 +1,151 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * \class document::StructFieldValue
+ * \ingroup fieldvalue
+ *
+ * \brief Stores a set of predefined field <-> fieldvalue mappings.
+ */
+#pragma once
+
+    // Not strictly needed to include exceptions.h, but to avoid clients needing
+    // to to catch FieldNotFoundException
+#include <vespa/document/base/exceptions.h>
+#include <vespa/document/datatype/structdatatype.h>
+#include <vespa/document/fieldvalue/structuredfieldvalue.h>
+#include <vespa/document/util/compressionconfig.h>
+#include <vespa/document/fieldvalue/serializablearray.h>
+#include <vector>
+
+namespace document {
+class Document;
+class DocumentType;
+class DocumentTypeRepo;
+class FieldValueWriter;
+class FixedTypeRepo;
+class FieldSet;
+
+class StructFieldValue : public StructuredFieldValue
+{
+public:
+    class Chunks {
+    public:
+        Chunks() : _sz(0) { }
+        SerializableArray & operator [] (size_t i) { return *_chunks[i]; }
+        const SerializableArray & operator [] (size_t i) const { return *_chunks[i]; }
+        void push_back(SerializableArray::UP item) {
+            assert(_sz < 2);
+            _chunks[_sz++].reset(item.release());
+        }
+        SerializableArray & back() { return *_chunks[_sz-1]; }
+        const SerializableArray & back() const { return *_chunks[_sz-1]; }
+        size_t size() const { return _sz; }
+        bool empty() const { return _sz == 0; }
+        void clear() {
+            _chunks[0].reset();
+            _chunks[1].reset();
+            _sz = 0;
+        }
+        void swap(Chunks & rhs) {
+            _chunks[0].swap(rhs._chunks[0]);
+            _chunks[1].swap(rhs._chunks[1]);
+            std::swap(_sz, rhs._sz);
+        }
+    private:
+        SerializableArray::CP _chunks[2];
+        size_t _sz;
+    };
+private:
+    Chunks   _chunks;
+
+    // As we do lazy deserialization, we need these saved
+    const DocumentTypeRepo *_repo;
+    const DocumentType     *_doc_type;
+    uint16_t                _version;
+    mutable bool            _hasChanged;
+
+public:
+    typedef std::unique_ptr<StructFieldValue> UP;
+    typedef vespalib::LinkedPtr<StructFieldValue> LP;
+    StructFieldValue(const DataType &type);
+    void swap(StructFieldValue & rhs);
+
+    void setRepo(const DocumentTypeRepo & repo) { _repo = & repo; }
+    const DocumentTypeRepo * getRepo() const { return _repo; }
+    void setDocumentType(const DocumentType & docType) { _doc_type = & docType; }
+
+    const StructDataType & getStructType() const { return static_cast<const StructDataType &>(getType()); }
+
+    void lazyDeserialize(const FixedTypeRepo &repo,
+                         uint16_t version,
+                         SerializableArray::EntryMap && fields,
+                         ByteBuffer::UP buffer,
+                         CompressionConfig::Type comp_type,
+                         int32_t uncompressed_length);
+
+    // returns false if the field could not be serialized.
+    bool serializeField(int raw_field_id, uint16_t version, FieldValueWriter &writer) const;
+    int fieldIdFromRawId(int raw_field_id, uint16_t version) const;
+
+    uint16_t getVersion() const { return _version; }
+
+    const Chunks & getChunks() const {  return _chunks; }
+
+    // raw_ids may contain ids for elements not in the struct's datatype.
+    void getRawFieldIds(std::vector<int> &raw_ids) const;
+    void getRawFieldIds(std::vector<int> &raw_ids, const FieldSet& fieldSet) const;
+
+    void accept(FieldValueVisitor &visitor) override { visitor.visit(*this); }
+    void accept(ConstFieldValueVisitor &visitor) const override { visitor.visit(*this); }
+
+    bool hasField(const vespalib::stringref & name) const override;
+    const Field& getField(const vespalib::stringref & name) const override;
+    void clear() override;
+
+    const CompressionConfig &getCompressionConfig() const
+    { return getStructType().getCompressionConfig(); }
+
+    // FieldValue implementation.
+    FieldValue& assign(const FieldValue&) override;
+    int compare(const FieldValue& other) const override;
+    StructFieldValue* clone() const  override{
+        return new StructFieldValue(*this);
+    }
+
+    void printXml(XmlOutputStream& out) const override;
+    void print(std::ostream& out, bool verbose, const std::string& indent) const override;
+
+    bool empty() const;
+
+    bool hasChanged() const override { return _hasChanged; }
+
+    uint32_t calculateChecksum() const;
+
+    /**
+     * Called by document to reset struct when deserializing where this struct
+     * has no content. This clears content and sets changed to false.
+     */
+    void reset();
+
+    DECLARE_IDENTIFIABLE_ABSTRACT(StructFieldValue);
+
+private:
+    void setFieldValue(const Field&, FieldValue::UP value) override;
+    FieldValue::UP getFieldValue(const Field&) const override;
+    bool getFieldValue(const Field&, FieldValue&) const override;
+    bool hasFieldValue(const Field&) const override;
+    void removeFieldValue(const Field&) override;
+    VESPA_DLL_LOCAL vespalib::ConstBufferRef getRawField(uint32_t id) const;
+
+    // Iterator implementation
+    class FieldIterator;
+    friend class FieldIterator;
+
+    StructuredIterator::UP getIterator(const Field* toFind) const override;
+
+    /** Called from Document when deserializing alters type. */
+    void setType(const DataType& type) override;
+    friend class Document; // Hide from others to prevent misuse
+};
+
+} // document
+
diff --git a/document/src/vespa/document/fieldvalue/structuredfieldvalue.cpp b/document/src/vespa/document/fieldvalue/structuredfieldvalue.cpp
new file mode 100644
index 00000000000..2fb0b148140
--- /dev/null
+++ b/document/src/vespa/document/fieldvalue/structuredfieldvalue.cpp
@@ -0,0 +1,176 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/document/fieldvalue/structuredfieldvalue.h>
+
+#include <vespa/document/base/field.h>
+#include <vespa/document/fieldvalue/fieldvalues.h>
+#include <vespa/log/log.h>
+
+LOG_SETUP(".document.fieldvalue.structured");
+
+namespace document {
+
+IMPLEMENT_IDENTIFIABLE_ABSTRACT(StructuredFieldValue, FieldValue);
+
+StructuredFieldValue::Iterator::Iterator()
+    : _owner(0),
+      _iterator(),
+      _field(0)
+{
+}
+
+StructuredFieldValue::Iterator::Iterator(const StructuredFieldValue& owner,
+                                         const Field* first)
+    : _owner(&const_cast<StructuredFieldValue&>(owner)),
+      _iterator(owner.getIterator(first).release()),
+      _field(_iterator->getNextField())
+{
+}
+
+StructuredFieldValue::Iterator
+StructuredFieldValue::Iterator::operator++(int) // postfix
+{
+    StructuredFieldValue::Iterator it(*this);
+    operator++();
+    return it;
+}
+
+StructuredFieldValue::StructuredFieldValue(const StructuredFieldValue& other)
+    : FieldValue(other),
+      _type(other._type)
+{
+}
+
+StructuredFieldValue::StructuredFieldValue(const DataType &type)
+    : FieldValue(),
+      _type(&type)
+{
+}
+
+void StructuredFieldValue::setType(const DataType& type)
+{
+    _type = &type;
+}
+
+StructuredFieldValue&
+StructuredFieldValue::operator=(const StructuredFieldValue& other)
+{
+    if (this == &other) {
+        return *this;
+    }
+
+    FieldValue::operator=(other);
+    _type = other._type;
+
+    return *this;
+}
+
+void StructuredFieldValue::setFieldValue(const Field & field, const FieldValue & value)
+{
+    if (!field.getDataType().isValueType(value) &&
+        !value.getDataType()->isA(field.getDataType()))
+    {
+        throw vespalib::IllegalArgumentException(
+                "Cannot assign value of type " + value.getDataType()->toString()
+                + "with value : '" + value.toString()
+                + "' to field " + field.getName().c_str() + " of type "
+                + field.getDataType().toString() + ".", VESPA_STRLOC);
+    }
+    setFieldValue(field, FieldValue::UP(value.clone()));
+}
+
+FieldValue::UP
+StructuredFieldValue::onGetNestedFieldValue(
+        FieldPath::const_iterator start,
+        FieldPath::const_iterator end_) const
+{
+    FieldValue::UP fv = getValue(start->getFieldRef());
+    if (fv.get() != NULL) {
+        if ((start + 1) != end_) {
+            return fv->getNestedFieldValue(start + 1, end_);
+        }
+    }
+    return fv;
+}
+
+FieldValue::IteratorHandler::ModificationStatus
+StructuredFieldValue::onIterateNested(
+        FieldPath::const_iterator start,
+        FieldPath::const_iterator end_,
+        IteratorHandler & handler) const
+{
+    IteratorHandler::StructScope autoScope(handler, *this);
+
+    if (start != end_) {
+        if (start->getType() == FieldPathEntry::STRUCT_FIELD) {
+            bool exists = getValue(start->getFieldRef(), start->getFieldValueToSet());
+            LOG(spam, "fieldRef = %s", start->getFieldRef().toString().c_str());
+            LOG(spam, "fieldValueToSet = %s", start->getFieldValueToSet().toString().c_str());
+            if (exists) {
+                IteratorHandler::ModificationStatus
+                    status = start->getFieldValueToSet().iterateNested(start + 1, end_, handler);
+                if (status == IteratorHandler::REMOVED) {
+                    LOG(spam, "field exists, status = REMOVED");
+                    const_cast<StructuredFieldValue&>(*this).remove(start->getFieldRef());
+                    return IteratorHandler::MODIFIED;
+                } else if (status == IteratorHandler::MODIFIED) {
+                    LOG(spam, "field exists, status = MODIFIED");
+                    const_cast<StructuredFieldValue&>(*this).setFieldValue(
+                            start->getFieldRef(), start->getFieldValueToSet());
+                    return IteratorHandler::MODIFIED;
+                } else {
+                    LOG(spam, "field exists, status = %u", status);
+                    return status;
+                }
+            } else if (handler.createMissingPath()) {
+                LOG(spam, "createMissingPath is true");
+                IteratorHandler::ModificationStatus status
+                    = start->getFieldValueToSet().iterateNested(start + 1, end_, handler);
+                if (status == IteratorHandler::MODIFIED) {
+                    LOG(spam, "field did not exist, status = MODIFIED");
+                    const_cast<StructuredFieldValue&>(*this).setFieldValue(
+                            start->getFieldRef(), start->getFieldValueToSet());
+                    return status;
+                }
+            }
+            LOG(spam, "field did not exist, returning NOT_MODIFIED");
+            return IteratorHandler::NOT_MODIFIED;
+        } else {
+            throw vespalib::IllegalArgumentException("Illegal field path for struct value");
+        }
+    } else {
+        IteratorHandler::ModificationStatus
+            status = handler.modify(const_cast<StructuredFieldValue&>(*this));
+        if (status == IteratorHandler::REMOVED) {
+            LOG(spam, "field REMOVED");
+            return status;
+        }
+
+        if (handler.handleComplex(*this)) {
+            LOG(spam, "handleComplex");
+            std::vector<const Field*> fieldsToRemove;
+            for (const_iterator it(begin()), mt(end()); it != mt; it++) {
+                IteratorHandler::ModificationStatus
+                    currStatus = getValue(it.field())->iterateNested(start, end_, handler);
+                if (currStatus == IteratorHandler::REMOVED) {
+                    fieldsToRemove.push_back(&it.field());
+                    status = IteratorHandler::MODIFIED;
+                } else if (currStatus == IteratorHandler::MODIFIED) {
+                    status = currStatus;
+                }
+            }
+
+            for (std::vector<const Field*>::iterator
+                     i = fieldsToRemove.begin(), last = fieldsToRemove.end();
+                 i != last; ++i)
+            {
+                const_cast<StructuredFieldValue&>(*this).remove(**i);
+            }
+        }
+
+        return status;
+    }
+}
+
+} // document
diff --git a/document/src/vespa/document/fieldvalue/structuredfieldvalue.h b/document/src/vespa/document/fieldvalue/structuredfieldvalue.h
new file mode 100644
index 00000000000..9f31296e9f2
--- /dev/null
+++ b/document/src/vespa/document/fieldvalue/structuredfieldvalue.h
@@ -0,0 +1,205 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * \class document::StructuredFieldValue
+ * \ingroup fieldvalue
+ *
+ * \brief Base class for Document and Struct field values.
+ *
+ * This class contains the common functionality between Document and Struct
+ * parts. This includes most their functionality.
+ */
+#pragma once
+
+#include <vespa/document/base/field.h>
+#include <vespa/document/fieldvalue/fieldvalue.h>
+
+namespace document {
+
+class ArrayFieldValue;
+class ByteFieldValue;
+class DoubleFieldValue;
+class FloatFieldValue;
+class IntFieldValue;
+class LongFieldValue;
+class RawFieldValue;
+class StringFieldValue;
+class StructFieldValue;
+class MapFieldValue;
+class WeightedSetFieldValue;
+
+class StructuredFieldValue : public FieldValue
+{
+    const DataType *_type;
+
+    UP onGetNestedFieldValue(FieldPath::const_iterator start, FieldPath::const_iterator end) const override;
+
+protected:
+    StructuredFieldValue(const DataType &type);
+    StructuredFieldValue(const StructuredFieldValue&);
+    StructuredFieldValue& operator=(const StructuredFieldValue&);
+    void swap(StructuredFieldValue & rhs) { std::swap(_type, rhs._type); }
+
+    /** Called from Document when deserializing alters type. */
+    virtual void setType(const DataType& type);
+    const DataType &getType() const { return *_type; }
+
+    struct StructuredIterator {
+        typedef std::unique_ptr<StructuredIterator> UP;
+        virtual ~StructuredIterator() {}
+
+        virtual const Field* getNextField() = 0;
+    };
+    class Iterator {
+        const StructuredFieldValue* _owner;
+        vespalib::LinkedPtr<StructuredIterator> _iterator;
+        const Field* _field;
+
+    public:
+        Iterator(); // Generate end iterator
+
+        // Generate begin iterator
+        Iterator(const StructuredFieldValue& owner, const Field* first);
+
+        const Field &field() const { return *_field; }
+        const Field &operator*() const { return field(); }
+        Iterator& operator++() {
+            _field = _iterator->getNextField();
+            return *this;
+        }
+        Iterator operator++(int); // postfix
+        bool operator==(const Iterator& other) const {
+            if (_field == 0 && other._field == 0)
+               // both at end()
+               return true;
+            if (_field == 0 || other._field == 0)
+                // one at end()
+                return false;
+            return (*_field == *other._field);
+        }
+        bool operator!=(const Iterator& other) const
+            { return !(operator==(other)); }
+    };
+
+        // Used to implement iterator
+    virtual StructuredIterator::UP getIterator(const Field* toFind) const = 0;
+
+        // As overloading doesn't work with polymorphy, have protected functions
+        // doing the functionality, such that we can make utility functions here
+    virtual bool hasFieldValue(const Field&) const = 0;
+    virtual void removeFieldValue(const Field&) = 0;
+    virtual FieldValue::UP getFieldValue(const Field&) const = 0;
+    virtual bool getFieldValue(const Field& field, FieldValue& value) const = 0;
+    virtual void setFieldValue(const Field&, FieldValue::UP value) = 0;
+    void setFieldValue(const Field & field, const FieldValue & value);
+
+    IteratorHandler::ModificationStatus
+    onIterateNested(FieldPath::const_iterator start, FieldPath::const_iterator end, IteratorHandler & handler) const override;
+
+public:
+    DECLARE_IDENTIFIABLE_ABSTRACT(StructuredFieldValue);
+
+
+    virtual StructuredFieldValue* clone() const = 0;
+    const DataType *getDataType() const override { return _type; }
+
+    /** Wrapper for DataType's hasField() function. */
+    virtual bool hasField(const vespalib::stringref & name) const = 0;
+    /**
+     * Wrapper for DataType's getField() function.
+     * @throws FieldNotFoundException If no field with given name exist.
+     */
+    virtual const Field& getField(const vespalib::stringref & name) const = 0;
+
+    /**
+     * Retrieve value of given field and assign it to given field.
+     *
+     * @return True if field is set and stored in value, false if unset.
+     * @throws vespalib::IllegalArgumentException If value given has wrong type
+     */
+    inline bool getValue(const Field& field, FieldValue& value) const
+        { return getFieldValue(field, value); }
+    /** @return Retrieve value of given field. Null pointer if not set. */
+    inline FieldValue::UP getValue(const Field& field) const
+        { return getFieldValue(field); }
+    /** @return Retrieve value of given field. Null pointer if not set. */
+    inline FieldValue::UP getValue(const vespalib::stringref & name) const
+        { return getFieldValue(getField(name)); }
+    /** @return True if value is set. */
+    inline bool hasValue(const Field& field) const
+        { return hasFieldValue(field); }
+
+    /**
+     * Set the given field to contain given value.
+     *
+     * @throws vespalib::IllegalArgumentException If value given has wrong type
+     */
+    inline void setValue(const Field& field, const FieldValue& value)
+        { setFieldValue(field, value); }
+    /** Remove the value of given field if it is set. */
+    inline void remove(const Field& field)
+        { removeFieldValue(field); }
+
+    virtual void clear() = 0;
+
+        // Utility functions for easy but less efficient access
+    inline bool hasValue(const vespalib::stringref & fieldName) const
+        { return hasFieldValue(getField(fieldName)); }
+    inline void remove(const vespalib::stringref & fieldName)
+        { removeFieldValue(getField(fieldName)); }
+    inline void setValue(const vespalib::stringref & fieldName, const FieldValue& value)
+        { setFieldValue(getField(fieldName), value); }
+    template<typename PrimitiveType>
+    void set(const Field& field, const PrimitiveType& value);
+    template<typename PrimitiveType>
+    void set(const vespalib::stringref & fieldName, const PrimitiveType& value);
+
+    size_t getSetFieldCount() const {
+        size_t count = 0;
+        for (const_iterator it(begin()), mt(end()); it != mt; ++it, ++count) {}
+        return count;
+    }
+    virtual bool empty() const = 0;
+
+    typedef Iterator const_iterator;
+    const_iterator begin() const { return const_iterator(*this, NULL); }
+    const_iterator end() const { return const_iterator(); }
+
+    /**
+     * return an iterator starting at field, or end() if field was not found
+     **/
+    const_iterator find(const Field& field) const {
+        return const_iterator(*this, &field);
+    }
+
+    template <typename T>
+    std::unique_ptr<T> getAs(const Field &field) const {
+        FieldValue::UP val = getValue(field);
+        T *t = Identifiable::cast<T *>(val.get());
+        if (val.get() && !t) {
+            throw vespalib::IllegalStateException("Field " + field.toString()
+                    + " has unexpected type.", VESPA_STRLOC);
+        }
+        val.release();
+        return std::unique_ptr<T>(t);
+    }
+};
+
+template<typename PrimitiveType>
+void
+StructuredFieldValue::set(const Field& field, const PrimitiveType& value)
+{
+    FieldValue::UP fval(field.getDataType().createFieldValue());
+    *fval = value;
+    setFieldValue(field, std::move(fval));
+}
+
+template<typename PrimitiveType>
+void
+StructuredFieldValue::set(const vespalib::stringref & fieldName,
+                          const PrimitiveType& value)
+{
+    set(getField(fieldName), value);
+}
+
+} // document
+
diff --git a/document/src/vespa/document/fieldvalue/tensorfieldvalue.cpp b/document/src/vespa/document/fieldvalue/tensorfieldvalue.cpp
new file mode 100644
index 00000000000..97f46a64980
--- /dev/null
+++ b/document/src/vespa/document/fieldvalue/tensorfieldvalue.cpp
@@ -0,0 +1,170 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "tensorfieldvalue.h"
+#include <vespa/vespalib/tensor/tensor.h>
+
+using vespalib::tensor::Tensor;
+
+namespace document {
+
+TensorFieldValue::TensorFieldValue()
+    : FieldValue(),
+      _tensor(),
+      _altered(true)
+{
+}
+
+TensorFieldValue::TensorFieldValue(const TensorFieldValue &rhs)
+    : FieldValue(),
+      _tensor(),
+      _altered(true)
+{
+    if (rhs._tensor) {
+        _tensor = rhs._tensor->clone();
+    }
+}
+
+
+TensorFieldValue::TensorFieldValue(TensorFieldValue &&rhs)
+    : FieldValue(),
+      _tensor(),
+      _altered(true)
+{
+    _tensor = std::move(rhs._tensor);
+}
+
+
+TensorFieldValue::~TensorFieldValue()
+{
+}
+
+
+TensorFieldValue &
+TensorFieldValue::operator=(const TensorFieldValue &rhs)
+{
+    if (this != &rhs) {
+        if (rhs._tensor) {
+            _tensor = rhs._tensor->clone();
+        } else {
+            _tensor.reset();
+        }
+        _altered = true;
+    }
+    return *this;
+}
+
+
+TensorFieldValue &
+TensorFieldValue::operator=(std::unique_ptr<Tensor> rhs)
+{
+    _tensor = std::move(rhs);
+    _altered = true;
+    return *this;
+}
+
+
+
+void
+TensorFieldValue::accept(FieldValueVisitor &visitor)
+{
+    visitor.visit(*this);
+}
+
+
+void
+TensorFieldValue::accept(ConstFieldValueVisitor &visitor) const
+{
+    visitor.visit(*this);
+}
+
+
+const DataType *
+TensorFieldValue::getDataType() const
+{
+    return DataType::TENSOR;
+}
+
+
+bool
+TensorFieldValue::hasChanged() const
+{
+    return _altered;
+}
+
+
+TensorFieldValue*
+TensorFieldValue::clone() const
+{
+    return new TensorFieldValue(*this);
+}
+
+
+void
+TensorFieldValue::print(std::ostream& out, bool verbose,
+                        const std::string& indent) const
+{
+    (void) verbose;
+    (void) indent;
+    out << "{TensorFieldValue::print not yet implemented}";
+}
+
+void
+TensorFieldValue::printXml(XmlOutputStream& out) const
+{
+    out << "{TensorFieldValue::printXml not yet implemented}";
+}
+
+
+FieldValue &
+TensorFieldValue::assign(const FieldValue &value)
+{
+    const TensorFieldValue *rhs =
+        Identifiable::cast<const TensorFieldValue *>(&value);
+    if (rhs != nullptr) {
+        *this = *rhs;
+    } else {
+        return FieldValue::assign(value);
+    }
+    return *this;
+}
+
+
+void
+TensorFieldValue::assignDeserialized(std::unique_ptr<Tensor> rhs)
+{
+    _tensor = std::move(rhs);
+    _altered = false; // Serialized form already exists
+}
+
+
+int
+TensorFieldValue::compare(const FieldValue &other) const
+{
+    if (this == &other) {
+        return 0;	// same identity
+    }
+    int diff = FieldValue::compare(other);
+    if (diff != 0) {
+        return diff;    // field type mismatch
+    }
+    const TensorFieldValue & rhs(static_cast<const TensorFieldValue &>(other));
+    if (!_tensor) {
+        return (rhs._tensor ? -1 : 0);
+    }
+    if (!rhs._tensor) {
+        return 1;
+    }
+    if (_tensor->equals(*rhs._tensor)) {
+        return 0;
+    }
+    assert(_tensor.get() != rhs._tensor.get());
+    // XXX: Wrong, compares identity of tensors instead of values
+    // Note: sorting can be dangerous due to this.
+    return ((_tensor.get()  < rhs._tensor.get()) ? -1 : 1);
+}
+
+
+IMPLEMENT_IDENTIFIABLE(TensorFieldValue, FieldValue);
+
+} // document
diff --git a/document/src/vespa/document/fieldvalue/tensorfieldvalue.h b/document/src/vespa/document/fieldvalue/tensorfieldvalue.h
new file mode 100644
index 00000000000..9e07eceab96
--- /dev/null
+++ b/document/src/vespa/document/fieldvalue/tensorfieldvalue.h
@@ -0,0 +1,46 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "fieldvalue.h"
+
+namespace vespalib { namespace tensor { class Tensor; } }
+
+namespace document {
+
+/**
+ * Field value representing a tensor.
+ */
+class TensorFieldValue : public FieldValue {
+private:
+    std::unique_ptr<vespalib::tensor::Tensor> _tensor;
+    bool _altered;
+public:
+    TensorFieldValue();
+    TensorFieldValue(const TensorFieldValue &rhs);
+    TensorFieldValue(TensorFieldValue &&rhs);
+    ~TensorFieldValue();
+
+    TensorFieldValue &operator=(const TensorFieldValue &rhs);
+    TensorFieldValue &operator=(std::unique_ptr<vespalib::tensor::Tensor> rhs);
+
+    virtual void accept(FieldValueVisitor &visitor) override;
+    virtual void accept(ConstFieldValueVisitor &visitor) const override;
+    virtual const DataType *getDataType() const override;
+    virtual bool hasChanged() const override;
+    virtual TensorFieldValue* clone() const override;
+    virtual void print(std::ostream& out, bool verbose,
+                       const std::string& indent) const override;
+    virtual void printXml(XmlOutputStream& out) const override;
+    virtual FieldValue &assign(const FieldValue &value) override;
+    const std::unique_ptr<vespalib::tensor::Tensor> &getAsTensorPtr() const {
+        return _tensor;
+    }
+    void assignDeserialized(std::unique_ptr<vespalib::tensor::Tensor> rhs);
+    virtual int compare(const FieldValue& other) const override;
+
+    DECLARE_IDENTIFIABLE(TensorFieldValue);
+};
+
+} // document
+
diff --git a/document/src/vespa/document/fieldvalue/weightedsetfieldvalue.cpp b/document/src/vespa/document/fieldvalue/weightedsetfieldvalue.cpp
new file mode 100644
index 00000000000..0cb3969a9aa
--- /dev/null
+++ b/document/src/vespa/document/fieldvalue/weightedsetfieldvalue.cpp
@@ -0,0 +1,223 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+
+#include <vespa/document/base/exceptions.h>
+#include <vespa/document/datatype/arraydatatype.h>
+#include <vespa/document/datatype/mapdatatype.h>
+#include <vespa/document/fieldvalue/arrayfieldvalue.h>
+#include <vespa/document/fieldvalue/weightedsetfieldvalue.h>
+#include <vespa/document/util/bytebuffer.h>
+#include <vespa/vespalib/objects/identifiable.h>
+
+using vespalib::Identifiable;
+
+LOG_SETUP(".document.fieldvalue.weightedset");
+
+/// \todo TODO (was warning):  Find a way to search through internal map without duplicating keys to create shared pointers.
+
+namespace document {
+
+IMPLEMENT_IDENTIFIABLE_ABSTRACT(WeightedSetFieldValue, CollectionFieldValue);
+
+namespace {
+const DataType &getKeyType(const DataType &type) {
+    const WeightedSetDataType *wtype =
+        Identifiable::cast<const WeightedSetDataType *>(&type);
+    if (!wtype) {
+        throw vespalib::IllegalArgumentException(
+                "Cannot generate a weighted set value with non-weighted set "
+                "type " + type.toString() + ".", VESPA_STRLOC);
+    }
+    return wtype->getNestedType();
+}
+}  // namespace
+
+WeightedSetFieldValue::WeightedSetFieldValue(const DataType &type)
+    : CollectionFieldValue(type),
+      _map_type(new MapDataType(getKeyType(type), *DataType::INT)),
+      _map(*_map_type),
+      _altered(true) {
+}
+
+WeightedSetFieldValue::~WeightedSetFieldValue()
+{
+}
+
+void WeightedSetFieldValue::verifyKey(const FieldValue & v)
+{
+    if (!getNestedType().isValueType(v)) {
+        throw InvalidDataTypeException(*v.getDataType(), getNestedType(),
+                                       VESPA_STRLOC);
+    }
+}
+
+bool
+WeightedSetFieldValue::add(const FieldValue& key, int weight)
+{
+    verifyKey(key);
+    const WeightedSetDataType & wdt(static_cast<const WeightedSetDataType&>(*_type));
+    _altered = true;
+    if (wdt.removeIfZero() && (weight == 0)) {
+        _map.erase(key);
+        return false;
+    }
+    return _map.insert(FieldValue::UP(key.clone()), FieldValue::UP(new IntFieldValue(weight)));
+}
+
+bool
+WeightedSetFieldValue::addIgnoreZeroWeight(const FieldValue& key,
+                                           int32_t weight)
+{
+    verifyKey(key);
+    _altered = true;
+    return _map.insert(FieldValue::UP(key.clone()),
+                       FieldValue::UP(new IntFieldValue(weight)));
+}
+
+void
+WeightedSetFieldValue::push_back(FieldValue::UP key, int weight)
+{
+    _altered = true;
+    _map.push_back(std::move(key), FieldValue::UP(new IntFieldValue(weight)));
+}
+
+void
+WeightedSetFieldValue::increment(const FieldValue& key, int val)
+{
+    verifyKey(key);
+    WeightedFieldValueMap::iterator it(_map.find(key));
+    const WeightedSetDataType & wdt(static_cast<const WeightedSetDataType&>(*_type));
+    if (wdt.createIfNonExistent()) {
+        if (it == _map.end()) {
+            _map.insert(FieldValue::UP(key.clone()), FieldValue::UP(new IntFieldValue(val)));
+        } else {
+            IntFieldValue& fv = static_cast<IntFieldValue&>(*it->second);
+            fv.setValue(fv.getValue() + val);
+            if (wdt.removeIfZero() && fv.getValue() == 0) {
+                _map.erase(key);
+            }
+        }
+    } else {
+        if (it == _map.end()) {
+           throw vespalib::IllegalStateException("Cannot modify non-existing "
+                    "entry in weightedset without createIfNonExistent set",
+                    VESPA_STRLOC);
+        }
+        IntFieldValue& fv = static_cast<IntFieldValue&>(*it->second);
+        fv.setValue(fv.getValue() + val);
+        if (wdt.removeIfZero() && fv.getValue() == 0) {
+            _map.erase(key);
+        }
+    }
+    _altered = true;
+}
+
+int32_t
+WeightedSetFieldValue::get(const FieldValue& key, int32_t defaultValue) const
+{
+    WeightedFieldValueMap::const_iterator it = find(key);
+    return (it == end()
+            ? defaultValue
+            : static_cast<const IntFieldValue&>(*it->second).getValue());
+}
+
+bool
+WeightedSetFieldValue::containsValue(const FieldValue& key) const
+{
+    return _map.contains(key);
+}
+
+bool
+WeightedSetFieldValue::removeValue(const FieldValue& key)
+{
+    bool result = _map.erase(key);
+    _altered |= result;
+    return result;
+}
+
+FieldValue&
+WeightedSetFieldValue::assign(const FieldValue& value)
+{
+    if (getDataType()->isValueType(value)) {
+        return operator=(static_cast<const WeightedSetFieldValue&>(value));
+    }
+    return FieldValue::assign(value);
+}
+
+int
+WeightedSetFieldValue::compare(const FieldValue& other) const
+{
+    int diff = CollectionFieldValue::compare(other);
+    if (diff != 0) return diff;
+
+    const WeightedSetFieldValue& wset(
+            dynamic_cast<const WeightedSetFieldValue&>(other));
+    return _map.compare(wset._map);
+}
+
+void
+WeightedSetFieldValue::printXml(XmlOutputStream& xos) const
+{
+    for (WeightedFieldValueMap::const_iterator it = _map.begin();
+         it != _map.end(); ++it)
+    {
+        const IntFieldValue& fv = static_cast<const IntFieldValue&>(*it->second);
+        xos << XmlTag("item") << XmlAttribute("weight", fv.getValue())
+            << *it->first
+            << XmlEndTag();
+    }
+}
+
+void
+WeightedSetFieldValue::print(std::ostream& out, bool verbose,
+                       const std::string& indent) const
+{
+    out << getDataType()->getName() << "(";
+
+    int count = 0;
+    for (WeightedFieldValueMap::const_iterator it = _map.begin();
+         it != _map.end(); ++it)
+    {
+        if (count++ != 0) {
+            out << ",";
+        }
+        out << "\n" << indent << "  ";
+        it->first->print(out, verbose, indent + "  ");
+        const IntFieldValue& fv = static_cast<const IntFieldValue&>(*it->second);
+        out << " - weight " << fv.getValue();
+    }
+    if (_map.size() > 0) out << "\n" << indent;
+    out << ")";
+}
+
+bool
+WeightedSetFieldValue::hasChanged() const
+{
+    // Keys are not allowed to change in a map, so the keys should not be
+    // referred to externally, and should thus not need to be checked.
+    return _altered;
+}
+
+WeightedSetFieldValue::const_iterator
+WeightedSetFieldValue::find(const FieldValue& key) const
+{
+    return _map.find(key);
+}
+
+WeightedSetFieldValue::iterator
+WeightedSetFieldValue::find(const FieldValue& key)
+{
+    return _map.find(key);
+}
+
+FieldValue::IteratorHandler::ModificationStatus
+WeightedSetFieldValue::onIterateNested(FieldPath::const_iterator start,
+                                       FieldPath::const_iterator end_,
+                                       IteratorHandler & handler) const
+{
+    LOG(spam, "iterating over WeightedSetFieldValue");
+    return _map.iterateNestedImpl(start, end_, handler, *this);
+}
+
+} // document
diff --git a/document/src/vespa/document/fieldvalue/weightedsetfieldvalue.h b/document/src/vespa/document/fieldvalue/weightedsetfieldvalue.h
new file mode 100644
index 00000000000..0963726307a
--- /dev/null
+++ b/document/src/vespa/document/fieldvalue/weightedsetfieldvalue.h
@@ -0,0 +1,152 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * \class document::WeightedSetFieldValue
+ * \ingroup fieldvalue
+ *
+ * \brief A fieldvalue containing fieldvalue <-> weight mappings.
+ */
+#pragma once
+
+#include <vespa/document/fieldvalue/collectionfieldvalue.h>
+#include <vespa/document/fieldvalue/mapfieldvalue.h>
+#include <vespa/document/fieldvalue/intfieldvalue.h>
+#include <vespa/document/datatype/weightedsetdatatype.h>
+#include <map>
+
+namespace document {
+
+class WeightedSetFieldValue : public CollectionFieldValue
+{
+public:
+    struct FieldValuePtrOrder {
+        template<typename T> bool operator()(const T& s1, const T& s2) const
+            { return *s1 < *s2; }
+    };
+    typedef MapFieldValue WeightedFieldValueMap;
+
+private:
+    std::shared_ptr<const MapDataType> _map_type;
+    WeightedFieldValueMap _map;
+    bool _altered;
+
+    void verifyKey(const FieldValue & key);
+    virtual bool addValue(const FieldValue& fval) { return add(fval, 1); }
+    virtual bool containsValue(const FieldValue& val) const;
+    virtual bool removeValue(const FieldValue& val);
+    virtual IteratorHandler::ModificationStatus onIterateNested(
+            FieldPath::const_iterator start,
+            FieldPath::const_iterator end,
+            IteratorHandler& handler) const;
+public:
+    typedef std::unique_ptr<WeightedSetFieldValue> UP;
+
+    /**
+     * @param wsetType Type of the weighted set. Must be a WeightedSetDataType,
+     *                 but does not enforce type compile time so it will be
+     *                 easier to create instances using field's getDataType().
+     */
+    WeightedSetFieldValue(const DataType &wsetType);
+    virtual ~WeightedSetFieldValue();
+
+    void accept(FieldValueVisitor &visitor) override { visitor.visit(*this); }
+    void accept(ConstFieldValueVisitor &visitor) const override { visitor.visit(*this); }
+
+    /**
+     * Add an item to the weighted set with the given weight. If removeIfZero
+     * is set in the data type and weight is zero, the new item will not be
+     * added and any existing item for the key will be immediately removed.
+     */
+    bool add(const FieldValue&, int32_t weight = 1);
+    /**
+     * Add an item to the weighted set, but do not erase the item if the
+     * weight is zero and removeIfZero is set in the weighted set's type.
+     */
+    bool addIgnoreZeroWeight(const FieldValue&, int32_t weight = 1);
+    void push_back(FieldValue::UP, int32_t weight);
+    void increment(const FieldValue& fval, int val = 1);
+    void decrement(const FieldValue& fval, int val = 1)
+        { increment(fval, -1*val); }
+    int32_t get(const FieldValue&, int32_t defaultValue = 0) const;
+
+        // CollectionFieldValue implementation
+    virtual bool isEmpty() const { return _map.isEmpty(); }
+    virtual size_t size() const { return _map.size(); }
+    virtual void clear() { _map.clear(); }
+    void reserve(size_t sz) { _map.reserve(sz); }
+    void resize(size_t sz) { _map.resize(sz); }
+
+        // FieldValue implementation
+    virtual FieldValue& assign(const FieldValue&);
+    virtual WeightedSetFieldValue* clone() const
+        { return new WeightedSetFieldValue(*this); }
+    virtual int compare(const FieldValue&) const;
+    virtual void printXml(XmlOutputStream& out) const;
+    virtual void print(std::ostream& out, bool verbose,
+                       const std::string& indent) const;
+    virtual bool hasChanged() const;
+
+        // Implements iterating through internal content.
+    typedef WeightedFieldValueMap::const_iterator const_iterator;
+    typedef WeightedFieldValueMap::iterator iterator;
+
+    const_iterator begin() const { return _map.begin(); }
+    iterator begin() { return _map.begin(); }
+
+    const_iterator end() const { return _map.end(); }
+    iterator end() { return _map.end(); }
+
+    const_iterator find(const FieldValue& fv) const;
+    iterator find(const FieldValue& fv);
+
+        // Utility functions for easy use of weighted sets of primitives
+
+    bool add(const vespalib::stringref & val, int32_t weight = 1)
+        { return add(*createNested() = val, weight); }
+    bool add(int32_t val, int32_t weight = 1)
+        { return add(*createNested() = val, weight); }
+    bool add(int64_t val, int32_t weight = 1)
+        { return add(*createNested() = val, weight); }
+    bool add(float val, int32_t weight = 1)
+        { return add(*createNested() = val, weight); }
+    bool add(double val, int32_t weight = 1)
+        { return add(*createNested() = val, weight); }
+
+    int32_t get(const vespalib::stringref & val) const
+        { return get(*createNested() = val); }
+    int32_t get(int32_t val) const
+        { return get(*createNested() = val); }
+    int32_t get(int64_t val) const
+        { return get(*createNested() = val); }
+    int32_t get(float val) const
+        { return get(*createNested() = val); }
+    int32_t get(double val) const
+        { return get(*createNested() = val); }
+
+    void increment(const vespalib::stringref & val, int32_t weight = 1)
+        { increment(*createNested() = val, weight); }
+    void increment(int32_t val, int32_t weight = 1)
+        { increment(*createNested() = val, weight); }
+    void increment(int64_t val, int32_t weight = 1)
+        { increment(*createNested() = val, weight); }
+    void increment(float val, int32_t weight = 1)
+        { increment(*createNested() = val, weight); }
+    void increment(double val, int32_t weight = 1)
+        { increment(*createNested() = val, weight); }
+
+    void decrement(const vespalib::stringref & val, int32_t weight = 1)
+        { decrement(*createNested() = val, weight); }
+    void decrement(int32_t val, int32_t weight = 1)
+        { decrement(*createNested() = val, weight); }
+    void decrement(int64_t val, int32_t weight = 1)
+        { decrement(*createNested() = val, weight); }
+    void decrement(float val, int32_t weight = 1)
+        { decrement(*createNested() = val, weight); }
+    void decrement(double val, int32_t weight = 1)
+        { decrement(*createNested() = val, weight); }
+
+    DECLARE_IDENTIFIABLE_ABSTRACT(WeightedSetFieldValue);
+
+};
+
+} // document
+
diff --git a/document/src/vespa/document/predicate/CMakeLists.txt b/document/src/vespa/document/predicate/CMakeLists.txt
new file mode 100644
index 00000000000..4c24b11a7ee
--- /dev/null
+++ b/document/src/vespa/document/predicate/CMakeLists.txt
@@ -0,0 +1,12 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_library(document_predicate OBJECT
+    SOURCES
+    predicate.cpp
+    predicate_builder.cpp
+    predicate_printer.cpp
+    predicate_slime_builder.cpp
+    predicate_slime_visitor.cpp
+    DEPENDS
+    AFTER
+    document_documentconfig
+)
diff --git a/document/src/vespa/document/predicate/predicate.cpp b/document/src/vespa/document/predicate/predicate.cpp
new file mode 100644
index 00000000000..b690a8e88d1
--- /dev/null
+++ b/document/src/vespa/document/predicate/predicate.cpp
@@ -0,0 +1,144 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/log/log.h>
+LOG_SETUP(".predicate");
+#include <vespa/fastos/fastos.h>
+
+#include "predicate.h"
+#include <vespa/vespalib/data/slime/slime.h>
+#include <algorithm>
+#include <set>
+
+using std::mismatch;
+using std::pair;
+using std::set;
+using std::string;
+using std::vector;
+using vespalib::Slime;
+using namespace vespalib::slime;
+
+namespace document {
+
+const string Predicate::NODE_TYPE("type");
+const string Predicate::KEY("key");
+const string Predicate::SET("feature_set");
+const string Predicate::RANGE_MIN("range_min");
+const string Predicate::RANGE_MAX("range_max");
+const string Predicate::CHILDREN("children");
+const string Predicate::PARTITIONS("partitions");
+const string Predicate::EDGE_PARTITIONS("edge_partitions");
+const string Predicate::HASHED_PARTITIONS("hashed_partitions");
+const string Predicate::HASHED_EDGE_PARTITIONS("hashed_edge_partitions");
+const string Predicate::HASH("hash");
+const string Predicate::PAYLOAD("payload");
+const string Predicate::LABEL("label");
+const string Predicate::VALUE("value");
+const string Predicate::LOWER_BOUND("lower_bound");
+const string Predicate::UPPER_BOUND("upper_bound");
+
+namespace {
+int compareSets(const Inspector &set1, const Inspector &set2) {
+    if (set1.entries() < set2.entries()) {
+        return -1;
+    } else if (set1.entries() > set2.entries()) {
+        return 1;
+    }
+    set<string> s1, s2;
+    for (size_t i = 0; i < set1.entries(); ++i) {
+        s1.insert(set1[i].asString().make_string());
+        s2.insert(set2[i].asString().make_string());
+    }
+    pair<set<string>::iterator, set<string>::iterator> r =
+        mismatch(s1.begin(), s1.end(), s2.begin());
+    if (r.first == s1.end()) { return 0; }
+    if (*r.first < *r.second) { return -1; }
+    return 1;
+}
+
+int compareLongs(const Inspector &long1, const Inspector &long2) {
+    if (long1.valid() && !long2.valid()) { return -1; }
+    if (!long1.valid() && long2.valid()) { return 1; }
+    if (long1.asLong() < long2.asLong()) { return -1; }
+    if (long1.asLong() > long2.asLong()) { return 1; }
+    return 0;
+}
+
+int compareNodes(const Inspector &n1, const Inspector &n2) {
+    int ret = compareLongs(n1[Predicate::NODE_TYPE], n2[Predicate::NODE_TYPE]);
+    if (ret) { return ret; }
+    if (n1[Predicate::CHILDREN].valid()) {
+        for (size_t i = 0; i < n1[Predicate::CHILDREN].entries(); ++i) {
+            int r = compareNodes(n1[Predicate::CHILDREN][i],
+                                 n2[Predicate::CHILDREN][i]);
+            if (r) { return r; }
+        }
+        return 0;
+    } else {
+        string key1 = n1[Predicate::KEY].asString().make_string();
+        string key2 = n2[Predicate::KEY].asString().make_string();
+        if (key1 < key2) { return -1; }
+        if (key1 > key2) { return 1; }
+        if (n1[Predicate::SET].valid()) {
+            return compareSets(n1[Predicate::SET], n2[Predicate::SET]);
+        } else {
+            int r = compareLongs(n1[Predicate::RANGE_MIN],
+                                 n2[Predicate::RANGE_MIN]);
+            if (r) { return r; }
+            return compareLongs(n1[Predicate::RANGE_MAX],
+                                n2[Predicate::RANGE_MAX]);
+        }
+    }
+}
+}  // namespace
+
+int Predicate::compare(const Slime &s1, const Slime &s2) {
+    return compareNodes(s1.get(), s2.get());
+}
+
+namespace {
+template <typename InsertIt>
+class InsertFromArray : public ArrayTraverser {
+    InsertIt _it;
+public:
+    InsertFromArray(InsertIt it) : _it(it) {}
+    ArrayTraverser &ref() { return *this; }
+    virtual void entry(size_t, const Inspector &inspector) {
+        *_it++ = inspector.asString().make_string();
+    }
+};
+
+template <typename InsertIt>
+InsertFromArray<InsertIt> make_insert_from_array(InsertIt it) {
+    return InsertFromArray<InsertIt>(it);
+}
+
+int64_t defaultUnlessDefined(const Inspector &i, int64_t default_value) {
+    if (i.valid()) {
+        return i.asLong();
+    }
+    return default_value;
+}
+}  // namespace
+
+
+FeatureBase::FeatureBase(const Inspector &inspector)
+    : _key(inspector[Predicate::KEY].asString().make_string()) {
+}
+
+
+FeatureSet::FeatureSet(const Inspector &inspector)
+    : FeatureBase(inspector), _features() {
+    inspector[Predicate::SET].traverse(
+            make_insert_from_array(back_inserter(_features)).ref());
+}
+
+
+FeatureRange::FeatureRange(const Inspector &inspector)
+    : FeatureBase(inspector),
+      _min(defaultUnlessDefined(inspector[Predicate::RANGE_MIN], LLONG_MIN)),
+      _max(defaultUnlessDefined(inspector[Predicate::RANGE_MAX], LLONG_MAX)),
+      _has_min(inspector[Predicate::RANGE_MIN].valid()),
+      _has_max(inspector[Predicate::RANGE_MAX].valid()) {
+}
+
+}  // namespace document
diff --git a/document/src/vespa/document/predicate/predicate.h b/document/src/vespa/document/predicate/predicate.h
new file mode 100644
index 00000000000..23968f0e904
--- /dev/null
+++ b/document/src/vespa/document/predicate/predicate.h
@@ -0,0 +1,126 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <memory>
+#include <string>
+#include <vector>
+
+namespace vespalib {
+class Slime;
+namespace slime { class Inspector; }
+}  // namespace vespalib
+
+namespace document {
+
+struct Predicate {
+    static const std::string NODE_TYPE;
+    static const std::string KEY;
+    static const std::string SET;
+    static const std::string RANGE_MIN;
+    static const std::string RANGE_MAX;
+    static const std::string CHILDREN;
+    static const std::string PARTITIONS;
+    static const std::string EDGE_PARTITIONS;
+    static const std::string HASHED_PARTITIONS;
+    static const std::string HASHED_EDGE_PARTITIONS;
+    static const std::string HASH;
+    static const std::string PAYLOAD;
+    static const std::string LABEL;
+    static const std::string VALUE;
+    static const std::string LOWER_BOUND;
+    static const std::string UPPER_BOUND;
+
+    static const int TYPE_CONJUNCTION = 1;
+    static const int TYPE_DISJUNCTION = 2;
+    static const int TYPE_NEGATION = 3;
+    static const int TYPE_FEATURE_SET = 4;
+    static const int TYPE_FEATURE_RANGE = 5;
+    static const int TYPE_TRUE = 6;
+    static const int TYPE_FALSE = 7;
+
+    static int compare(const vespalib::Slime &s1, const vespalib::Slime &s2);
+};
+
+struct PredicateNode {
+    typedef std::unique_ptr<PredicateNode> UP;
+
+    virtual ~PredicateNode() {}
+};
+
+class FeatureBase : public PredicateNode {
+    const std::string _key;
+public:
+    FeatureBase(const vespalib::slime::Inspector &inspector);
+
+    std::string getKey() const { return _key; }
+};
+
+
+class FeatureSet : public FeatureBase {
+    std::vector<std::string> _features;
+
+public:
+    FeatureSet(const vespalib::slime::Inspector &inspector);
+
+    size_t getSize() const { return _features.size(); }
+    std::string operator[](size_t i) const { return _features[i]; }
+};
+
+
+class FeatureRange : public FeatureBase {
+    const long _min;
+    const long _max;
+    const bool _has_min;
+    const bool _has_max;
+public:
+    FeatureRange(const vespalib::slime::Inspector &inspector);
+
+    long getMin() const { return _min; }
+    long getMax() const { return _max; }
+    bool hasMin() const { return _has_min; }
+    bool hasMax() const { return _has_max; }
+};
+
+
+class Negation : public PredicateNode {
+    PredicateNode::UP _child;
+
+public:
+    Negation(PredicateNode::UP child) : _child(std::move(child)) {}
+
+    PredicateNode& getChild() const { return *_child; }
+};
+
+
+class IntermediatePredicateNode : public PredicateNode {
+    std::vector<PredicateNode *> _children;
+
+public:
+    IntermediatePredicateNode(const std::vector<PredicateNode *> children)
+        : _children(children) {}
+    ~IntermediatePredicateNode() {
+        for (size_t i = 0; i < _children.size(); ++i)
+            delete _children[i];
+    }
+
+    size_t getSize() const { return _children.size(); }
+    PredicateNode *operator[](size_t i) const { return _children[i]; }
+};
+
+
+struct Conjunction : IntermediatePredicateNode {
+    Conjunction(const std::vector<PredicateNode *> children)
+        : IntermediatePredicateNode(children) {}
+};
+
+struct Disjunction : IntermediatePredicateNode {
+    Disjunction(const std::vector<PredicateNode *> children)
+        : IntermediatePredicateNode(children) {}
+};
+
+struct TruePredicate : PredicateNode {};
+struct FalsePredicate : PredicateNode {};
+
+}  // namespace document
+
diff --git a/document/src/vespa/document/predicate/predicate_builder.cpp b/document/src/vespa/document/predicate/predicate_builder.cpp
new file mode 100644
index 00000000000..98bc05eab29
--- /dev/null
+++ b/document/src/vespa/document/predicate/predicate_builder.cpp
@@ -0,0 +1,52 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/log/log.h>
+LOG_SETUP(".predicate_builder");
+#include <vespa/fastos/fastos.h>
+
+#include "predicate.h"
+#include "predicate_builder.h"
+#include <vespa/vespalib/data/slime/inspector.h>
+
+using vespalib::slime::Inspector;
+
+namespace document {
+
+void PredicateBuilder::visitFeatureSet(const Inspector &i) {
+    _nodes.push_back(new FeatureSet(i));
+}
+
+void PredicateBuilder::visitFeatureRange(const Inspector &i) {
+    _nodes.push_back(new FeatureRange(i));
+}
+
+void PredicateBuilder::visitNegation(const Inspector &i) {
+    visitChildren(i);
+    _nodes.back() = new Negation(PredicateNode::UP(_nodes.back()));
+}
+
+void PredicateBuilder::visitConjunction(const Inspector &i) {
+    std::vector<PredicateNode *> nodes;
+    nodes.swap(_nodes);
+    visitChildren(i);
+    nodes.push_back(new Conjunction(_nodes));
+    nodes.swap(_nodes);
+}
+
+void PredicateBuilder::visitDisjunction(const Inspector &i) {
+    std::vector<PredicateNode *> nodes;
+    nodes.swap(_nodes);
+    visitChildren(i);
+    nodes.push_back(new Disjunction(_nodes));
+    nodes.swap(_nodes);
+}
+
+void PredicateBuilder::visitTrue(const Inspector &) {
+    _nodes.push_back(new TruePredicate);
+}
+
+void PredicateBuilder::visitFalse(const Inspector &) {
+    _nodes.push_back(new FalsePredicate);
+}
+
+}  // namespace document
diff --git a/document/src/vespa/document/predicate/predicate_builder.h b/document/src/vespa/document/predicate/predicate_builder.h
new file mode 100644
index 00000000000..68fa0caa0dc
--- /dev/null
+++ b/document/src/vespa/document/predicate/predicate_builder.h
@@ -0,0 +1,32 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "predicate_slime_visitor.h"
+#include <memory>
+#include <vector>
+
+namespace document {
+class PredicateNode;
+
+class PredicateBuilder : private PredicateSlimeVisitor {
+    std::vector<PredicateNode *>_nodes;
+
+    virtual void visitFeatureSet(const vespalib::slime::Inspector &i);
+    virtual void visitFeatureRange(const vespalib::slime::Inspector &i);
+    virtual void visitNegation(const vespalib::slime::Inspector &i);
+    virtual void visitConjunction(const vespalib::slime::Inspector &i);
+    virtual void visitDisjunction(const vespalib::slime::Inspector &i);
+    virtual void visitTrue(const vespalib::slime::Inspector &i);
+    virtual void visitFalse(const vespalib::slime::Inspector &i);
+
+public:
+    std::unique_ptr<PredicateNode> build(const vespalib::slime::Inspector &i) {
+        visit(i);
+        assert(_nodes.size() == 1);
+        return std::unique_ptr<PredicateNode>(_nodes.front());
+    }
+};
+
+}  // namespace document
+
diff --git a/document/src/vespa/document/predicate/predicate_printer.cpp b/document/src/vespa/document/predicate/predicate_printer.cpp
new file mode 100644
index 00000000000..c8519866d11
--- /dev/null
+++ b/document/src/vespa/document/predicate/predicate_printer.cpp
@@ -0,0 +1,101 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/log/log.h>
+LOG_SETUP(".predicate_printer");
+#include <vespa/fastos/fastos.h>
+
+#include "predicate.h"
+#include "predicate_printer.h"
+#include <vespa/document/util/stringutil.h>
+#include <vespa/vespalib/data/slime/slime.h>
+
+using vespalib::Slime;
+using vespalib::slime::Inspector;
+
+namespace document {
+
+namespace {
+void printEscapedString(vespalib::asciistream &out, const Inspector &in) {
+    out << "'" << StringUtil::escape(in.asString().make_string(), '\'') << "'";
+}
+}  // namespace
+
+void PredicatePrinter::visitFeatureSet(const Inspector &in) {
+    printEscapedString(_out, in[Predicate::KEY]);
+    if (_negated) {
+        _out << " not";
+    }
+    _out << " in [";
+    for (size_t i = 0; i < in[Predicate::SET].entries(); ++i) {
+        if (i) {
+            _out << ",";
+        }
+        printEscapedString(_out, in[Predicate::SET][i]);
+    }
+    _out << "]";
+}
+
+void PredicatePrinter::visitFeatureRange(const Inspector &in) {
+    printEscapedString(_out, in[Predicate::KEY]);
+    if (_negated) {
+        _out << " not";
+    }
+    bool has_min = in[Predicate::RANGE_MIN].valid();
+    bool has_max = in[Predicate::RANGE_MAX].valid();
+    _out << " in [";
+    if (has_min) _out << in[Predicate::RANGE_MIN].asLong();
+    _out << "..";
+    if (has_max) _out << in[Predicate::RANGE_MAX].asLong();
+    _out << "]";
+}
+
+void PredicatePrinter::visitNegation(const Inspector &in) {
+    bool n = _negated;
+    _negated = !_negated;
+    visitChildren(in);
+    _negated = n;
+}
+
+void PredicatePrinter::visitConjunction(const Inspector &in) {
+    if (_negated)
+        _out << "not ";
+    _negated = false;
+    _out << "(";
+    for (size_t i = 0; i < in[Predicate::CHILDREN].children(); ++i) {
+        if (i) {
+            _out << " and ";
+        }
+        visit(in[Predicate::CHILDREN][i]);
+    }
+    _out << ")";
+}
+
+void PredicatePrinter::visitDisjunction(const Inspector &in) {
+    if (_negated)
+        _out << "not ";
+    _negated = false;
+    _out << "(";
+    for (size_t i = 0; i < in[Predicate::CHILDREN].children(); ++i) {
+        if (i) {
+            _out << " or ";
+        }
+        visit(in[Predicate::CHILDREN][i]);
+    }
+    _out << ")";
+}
+
+void PredicatePrinter::visitTrue(const Inspector &) {
+    _out << "true";
+}
+
+void PredicatePrinter::visitFalse(const Inspector &) {
+    _out << "false";
+}
+
+vespalib::string PredicatePrinter::print(const Slime &slime) {
+    PredicatePrinter printer;
+    printer.visit(slime.get());
+    return printer.str();
+}
+
+}  // namespace document
diff --git a/document/src/vespa/document/predicate/predicate_printer.h b/document/src/vespa/document/predicate/predicate_printer.h
new file mode 100644
index 00000000000..9dd8b0fcf16
--- /dev/null
+++ b/document/src/vespa/document/predicate/predicate_printer.h
@@ -0,0 +1,32 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "predicate_slime_visitor.h"
+#include <vespa/vespalib/stllike/asciistream.h>
+
+namespace vespalib { class Slime; }
+
+namespace document {
+
+class PredicatePrinter : PredicateSlimeVisitor {
+    vespalib::asciistream _out;
+    bool _negated;
+
+    virtual void visitFeatureSet(const vespalib::slime::Inspector &i);
+    virtual void visitFeatureRange(const vespalib::slime::Inspector &i);
+    virtual void visitNegation(const vespalib::slime::Inspector &i);
+    virtual void visitConjunction(const vespalib::slime::Inspector &i);
+    virtual void visitDisjunction(const vespalib::slime::Inspector &i);
+    virtual void visitTrue(const vespalib::slime::Inspector &i);
+    virtual void visitFalse(const vespalib::slime::Inspector &i);
+
+    vespalib::string str() const { return _out.str(); }
+
+    PredicatePrinter() : _out(), _negated(false) {}
+public:
+    static vespalib::string print(const vespalib::Slime &slime);
+};
+
+}  // namespace document
+
diff --git a/document/src/vespa/document/predicate/predicate_slime_builder.cpp b/document/src/vespa/document/predicate/predicate_slime_builder.cpp
new file mode 100644
index 00000000000..18fa67d7342
--- /dev/null
+++ b/document/src/vespa/document/predicate/predicate_slime_builder.cpp
@@ -0,0 +1,203 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/log/log.h>
+LOG_SETUP(".predicate_slime_builder");
+#include <vespa/fastos/fastos.h>
+
+#include "predicate.h"
+#include "predicate_slime_builder.h"
+
+
+using vespalib::Slime;
+using vespalib::slime::ArrayInserter;
+using vespalib::slime::Cursor;
+
+namespace document {
+
+PredicateSlimeBuilder::PredicateSlimeBuilder()
+    : _slime(new Slime),
+      _cursor(&_slime->setObject()) {
+}
+
+PredicateSlimeBuilder &PredicateSlimeBuilder::feature(const std::string &key) {
+    _cursor->setString(Predicate::KEY, key);
+    return *this;
+}
+
+PredicateSlimeBuilder &PredicateSlimeBuilder::value(const std::string &val) {
+    _cursor->setLong(Predicate::NODE_TYPE, Predicate::TYPE_FEATURE_SET);
+    Cursor *arr = &(*_cursor)[Predicate::SET];
+    if (!arr->valid()) {
+        arr = &_cursor->setArray(Predicate::SET);
+    }
+    arr->addString(val);
+    return *this;
+}
+
+PredicateSlimeBuilder &PredicateSlimeBuilder::range(int64_t lower,
+                                                    int64_t upper) {
+    _cursor->setLong(Predicate::NODE_TYPE, Predicate::TYPE_FEATURE_RANGE);
+    _cursor->setLong(Predicate::RANGE_MIN, lower);
+    _cursor->setLong(Predicate::RANGE_MAX, upper);
+    return *this;
+}
+
+PredicateSlimeBuilder &PredicateSlimeBuilder::greaterEqual(int64_t lower) {
+    _cursor->setLong(Predicate::NODE_TYPE, Predicate::TYPE_FEATURE_RANGE);
+    _cursor->setLong(Predicate::RANGE_MIN, lower);
+    return *this;
+}
+
+PredicateSlimeBuilder &PredicateSlimeBuilder::lessEqual(int64_t upper) {
+    _cursor->setLong(Predicate::NODE_TYPE, Predicate::TYPE_FEATURE_RANGE);
+    _cursor->setLong(Predicate::RANGE_MAX, upper);
+    return *this;
+}
+
+PredicateSlimeBuilder &PredicateSlimeBuilder::neg() {
+    _cursor->setLong(Predicate::NODE_TYPE, Predicate::TYPE_NEGATION);
+    _cursor = &_cursor->setArray(Predicate::CHILDREN).addObject();
+    return *this;
+}
+
+namespace {
+template <typename InitList>
+void intermediateNode(long type, InitList list, Cursor &cursor) {
+    cursor.setLong(Predicate::NODE_TYPE, type);
+    Cursor &arr = cursor.setArray(Predicate::CHILDREN);
+    for (auto it = list.begin(); it != list.end(); ++it) {
+        inject((*it)->get(), ArrayInserter(arr));
+    }
+}
+}  // namespace
+
+PredicateSlimeBuilder &PredicateSlimeBuilder::and_node(
+        std::initializer_list<SlimeUP> list) {
+    intermediateNode(Predicate::TYPE_CONJUNCTION, list, *_cursor);
+    return *this;
+}
+
+PredicateSlimeBuilder &PredicateSlimeBuilder::or_node(
+        std::initializer_list<SlimeUP> list) {
+    intermediateNode(Predicate::TYPE_DISJUNCTION, list, *_cursor);
+    return *this;
+}
+
+PredicateSlimeBuilder &PredicateSlimeBuilder::and_node(SlimeUP s1,
+                                                       SlimeUP s2) {
+    return and_node({std::move(s1), std::move(s2)});
+}
+
+PredicateSlimeBuilder &PredicateSlimeBuilder::or_node(SlimeUP s1, SlimeUP s2) {
+    return or_node({std::move(s1), std::move(s2)});
+}
+
+PredicateSlimeBuilder &PredicateSlimeBuilder::true_predicate() {
+    _cursor->setLong(Predicate::NODE_TYPE, Predicate::TYPE_TRUE);
+    return *this;
+}
+
+PredicateSlimeBuilder &PredicateSlimeBuilder::false_predicate() {
+    _cursor->setLong(Predicate::NODE_TYPE, Predicate::TYPE_FALSE);
+    return *this;
+}
+
+std::unique_ptr<vespalib::Slime> PredicateSlimeBuilder::build() {
+    std::unique_ptr<Slime> s = std::move(_slime);
+    _slime.reset(new Slime);
+    _cursor = &_slime->setObject();
+    return s;
+}
+
+namespace predicate_slime_builder {
+
+SlimeUP featureSet(const std::string &key,
+                   const std::initializer_list<std::string> &values) {
+    SlimeUP slime(new Slime);
+    Cursor &cursor = slime->setObject();
+    cursor.setLong(Predicate::NODE_TYPE, Predicate::TYPE_FEATURE_SET);
+    cursor.setString(Predicate::KEY, key);
+    Cursor &arr = cursor.setArray(Predicate::SET);
+    for (const auto &value : values) {
+        arr.addString(value);
+    }
+    return slime;
+}
+
+SlimeUP emptyRange(const std::string &key) {
+    SlimeUP slime(new Slime);
+    Cursor &cursor = slime->setObject();
+    cursor.setLong(Predicate::NODE_TYPE, Predicate::TYPE_FEATURE_RANGE);
+    cursor.setString(Predicate::KEY, key);
+    return slime;
+}
+
+SlimeUP featureRange(const std::string &key, int64_t lower, int64_t upper) {
+    SlimeUP slime = emptyRange(key);
+    Cursor &cursor = slime->get();
+    cursor.setLong(Predicate::RANGE_MIN, lower);
+    cursor.setLong(Predicate::RANGE_MAX, upper);
+    return slime;
+}
+
+SlimeUP greaterEqual(const std::string &key, int64_t lower) {
+    SlimeUP slime = emptyRange(key);
+    Cursor &cursor = slime->get();
+    cursor.setLong(Predicate::RANGE_MIN, lower);
+    return slime;
+}
+
+SlimeUP lessEqual(const std::string &key, int64_t upper) {
+    SlimeUP slime = emptyRange(key);
+    Cursor &cursor = slime->get();
+    cursor.setLong(Predicate::RANGE_MAX, upper);
+    return slime;
+}
+
+SlimeUP neg(SlimeUP child) {
+    SlimeUP slime(new Slime);
+    Cursor &cursor = slime->setObject();
+    cursor.setLong(Predicate::NODE_TYPE, Predicate::TYPE_NEGATION);
+    Cursor &arr = cursor.setArray(Predicate::CHILDREN);
+    inject(child->get(), ArrayInserter(arr));
+    return slime;
+}
+
+SlimeUP andNode(const std::initializer_list<SlimeUP> &children) {
+    SlimeUP slime(new Slime);
+    Cursor &cursor = slime->setObject();
+    cursor.setLong(Predicate::NODE_TYPE, Predicate::TYPE_CONJUNCTION);
+    Cursor &arr = cursor.setArray(Predicate::CHILDREN);
+    for (const auto &child : children) {
+        inject(child->get(), ArrayInserter(arr));
+    }
+    return slime;
+}
+
+SlimeUP orNode(const std::initializer_list<SlimeUP> &children) {
+    SlimeUP slime(new Slime);
+    Cursor &cursor = slime->setObject();
+    cursor.setLong(Predicate::NODE_TYPE, Predicate::TYPE_DISJUNCTION);
+    Cursor &arr = cursor.setArray(Predicate::CHILDREN);
+    for (const auto &child : children) {
+        inject(child->get(), ArrayInserter(arr));
+    }
+    return slime;
+}
+
+SlimeUP truePredicate() {
+    SlimeUP slime(new Slime);
+    Cursor &cursor = slime->setObject();
+    cursor.setLong(Predicate::NODE_TYPE, Predicate::TYPE_TRUE);
+    return slime;
+}
+
+SlimeUP falsePredicate() {
+    SlimeUP slime(new Slime);
+    Cursor &cursor = slime->setObject();
+    cursor.setLong(Predicate::NODE_TYPE, Predicate::TYPE_FALSE);
+    return slime;
+}
+
+}  // namespace predicate_slime_builder
+}  // namespace document
diff --git a/document/src/vespa/document/predicate/predicate_slime_builder.h b/document/src/vespa/document/predicate/predicate_slime_builder.h
new file mode 100644
index 00000000000..6ab6f6367dc
--- /dev/null
+++ b/document/src/vespa/document/predicate/predicate_slime_builder.h
@@ -0,0 +1,57 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/vespalib/data/slime/slime.h>
+#include <string>
+
+namespace document {
+
+class PredicateSlimeBuilder {
+    std::unique_ptr<vespalib::Slime> _slime;
+    vespalib::slime::Cursor *_cursor;
+
+    typedef std::unique_ptr<vespalib::Slime> SlimeUP;
+
+public:
+    PredicateSlimeBuilder();
+
+    PredicateSlimeBuilder &feature(const std::string &key);
+    PredicateSlimeBuilder &value(const std::string &key);
+    PredicateSlimeBuilder &range(int64_t lower, int64_t upper);
+    PredicateSlimeBuilder &greaterEqual(int64_t lower);
+    PredicateSlimeBuilder &lessEqual(int64_t upper);
+    PredicateSlimeBuilder &neg();
+    PredicateSlimeBuilder &and_node(std::initializer_list<SlimeUP> list);
+    PredicateSlimeBuilder &or_node(std::initializer_list<SlimeUP> list);
+
+    PredicateSlimeBuilder &and_node(SlimeUP s1, SlimeUP s2);
+    PredicateSlimeBuilder &or_node(SlimeUP s1, SlimeUP s2);
+    PredicateSlimeBuilder &true_predicate();
+    PredicateSlimeBuilder &false_predicate();
+    SlimeUP build();
+
+    // for converting builders to slime objects in initializer lists.
+    operator SlimeUP() { return build(); }
+};
+
+namespace predicate_slime_builder {
+
+typedef std::unique_ptr<vespalib::Slime> SlimeUP;
+
+SlimeUP featureSet(const std::string &key,
+                   const std::initializer_list<std::string> &values);
+SlimeUP featureRange(const std::string &key, int64_t lower, int64_t upper);
+SlimeUP greaterEqual(const std::string &key, int64_t lower);
+SlimeUP lessEqual(const std::string &key, int64_t upper);
+SlimeUP emptyRange(const std::string &key);
+SlimeUP neg(SlimeUP child);
+SlimeUP andNode(const std::initializer_list<SlimeUP> &children);
+SlimeUP orNode(const std::initializer_list<SlimeUP> &children);
+SlimeUP truePredicate();
+SlimeUP falsePredicate();
+
+}  // namespace predicate_slime_builder
+
+}  // namespace document
+
diff --git a/document/src/vespa/document/predicate/predicate_slime_visitor.cpp b/document/src/vespa/document/predicate/predicate_slime_visitor.cpp
new file mode 100644
index 00000000000..cddd2a1c2c7
--- /dev/null
+++ b/document/src/vespa/document/predicate/predicate_slime_visitor.cpp
@@ -0,0 +1,34 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/log/log.h>
+LOG_SETUP(".predicate_slime_visitor");
+#include <vespa/fastos/fastos.h>
+
+#include "predicate.h"
+#include "predicate_slime_visitor.h"
+#include <vespa/vespalib/data/slime/inspector.h>
+
+using vespalib::slime::Inspector;
+
+namespace document {
+
+void PredicateSlimeVisitor::visit(const Inspector &inspector) {
+    switch (inspector[Predicate::NODE_TYPE].asLong()) {
+    case Predicate::TYPE_CONJUNCTION: visitConjunction(inspector); break;
+    case Predicate::TYPE_DISJUNCTION: visitDisjunction(inspector); break;
+    case Predicate::TYPE_NEGATION: visitNegation(inspector); break;
+    case Predicate::TYPE_FEATURE_SET: visitFeatureSet(inspector); break;
+    case Predicate::TYPE_FEATURE_RANGE: visitFeatureRange(inspector); break;
+    case Predicate::TYPE_TRUE: visitTrue(inspector); break;
+    case Predicate::TYPE_FALSE: visitFalse(inspector); break;
+    default: break;
+    }
+}
+
+void PredicateSlimeVisitor::visitChildren(const Inspector &inspector) {
+    for (size_t i = 0; i < inspector[Predicate::CHILDREN].children(); ++i) {
+        visit(inspector[Predicate::CHILDREN][i]);
+    }
+}
+
+}  // namespace document
diff --git a/document/src/vespa/document/predicate/predicate_slime_visitor.h b/document/src/vespa/document/predicate/predicate_slime_visitor.h
new file mode 100644
index 00000000000..c35a9adb666
--- /dev/null
+++ b/document/src/vespa/document/predicate/predicate_slime_visitor.h
@@ -0,0 +1,30 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+namespace vespalib {
+namespace slime { class Inspector; }
+}  // namespace vespalib
+
+namespace document {
+
+class PredicateSlimeVisitor {
+    virtual void visitFeatureSet(const vespalib::slime::Inspector &i) = 0;
+    virtual void visitFeatureRange(const vespalib::slime::Inspector &i) = 0;
+    virtual void visitNegation(const vespalib::slime::Inspector &i) = 0;
+    virtual void visitConjunction(const vespalib::slime::Inspector &i) = 0;
+    virtual void visitDisjunction(const vespalib::slime::Inspector &i) = 0;
+    virtual void visitTrue(const vespalib::slime::Inspector &i) = 0;
+    virtual void visitFalse(const vespalib::slime::Inspector &i) = 0;
+
+protected:
+    void visitChildren(const vespalib::slime::Inspector &i);
+
+public:
+    virtual ~PredicateSlimeVisitor() {}
+
+    void visit(const vespalib::slime::Inspector &i);
+};
+
+}  // namespace document
+
diff --git a/document/src/vespa/document/repo/.gitignore b/document/src/vespa/document/repo/.gitignore
new file mode 100644
index 00000000000..583460ae288
--- /dev/null
+++ b/document/src/vespa/document/repo/.gitignore
@@ -0,0 +1,3 @@
+*.So
+.depend
+Makefile
diff --git a/document/src/vespa/document/repo/CMakeLists.txt b/document/src/vespa/document/repo/CMakeLists.txt
new file mode 100644
index 00000000000..e8e6c6a0141
--- /dev/null
+++ b/document/src/vespa/document/repo/CMakeLists.txt
@@ -0,0 +1,10 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_library(document_repo OBJECT
+    SOURCES
+    configbuilder.cpp
+    documenttyperepo.cpp
+    fixedtyperepo.cpp
+    DEPENDS
+    AFTER
+    document_documentconfig
+)
diff --git a/document/src/vespa/document/repo/configbuilder.cpp b/document/src/vespa/document/repo/configbuilder.cpp
new file mode 100644
index 00000000000..9b1701b8f41
--- /dev/null
+++ b/document/src/vespa/document/repo/configbuilder.cpp
@@ -0,0 +1,20 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".configbuilder");
+
+#include "configbuilder.h"
+
+namespace document {
+namespace config_builder {
+int32_t createFieldId(const vespalib::string &name, int32_t type, int ver) {
+    StructDataType dummy("dummy", type);
+    Field f(name, dummy, true);
+    return f.getId(ver);
+}
+
+int32_t DatatypeConfig::id_counter = 100;
+
+}  // namespace config_builder
+}  // namespace document
diff --git a/document/src/vespa/document/repo/configbuilder.h b/document/src/vespa/document/repo/configbuilder.h
new file mode 100644
index 00000000000..d933a05cbe5
--- /dev/null
+++ b/document/src/vespa/document/repo/configbuilder.h
@@ -0,0 +1,170 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/document/base/field.h>
+#include <vespa/document/config/config-documenttypes.h>
+#include <vespa/document/datatype/datatype.h>
+#include <vespa/document/datatype/structdatatype.h>
+#include <vespa/vespalib/stllike/string.h>
+
+namespace document {
+namespace config_builder {
+
+class TypeOrId;
+
+struct DatatypeConfig : DocumenttypesConfig::Documenttype::Datatype {
+    static int32_t id_counter;
+    std::vector<DatatypeConfig> nested_types;
+
+    DatatypeConfig() {
+        id = ++id_counter;
+    }
+    DatatypeConfig &setId(int32_t i) { id = i; return *this; }
+    void addNestedType(const TypeOrId &t);
+};
+
+int32_t createFieldId(const vespalib::string &name, int32_t type, int ver);
+
+struct TypeOrId {
+    int32_t id;
+    bool has_type;
+    DatatypeConfig type;
+
+    TypeOrId(int32_t i) : id(i), has_type(false), type() {}
+    TypeOrId(const DatatypeConfig &t) : id(t.id), has_type(true), type(t) {}
+};
+
+inline void DatatypeConfig::addNestedType(const TypeOrId &t) {
+    if (t.has_type) {
+        nested_types.insert(nested_types.end(),
+                            t.type.nested_types.begin(),
+                            t.type.nested_types.end());
+        nested_types.push_back(t.type);
+    }
+}
+
+struct Struct : DatatypeConfig {
+    Struct(const vespalib::string &name) {
+        type = STRUCT;
+        sstruct.name = name;
+    }
+    Struct &setCompression(Sstruct::Compression::Type t, int32_t level,
+                           int32_t threshold, int32_t min_size) {
+        sstruct.compression.type = t;
+        sstruct.compression.level = level;
+        sstruct.compression.threshold = threshold;
+        sstruct.compression.minsize = min_size;
+        return *this;
+    }
+    Struct &addField(const vespalib::string &name, TypeOrId data_type) {
+        addNestedType(data_type);
+        sstruct.field.resize(sstruct.field.size() + 1);
+        sstruct.field.back().name = name;
+        sstruct.field.back().id = createFieldId(name, data_type.id, 7);
+        sstruct.field.back().idV6 = createFieldId(name, data_type.id, 6);
+        sstruct.field.back().datatype = data_type.id;
+        return *this;
+    }
+    Struct &setId(int32_t i) { DatatypeConfig::setId(i); return *this; }
+};
+
+struct Array : DatatypeConfig {
+    Array(TypeOrId nested_type) {
+        addNestedType(nested_type);
+        type = ARRAY;
+        array.element.id = nested_type.id;
+    }
+};
+
+struct Wset : DatatypeConfig {
+    Wset(TypeOrId nested_type) {
+        addNestedType(nested_type);
+        type = WSET;
+        wset.key.id = nested_type.id;
+    }
+    Wset &removeIfZero() { wset.removeifzero = true; return *this; }
+    Wset &createIfNonExistent() {
+        wset.createifnonexistent = true;
+        return *this;
+    }
+};
+
+struct Map : DatatypeConfig {
+    Map(TypeOrId key_type, TypeOrId value_type) {
+        addNestedType(key_type);
+        addNestedType(value_type);
+        type = MAP;
+        map.key.id = key_type.id;
+        map.value.id = value_type.id;
+    }
+};
+
+struct AnnotationRef : DatatypeConfig {
+    AnnotationRef(int32_t annotation_type_id) {
+        type = ANNOTATIONREF;
+        annotationref.annotation.id = annotation_type_id;
+    }
+};
+
+inline void addType(const DatatypeConfig &type,
+                    DocumenttypesConfig::Documenttype &doc_type) {
+    doc_type.datatype.insert(doc_type.datatype.end(),
+                             type.nested_types.begin(),
+                             type.nested_types.end());
+    doc_type.datatype.push_back(type);
+}
+
+struct DocTypeRep {
+    DocumenttypesConfig::Documenttype &doc_type;
+
+    DocTypeRep(DocumenttypesConfig::Documenttype &type)
+        : doc_type(type) {}
+    DocTypeRep &inherit(int32_t id) {
+        doc_type.inherits.resize(doc_type.inherits.size() + 1);
+        doc_type.inherits.back().id = id;
+        return *this;
+    }
+    DocTypeRep &annotationType(int32_t id, const vespalib::string &name,
+                               int32_t datatype) {
+        doc_type.annotationtype.resize(doc_type.annotationtype.size() + 1);
+        doc_type.annotationtype.back().id = id;
+        doc_type.annotationtype.back().name = name;
+        doc_type.annotationtype.back().datatype = datatype;
+        return *this;
+    }
+    DocTypeRep &annotationType(int32_t id, const vespalib::string &name,
+                               const DatatypeConfig &type) {
+        addType(type, doc_type);
+        return annotationType(id, name, type.id);
+    }
+};
+
+class DocumenttypesConfigBuilderHelper {
+    ::document::DocumenttypesConfigBuilder _config;
+
+public:
+    DocumenttypesConfigBuilderHelper() {}
+    DocumenttypesConfigBuilderHelper(const DocumenttypesConfig &c)
+        : _config(c) {}
+
+    DocTypeRep document(int32_t id, const vespalib::string &name,
+                        const DatatypeConfig &header,
+                        const DatatypeConfig &body) {
+        assert(header.type == DatatypeConfig::STRUCT);
+        assert(body.type == DatatypeConfig::STRUCT);
+        _config.documenttype.resize(_config.documenttype.size() + 1);
+        _config.documenttype.back().id = id;
+        _config.documenttype.back().name = name;
+        _config.documenttype.back().headerstruct = header.id;
+        _config.documenttype.back().bodystruct = body.id;
+        addType(header, _config.documenttype.back());
+        addType(body, _config.documenttype.back());
+        return DocTypeRep(_config.documenttype.back());
+    }
+
+    ::document::DocumenttypesConfigBuilder &config() { return _config; }
+};
+}  // namespace config_builder
+}  // namespace document
+
diff --git a/document/src/vespa/document/repo/documenttyperepo.cpp b/document/src/vespa/document/repo/documenttyperepo.cpp
new file mode 100644
index 00000000000..fd0feb35064
--- /dev/null
+++ b/document/src/vespa/document/repo/documenttyperepo.cpp
@@ -0,0 +1,589 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".documenttyperepo");
+
+#include "documenttyperepo.h"
+
+#include <vespa/document/datatype/annotationreferencedatatype.h>
+#include <vespa/document/datatype/annotationtype.h>
+#include <vespa/document/datatype/arraydatatype.h>
+#include <vespa/document/datatype/documenttype.h>
+#include <vespa/document/datatype/mapdatatype.h>
+#include <vespa/document/datatype/positiondatatype.h>
+#include <vespa/document/datatype/urldatatype.h>
+#include <vespa/document/datatype/weightedsetdatatype.h>
+#include <vespa/vespalib/objects/identifiable.h>
+#include <vespa/vespalib/stllike/hash_map.h>
+#include <vespa/vespalib/util/closure.h>
+#include <vespa/vespalib/util/exceptions.h>
+#include <vespa/vespalib/util/stringfmt.h>
+#include <vespa/config/print/fileconfigreader.h>
+#include <fstream>
+#include <memory>
+#include <utility>
+
+using std::unique_ptr;
+using std::fstream;
+using std::make_pair;
+using std::pair;
+using std::vector;
+using vespalib::Closure1;
+using vespalib::Identifiable;
+using vespalib::IllegalArgumentException;
+using vespalib::hash_map;
+using vespalib::make_string;
+using vespalib::string;
+using vespalib::stringref;
+
+namespace document {
+
+namespace {
+template <typename Container>
+void DeleteContent(Container &c) {
+    for (typename Container::iterator it = c.begin(); it != c.end(); ++it) {
+        delete *it;
+    }
+}
+template <typename Map>
+void DeleteMapContent(Map &m) {
+    for (typename Map::iterator it = m.begin(); it != m.end(); ++it) {
+        delete it->second;
+    }
+}
+
+// A collection of data types.
+class Repo {
+    vector<const DataType *> _owned_types;
+    hash_map<int32_t, const DataType *> _types;
+    hash_map<string, const DataType *> _name_map;
+
+public:
+    ~Repo() { DeleteContent(_owned_types); }
+
+    void inherit(const Repo &parent);
+    bool addDataType(const DataType &type);
+    template <typename T> void addDataType(unique_ptr<T> type);
+
+    const DataType *lookup(int32_t id) const;
+    const DataType *lookup(const stringref &name) const;
+    const DataType &findOrThrow(int32_t id) const;
+};
+
+void Repo::inherit(const Repo &parent) {
+    _types.insert(parent._types.begin(), parent._types.end());
+    _name_map.insert(parent._name_map.begin(), parent._name_map.end());
+}
+
+// Returns true if a reference to type is stored.
+bool Repo::addDataType(const DataType &type) {
+    const DataType *& data_type = _types[type.getId()];
+    if (data_type) {
+        if (*data_type == type) {
+            return false;  // Redefinition of identical type is ok.
+        }
+        throw IllegalArgumentException(
+                make_string("Redefinition of data type %d, \"%s\". "
+                            "Previously defined as \"%s\".",
+                            type.getId(), type.getName().c_str(),
+                            data_type->getName().c_str()));
+    }
+    const DataType *& data_type_by_name = _name_map[type.getName()];
+    if (data_type_by_name) {
+        throw IllegalArgumentException(
+                make_string("Redefinition of data type \"%s\", with id %d."
+                            " Previously defined with id %d.",
+                            type.getName().c_str(), type.getId(),
+                            data_type_by_name->getId()));
+    }
+    data_type = &type;
+    data_type_by_name = &type;
+    return true;
+}
+
+template <typename T>
+void Repo::addDataType(unique_ptr<T> type) {
+    if (addDataType(*type)) {
+        _owned_types.push_back(type.release());
+    }
+}
+
+template <typename Map>
+typename Map::mapped_type FindPtr(const Map &m, typename Map::key_type key) {
+    typename Map::const_iterator it = m.find(key);
+    if (it != m.end()) {
+        return it->second;
+    }
+    return typename Map::mapped_type();
+}
+
+const DataType *Repo::lookup(int32_t id) const {
+    return FindPtr(_types, id);
+}
+
+const DataType *Repo::lookup(const stringref &n) const {
+    return FindPtr(_name_map, n);
+}
+
+const DataType &Repo::findOrThrow(int32_t id) const {
+    const DataType *type = lookup(id);
+    if (type) {
+        return *type;
+    }
+    throw IllegalArgumentException(make_string("Unknown datatype %d", id));
+}
+
+class AnnotationTypeRepo {
+    vector<const AnnotationType *> _owned_types;
+    hash_map<int32_t, AnnotationType *> _annotation_types;
+
+public:
+    ~AnnotationTypeRepo() { DeleteContent(_owned_types); }
+
+    void inherit(const AnnotationTypeRepo &parent);
+    void addAnnotationType(AnnotationType::UP annotation_type);
+    void setAnnotationDataType(int32_t id, const DataType &datatype);
+
+    const AnnotationType *lookup(int32_t id) const;
+};
+
+void AnnotationTypeRepo::inherit(const AnnotationTypeRepo &parent) {
+    _annotation_types.insert(parent._annotation_types.begin(),
+                             parent._annotation_types.end());
+}
+
+void AnnotationTypeRepo::addAnnotationType(AnnotationType::UP type) {
+    AnnotationType *& a_type = _annotation_types[type->getId()];
+    if (a_type) {
+        if (*type != *a_type) {
+            throw IllegalArgumentException(
+                make_string("Redefinition of annotation type %d, \"%s\". "
+                            "Previously defined as \"%s\".",
+                            type->getId(), type->getName().c_str(),
+                            a_type->getName().c_str()));
+        }
+    } else {
+        a_type = type.get();
+        _owned_types.push_back(type.release());
+    }
+}
+
+void AnnotationTypeRepo::setAnnotationDataType(int32_t id, const DataType &d) {
+    AnnotationType *annotation_type = _annotation_types[id];
+    assert(annotation_type);
+    if (!annotation_type->getDataType()) {
+        annotation_type->setDataType(d);
+    } else if (*(annotation_type->getDataType()) != d) {
+        throw IllegalArgumentException(
+            make_string("Redefinition of annotation type %d, \"%s\" = '%s'. "
+                        "Previously defined as '%s'.",
+                        annotation_type->getId(),
+                        annotation_type->getName().c_str(),
+                        annotation_type->getDataType()->toString().c_str(),
+                        d.toString().c_str()));
+    }
+}
+
+const AnnotationType *AnnotationTypeRepo::lookup(int32_t id) const {
+    return FindPtr(_annotation_types, id);
+}
+
+}  // namespace
+
+// Combination of a DocumentType and a collection of DataTypes
+// associated with it.
+struct DataTypeRepo {
+    typedef unique_ptr<DataTypeRepo> UP;
+
+    DocumentType *doc_type;
+    Repo repo;
+    AnnotationTypeRepo annotations;
+
+    DataTypeRepo() : doc_type(0) {}
+    ~DataTypeRepo() { delete doc_type; }
+};
+
+namespace {
+void addAnnotationType(
+        const DocumenttypesConfig::Documenttype::Annotationtype &type,
+        AnnotationTypeRepo &annotations) {
+    AnnotationType::UP a(new AnnotationType(type.id, type.name));
+    annotations.addAnnotationType(std::move(a));
+}
+
+void addAnnotationTypes(
+        const vector<DocumenttypesConfig::Documenttype::Annotationtype> &types,
+        AnnotationTypeRepo &annotations) {
+    for (size_t i = 0; i < types.size(); ++i) {
+        addAnnotationType(types[i], annotations);
+    }
+}
+
+void setAnnotationDataTypes(
+        const vector<DocumenttypesConfig::Documenttype::Annotationtype> &types,
+        AnnotationTypeRepo &annotations, const Repo &repo) {
+    for (size_t i = 0; i < types.size(); ++i) {
+        if (types[i].datatype == -1) {
+            continue;
+        }
+        const DataType &datatype = repo.findOrThrow(types[i].datatype);
+        annotations.setAnnotationDataType(types[i].id, datatype);
+    }
+}
+
+typedef DocumenttypesConfig::Documenttype::Datatype Datatype;
+
+void addField(const Datatype::Sstruct::Field &field, const Repo &repo,
+              StructDataType &struct_type, bool isHeaderField) {
+    LOG(spam, "Adding field %s to %s (header: %s)",
+        field.name.c_str(), struct_type.getName().c_str(),
+        isHeaderField ? "yes" : "no");
+    const DataType &field_type = repo.findOrThrow(field.datatype);
+    struct_type.addField(Field(field.name, field.id, field.idV6,
+                               field_type, isHeaderField));
+}
+
+bool hasSuffix(const string &s, const string &suffix) {
+    string::size_type pos = s.rfind(suffix.c_str());
+    return pos != string::npos && pos == s.size() - suffix.size();
+}
+
+void addStruct(int32_t id, const Datatype::Sstruct &s, Repo &repo) {
+    // TODO(thomasg): Ugly stuff, remove when we fix config.
+    std::string name = s.name;
+    std::string::size_type pos = name.rfind(".body");
+    bool useUglyStructHack = false;
+    if (pos != std::string::npos) {
+        name = name.substr(0, pos) + ".header";
+        // If header already exists, we'll just reuse its struct verbatim so no
+        // need to set new ID here.
+        useUglyStructHack = true;
+    } else if (name.rfind(".header") != std::string::npos) {
+        const DataType *existing = repo.lookup(name);
+        if (existing) {
+            LOG(spam, "Reusing id %u from body struct since its fields "
+                "have already been inserted",
+                existing->getId());
+            id = existing->getId();
+        }
+        useUglyStructHack = true;
+    }
+
+    LOG(debug, "Adding struct type %s (%s) with id %u",
+        s.name.c_str(), name.c_str(), id);
+
+    StructDataType::UP struct_type_ap;
+    StructDataType *struct_type;
+    const DataType *existing = repo.lookup(name);
+    if (useUglyStructHack && existing) {
+        LOG(spam, "Type %s already existed", name.c_str());
+        const StructDataType& cdt =
+            Identifiable::cast<const StructDataType&>(*existing);
+        struct_type = const_cast<StructDataType*>(&cdt);
+    } else {
+        const DataType *existing_retry = repo.lookup(id);
+        LOG(spam, "Type %s not found, adding it", name.c_str());
+        struct_type_ap.reset(new StructDataType(name, id));
+        struct_type = struct_type_ap.get();
+        repo.addDataType(std::move(struct_type_ap));
+        if (existing_retry) {
+            return;
+        }
+    }
+
+    CompressionConfig::Type type = CompressionConfig::NONE;
+    if (s.compression.type == Datatype::Sstruct::Compression::LZ4) {
+        type = CompressionConfig::LZ4;
+    }
+
+    struct_type->setCompressionConfig(
+            CompressionConfig(type, s.compression.level,
+                              s.compression.threshold, s.compression.minsize));
+
+    for (size_t i = 0; i < s.field.size(); ++i) {
+        addField(s.field[i], repo, *struct_type, hasSuffix(s.name, ".header"));
+    }
+}
+
+void addArray(int32_t id, const Datatype::Array &a, Repo &repo) {
+    const DataType &nested = repo.findOrThrow(a.element.id);
+    repo.addDataType(DataType::UP(new ArrayDataType(nested, id)));
+}
+
+void addWset(int32_t id, const Datatype::Wset &w, Repo &repo) {
+    const DataType &key = repo.findOrThrow(w.key.id);
+    repo.addDataType(DataType::UP(new WeightedSetDataType(
+                            key, w.createifnonexistent, w.removeifzero, id)));
+}
+
+void addMap(int32_t id, const Datatype::Map &m, Repo &repo) {
+    const DataType &key = repo.findOrThrow(m.key.id);
+    const DataType &value = repo.findOrThrow(m.value.id);
+    repo.addDataType(DataType::UP(new MapDataType(key, value, id)));
+}
+
+void addAnnotationRef(int32_t id, const Datatype::Annotationref &a, Repo &r,
+                      const AnnotationTypeRepo &annotations) {
+    const AnnotationType *type = annotations.lookup(a.annotation.id);
+    if (!type) {
+        throw IllegalArgumentException(
+                make_string("Unknown AnnotationType %d", a.annotation.id));
+    }
+    r.addDataType(DataType::UP(new AnnotationReferenceDataType(*type, id)));
+}
+
+void addDataType(const Datatype &type, Repo &repo,
+                 const AnnotationTypeRepo &a_repo) {
+    switch (type.type) {
+    case Datatype::STRUCT:
+        return addStruct(type.id, type.sstruct, repo);
+    case Datatype::ARRAY:
+        return addArray(type.id, type.array, repo);
+    case Datatype::WSET:
+        return addWset(type.id, type.wset, repo);
+    case Datatype::MAP:
+        return addMap(type.id, type.map, repo);
+    case Datatype::ANNOTATIONREF:
+        return addAnnotationRef(type.id, type.annotationref, repo, a_repo);
+    default:
+        throw IllegalArgumentException(
+                make_string("Unknown datatype type %d for id %d",
+                            type.type, type.id));
+    }
+}
+
+void addDataTypes(const vector<Datatype> &types, Repo &repo,
+                  const AnnotationTypeRepo &a_repo) {
+    for (size_t i = 0; i < types.size(); ++i) {
+        addDataType(types[i], repo, a_repo);
+    }
+}
+
+typedef hash_map<int32_t, DataTypeRepo *> DocumentTypeMap;
+void addDocumentTypes(const DocumentTypeMap &type_map, Repo &repo) {
+    for (DocumentTypeMap::const_iterator
+             it = type_map.begin(); it != type_map.end(); ++it) {
+        repo.addDataType(*it->second->doc_type);
+    }
+}
+
+void addDefaultDocument(DocumentTypeMap &type_map) {
+    DataTypeRepo::UP data_types(new DataTypeRepo);
+    vector<const DataType *> default_types = DataType::getDefaultDataTypes();
+    for (size_t i = 0; i < default_types.size(); ++i) {
+        data_types->repo.addDataType(*default_types[i]);
+    }
+    data_types->repo.addDataType(UrlDataType::getInstance());
+    data_types->repo.addDataType(PositionDataType::getInstance());
+    data_types->doc_type = new DocumentType("document", 8);
+
+    vector<const AnnotationType *> annotation_types(
+            AnnotationType::getDefaultAnnotationTypes());
+    for(size_t i(0); i < annotation_types.size(); ++i) {
+        data_types->annotations.addAnnotationType(
+                AnnotationType::UP(new AnnotationType(*annotation_types[i])));
+    }
+
+    type_map[data_types->doc_type->getId()] = data_types.release();
+}
+
+const DataTypeRepo &lookupRepo(int32_t id, const DocumentTypeMap &type_map) {
+    DocumentTypeMap::const_iterator it = type_map.find(id);
+    if (it == type_map.end()) {
+        throw IllegalArgumentException(
+                make_string("Unable to find document type %d.", id));
+    }
+    return *it->second;
+}
+
+void inheritDataTypes(
+        const vector<DocumenttypesConfig::Documenttype::Inherits> &base_types,
+        const DocumentTypeMap &type_map, Repo &repo) {
+    repo.inherit(lookupRepo(DataType::T_DOCUMENT, type_map).repo);
+    for (size_t i = 0; i < base_types.size(); ++i) {
+        repo.inherit(lookupRepo(base_types[i].id, type_map).repo);
+    }
+}
+
+void inheritAnnotationTypes(
+        const vector<DocumenttypesConfig::Documenttype::Inherits> &base_types,
+        const DocumentTypeMap &type_map, AnnotationTypeRepo &repo) {
+    repo.inherit(lookupRepo(DataType::T_DOCUMENT, type_map).annotations);
+    for (size_t i = 0; i < base_types.size(); ++i) {
+        repo.inherit(lookupRepo(base_types[i].id, type_map).annotations);
+    }
+}
+
+void inheritDocumentTypes(
+        const vector<DocumenttypesConfig::Documenttype::Inherits> &base_types,
+        const DocumentTypeMap &type_map, DocumentType &doc_type) {
+    for (size_t i = 0; i < base_types.size(); ++i) {
+        const DataTypeRepo &parent = lookupRepo(base_types[i].id, type_map);
+        doc_type.inherit(*parent.doc_type);
+    }
+}
+
+DataTypeRepo::UP makeDataTypeRepo(
+        const DocumentType &doc_type,
+        const DocumentTypeMap &type_map) {
+    DataTypeRepo::UP data_types(new DataTypeRepo);
+    data_types->repo.inherit(lookupRepo(DataType::T_DOCUMENT, type_map).repo);
+    data_types->annotations.inherit(
+            lookupRepo(DataType::T_DOCUMENT, type_map).annotations);
+    data_types->doc_type = doc_type.clone();
+    return data_types;
+}
+
+void addFieldSet(const DocumenttypesConfig::Documenttype::FieldsetsMap & fsv, DocumentType &doc_type) {
+    for (DocumenttypesConfig::Documenttype::FieldsetsMap::const_iterator it(fsv.begin()), mt(fsv.end()); it != mt; it++) {
+        const DocumenttypesConfig::Documenttype::Fieldsets & fs(it->second);
+        DocumentType::FieldSet::Fields fields;
+        for (size_t j(0); j < fs.fields.size(); j++) {
+            fields.insert(fs.fields[j]);
+        }
+        doc_type.addFieldSet(it->first, fields);
+    }
+}
+
+void configureDataTypeRepo(
+        const DocumenttypesConfig::Documenttype &doc_type,
+        DocumentTypeMap &type_map) {
+    DataTypeRepo *data_types = type_map[doc_type.id];
+    inheritAnnotationTypes(
+            doc_type.inherits, type_map, data_types->annotations);
+    addAnnotationTypes(doc_type.annotationtype, data_types->annotations);
+    inheritDataTypes(doc_type.inherits, type_map, data_types->repo);
+    addDataTypes(doc_type.datatype, data_types->repo, data_types->annotations);
+    setAnnotationDataTypes(doc_type.annotationtype, data_types->annotations,
+                           data_types->repo);
+    inheritDocumentTypes(doc_type.inherits, type_map, *data_types->doc_type);
+    addFieldSet(doc_type.fieldsets, *data_types->doc_type);
+}
+
+void addDataTypeRepo(DataTypeRepo::UP data_types, DocumentTypeMap &doc_types) {
+    DataTypeRepo *& p = doc_types[data_types->doc_type->getId()];
+    if (p) {
+        LOG(warning, "Type repo already exists for id %d.",
+            data_types->doc_type->getId());
+        throw IllegalArgumentException("Trying to redefine a document type.");
+    }
+    p = data_types.release();
+}
+
+DataTypeRepo::UP makeSkeletonDataTypeRepo(
+        const DocumenttypesConfig::Documenttype &type) {
+    DataTypeRepo::UP data_types(new DataTypeRepo);
+    StructDataType::UP
+        type_ap(new StructDataType(type.name + ".header", type.headerstruct));
+    data_types->doc_type = new DocumentType(type.name, type.id, *type_ap);
+    data_types->repo.addDataType(std::move(type_ap));
+    return data_types;
+}
+
+void createAllDocumentTypes(const DocumenttypesConfig::DocumenttypeVector &t,
+                            DocumentTypeMap &type_map) {
+    for (size_t i = 0; i < t.size(); ++i) {
+        addDataTypeRepo(makeSkeletonDataTypeRepo(t[i]), type_map);
+    }
+}
+
+void addAllDocumentTypesToRepos(DocumentTypeMap &type_map) {
+    for (DocumentTypeMap::const_iterator
+             it = type_map.begin(); it != type_map.end(); ++it) {
+        addDocumentTypes(type_map, it->second->repo);
+    }
+}
+
+void configureAllRepos(const DocumenttypesConfig::DocumenttypeVector &t,
+                       DocumentTypeMap &type_map) {
+    for (size_t i = 0; i < t.size(); ++i) {
+        configureDataTypeRepo(t[i], type_map);
+    }
+}
+
+}  // namespace
+
+DocumentTypeRepo::DocumentTypeRepo() {
+    addDefaultDocument(_doc_types);
+}
+
+DocumentTypeRepo::DocumentTypeRepo(const DocumentType & type) {
+    addDefaultDocument(_doc_types);
+    try {
+        addDataTypeRepo(makeDataTypeRepo(type, _doc_types), _doc_types);
+    } catch (...) {
+        DeleteMapContent(_doc_types);
+        throw;
+    }
+}
+
+DocumentTypeRepo::DocumentTypeRepo(const DocumenttypesConfig &config) {
+    addDefaultDocument(_doc_types);
+    try {
+        createAllDocumentTypes(config.documenttype, _doc_types);
+        addAllDocumentTypesToRepos(_doc_types);
+        configureAllRepos(config.documenttype, _doc_types);
+    } catch (...) {
+        DeleteMapContent(_doc_types);
+        throw;
+    }
+}
+
+DocumentTypeRepo::~DocumentTypeRepo() {
+    DeleteMapContent(_doc_types);
+}
+
+const DocumentType *DocumentTypeRepo::getDocumentType(int32_t type_id) const {
+    const DataTypeRepo *repo = FindPtr(_doc_types, type_id);
+    return repo ? repo->doc_type : 0;
+}
+
+const DocumentType *DocumentTypeRepo::getDocumentType(const stringref &name) const {
+    DocumentTypeMap::const_iterator it =
+        _doc_types.find(DocumentType::createId(name));
+
+    if (it != _doc_types.end() && it->second->doc_type->getName() == name) {
+        return it->second->doc_type;
+    }
+    for (it = _doc_types.begin(); it != _doc_types.end(); ++it) {
+        if (it->second->doc_type->getName() == name) {
+            return it->second->doc_type;
+        }
+    }
+    return 0;
+}
+
+const DataType *
+DocumentTypeRepo::getDataType(const DocumentType &doc_type, int32_t id) const {
+    const DataTypeRepo *dt_repo = FindPtr(_doc_types, doc_type.getId());
+    return dt_repo ? dt_repo->repo.lookup(id) : 0;
+}
+
+const DataType *
+DocumentTypeRepo::getDataType(
+        const DocumentType &doc_type, const stringref &name) const {
+    const DataTypeRepo *dt_repo = FindPtr(_doc_types, doc_type.getId());
+    return dt_repo ? dt_repo->repo.lookup(name) : 0;
+}
+
+const AnnotationType *DocumentTypeRepo::getAnnotationType(
+        const DocumentType &doc_type, int32_t id) const {
+    const DataTypeRepo *dt_repo = FindPtr(_doc_types, doc_type.getId());
+    return dt_repo ? dt_repo->annotations.lookup(id) : 0;
+}
+
+void DocumentTypeRepo::forEachDocumentType(
+        Closure1<const DocumentType &> &c) const {
+    for (DocumentTypeMap::const_iterator
+             it = _doc_types.begin(); it != _doc_types.end(); ++it) {
+        c.call(*it->second->doc_type);
+    }
+}
+
+DocumenttypesConfig readDocumenttypesConfig(const char *file_name) {
+    config::FileConfigReader<DocumenttypesConfig> reader(file_name);
+    return DocumenttypesConfig(*reader.read());
+}
+
+}  // namespace document
diff --git a/document/src/vespa/document/repo/documenttyperepo.h b/document/src/vespa/document/repo/documenttyperepo.h
new file mode 100644
index 00000000000..f3542d3cae2
--- /dev/null
+++ b/document/src/vespa/document/repo/documenttyperepo.h
@@ -0,0 +1,51 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <memory>
+#include <vespa/vespalib/stllike/hash_map.h>
+#include <vespa/vespalib/stllike/string.h>
+#include <vespa/vespalib/util/closure.h>
+#include <vespa/document/config/config-documenttypes.h>
+
+namespace document {
+class AnnotationType;
+class DataType;
+class DataTypeRepo;
+class DocumentType;
+
+class DocumentTypeRepo {
+    typedef vespalib::hash_map<int32_t, DataTypeRepo *> DocumentTypeMap;
+
+    DocumentTypeMap _doc_types;
+
+public:
+    typedef std::shared_ptr<DocumentTypeRepo> SP;
+    typedef std::unique_ptr<DocumentTypeRepo> UP;
+
+    // This one should only be used for testing. If you do not have any config.
+    explicit DocumentTypeRepo(const DocumentType & docType);
+
+    DocumentTypeRepo();
+    explicit DocumentTypeRepo(const DocumenttypesConfig & config);
+    ~DocumentTypeRepo();
+
+    const DocumentType *getDocumentType(int32_t doc_type_id) const;
+    const DocumentType *getDocumentType(const vespalib::stringref &name) const;
+    const DataType *getDataType(const DocumentType &doc_type, int32_t id) const;
+    const DataType *getDataType(const DocumentType &doc_type,
+                                const vespalib::stringref &name) const;
+    const AnnotationType *getAnnotationType(const DocumentType &doc_type,
+                                            int32_t id) const;
+    void forEachDocumentType(
+            vespalib::Closure1<const DocumentType &> &c) const;
+
+private:
+    DocumentTypeRepo(const DocumentTypeRepo &);
+    DocumentTypeRepo &operator=(const DocumentTypeRepo &);
+};
+
+DocumenttypesConfig readDocumenttypesConfig(const char *file_name);
+
+}  // namespace document
+
diff --git a/document/src/vespa/document/repo/fixedtyperepo.cpp b/document/src/vespa/document/repo/fixedtyperepo.cpp
new file mode 100644
index 00000000000..e6914f6cc06
--- /dev/null
+++ b/document/src/vespa/document/repo/fixedtyperepo.cpp
@@ -0,0 +1,17 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".fixedtyperepo");
+
+#include "fixedtyperepo.h"
+
+namespace document {
+
+FixedTypeRepo::FixedTypeRepo(const DocumentTypeRepo &repo,
+                             const vespalib::string &type)
+    : _repo(&repo), _doc_type(repo.getDocumentType(type)) {
+    assert(_doc_type);
+}
+
+}  // namespace document
diff --git a/document/src/vespa/document/repo/fixedtyperepo.h b/document/src/vespa/document/repo/fixedtyperepo.h
new file mode 100644
index 00000000000..1310b993caa
--- /dev/null
+++ b/document/src/vespa/document/repo/fixedtyperepo.h
@@ -0,0 +1,35 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "documenttyperepo.h"
+#include <vespa/document/datatype/datatype.h>
+
+namespace document {
+
+// Combines a DocumentTypeRepo and a DocumentType to allow easy access
+// to the types contained in the DocumentType's namespace.
+class FixedTypeRepo {
+    const DocumentTypeRepo *_repo;
+    const DocumentType *_doc_type;
+
+public:
+    explicit FixedTypeRepo(const DocumentTypeRepo &repo)
+        : _repo(&repo), _doc_type(repo.getDocumentType(DataType::T_DOCUMENT)) { }
+    FixedTypeRepo(const DocumentTypeRepo &repo, const DocumentType &doc_type)
+        : _repo(&repo), _doc_type(&doc_type) {}
+    FixedTypeRepo(const DocumentTypeRepo &repo, const vespalib::string &type);
+
+    const DataType *getDataType(int32_t id) const
+    { return _repo->getDataType(*_doc_type, id); }
+    const DataType *getDataType(const vespalib::string &name) const
+    { return _repo->getDataType(*_doc_type, name); }
+    const AnnotationType *getAnnotationType(int32_t id) const
+    { return _repo->getAnnotationType(*_doc_type, id); }
+
+    const DocumentTypeRepo &getDocumentTypeRepo() const { return *_repo; }
+    const DocumentType &getDocumentType() const { return *_doc_type; }
+};
+
+}  // namespace document
+
diff --git a/document/src/vespa/document/select/.gitignore b/document/src/vespa/document/select/.gitignore
new file mode 100644
index 00000000000..69e6eac20bb
--- /dev/null
+++ b/document/src/vespa/document/select/.gitignore
@@ -0,0 +1,5 @@
+Makefile
+.depend*
+.*.swp
+*.So
+
diff --git a/document/src/vespa/document/select/CMakeLists.txt b/document/src/vespa/document/select/CMakeLists.txt
new file mode 100644
index 00000000000..f58037af581
--- /dev/null
+++ b/document/src/vespa/document/select/CMakeLists.txt
@@ -0,0 +1,27 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_library(document_select OBJECT
+    SOURCES
+    branch.cpp
+    compare.cpp
+    constant.cpp
+    doctype.cpp
+    operator.cpp
+    parser.cpp
+    result.cpp
+    value.cpp
+    valuenode.cpp
+    orderingselector.cpp
+    orderingspecification.cpp
+    resultlist.cpp
+    invalidconstant.cpp
+    bodyfielddetector.cpp
+    simpleparser.cpp
+    resultset.cpp
+    traversingvisitor.cpp
+    cloningvisitor.cpp
+    context.cpp
+    gid_filter.cpp
+    DEPENDS
+    AFTER
+    document_documentconfig
+)
diff --git a/document/src/vespa/document/select/bodyfielddetector.cpp b/document/src/vespa/document/select/bodyfielddetector.cpp
new file mode 100644
index 00000000000..873d3a77808
--- /dev/null
+++ b/document/src/vespa/document/select/bodyfielddetector.cpp
@@ -0,0 +1,52 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".bodyfielddetector");
+
+#include "bodyfielddetector.h"
+
+#include <vespa/document/base/exceptions.h>
+#include <vespa/document/repo/documenttyperepo.h>
+#include <vespa/document/datatype/documenttype.h>
+#include <vespa/vespalib/util/closure.h>
+#include "valuenode.h"
+
+namespace document
+{
+
+namespace select
+{
+
+
+void
+BodyFieldDetector::detectFieldType(const FieldValueNode *expr,
+                                   const DocumentType &type)
+{
+    if (type.getName() != expr->getDocType()) {
+        return;
+    }
+    try {
+        FieldPath::UP path(type.buildFieldPath(expr->getFieldName()));
+        if (path.get() && path->size() != 0) {
+            if ((*path)[0].getFieldRef().isHeaderField()) {
+                foundHeaderField = true;
+            } else {
+                foundBodyField = true;
+            }
+        }
+    } catch (FieldNotFoundException &) {
+    }
+}
+
+
+void
+BodyFieldDetector::visitFieldValueNode(const FieldValueNode& expr)
+{
+    _repo.forEachDocumentType(
+            *makeClosure(this, &BodyFieldDetector::detectFieldType, &expr));
+}
+
+
+}  // namespace select
+}  // namespace document
diff --git a/document/src/vespa/document/select/bodyfielddetector.h b/document/src/vespa/document/select/bodyfielddetector.h
new file mode 100644
index 00000000000..a4716d1a175
--- /dev/null
+++ b/document/src/vespa/document/select/bodyfielddetector.h
@@ -0,0 +1,52 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include "traversingvisitor.h"
+
+namespace document {
+class DocumentTypeRepo;
+class DocumentType;
+
+namespace select {
+
+class BodyFieldDetector : public TraversingVisitor
+{
+    const DocumentTypeRepo &_repo;
+
+    void detectFieldType(const FieldValueNode *expr, const DocumentType &type);
+
+public:
+    BodyFieldDetector(const DocumentTypeRepo &repo)
+        : _repo(repo), foundBodyField(false), foundHeaderField(false)
+    {
+    }
+
+    bool foundBodyField;
+    bool foundHeaderField;
+
+    virtual void visitDocumentType(const DocType&) {
+            // Need to deserialize header to know document type
+        foundHeaderField = true;
+    }
+
+    void visitFieldValueNode(const FieldValueNode& expr);
+};
+
+class NeedDocumentDetector : public TraversingVisitor
+{
+private:
+    bool _needDocument;
+    virtual void visitDocumentType(const DocType &) {
+        _needDocument = true;
+    }
+    virtual void visitFieldValueNode(const FieldValueNode &) {
+        _needDocument = true;
+    }
+public:
+    NeedDocumentDetector() : _needDocument(false) { }
+    bool needDocument() const { return _needDocument; }
+};
+
+}  // namespace select
+}  // namespace document
+
diff --git a/document/src/vespa/document/select/branch.cpp b/document/src/vespa/document/select/branch.cpp
new file mode 100644
index 00000000000..90b9462f7b9
--- /dev/null
+++ b/document/src/vespa/document/select/branch.cpp
@@ -0,0 +1,145 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "branch.h"
+#include "visitor.h"
+
+namespace document {
+namespace select {
+
+And::And(std::unique_ptr<Node> left, std::unique_ptr<Node> right, const char* name)
+    : Branch(name ? name : "AND"),
+      _left(std::move(left)),
+      _right(std::move(right))
+{
+    assert(_left.get());
+    assert(_right.get());
+}
+
+
+void
+And::visit(Visitor &v) const
+{
+    v.visitAndBranch(*this);
+}
+
+
+void
+And::print(std::ostream& out, bool verbose, const std::string& indent) const
+{
+    if (_parentheses) out << '(';
+    _left->print(out, verbose, indent);
+    out << " " << _name << " ";
+    _right->print(out, verbose, indent);
+    if (_parentheses) out << ')';
+}
+
+namespace {
+    template<typename T>
+    ResultList traceAndValue(const T& value, std::ostream& out,
+                         const Node& leftNode, const Node& rightNode)
+    {
+        ResultList left = leftNode.contains(value);
+        out << "And - Left branch returned " << left << ".\n";
+        ResultList right = rightNode.contains(value);
+        out << "And - Right branch returned " << right << ".\n";
+        return left && right;
+    }
+}
+
+ResultList
+And::trace(const Context& context, std::ostream& out) const
+{
+    return traceAndValue(context, out, *_left, *_right);
+}
+
+Or::Or(std::unique_ptr<Node> left, std::unique_ptr<Node> right, const char* name)
+    : Branch(name ? name : "OR"),
+      _left(std::move(left)),
+      _right(std::move(right))
+{
+    assert(_left.get());
+    assert(_right.get());
+}
+
+
+void
+Or::visit(Visitor &v) const
+{
+    v.visitOrBranch(*this);
+}
+
+
+void
+Or::print(std::ostream& out, bool verbose, const std::string& indent) const
+{
+    if (_parentheses) out << '(';
+    _left->print(out, verbose, indent);
+    out << " " << _name << " ";
+    _right->print(out, verbose, indent);
+    if (_parentheses) out << ')';
+}
+
+namespace {
+    template<typename T>
+    ResultList traceOrValue(const T& value, std::ostream& out,
+                         const Node& leftNode, const Node& rightNode)
+    {
+        ResultList left = leftNode.contains(value);
+        out << "Or - Left branch returned " << left << ".\n";
+        ResultList right = rightNode.contains(value);
+        out << "Or - Right branch returned " << right << ".\n";
+        return left || right;
+    }
+}
+
+ResultList
+Or::trace(const Context& context, std::ostream& out) const
+{
+    return traceOrValue(context, out, *_left, *_right);
+}
+
+Not::Not(std::unique_ptr<Node> child, const char* name)
+    : Branch(name ? name : "NOT"),
+      _child(std::move(child))
+{
+    assert(_child.get());
+}
+
+
+void
+Not::visit(Visitor &v) const
+{
+    v.visitNotBranch(*this);
+}
+
+void
+Not::print(std::ostream& out, bool verbose, const std::string& indent) const
+{
+    if (_parentheses) out << '(';
+    out << _name << " ";
+    _child->print(out, verbose, indent);
+    if (_parentheses) out << ')';
+}
+
+namespace {
+    template<typename T>
+    ResultList traceNotValue(const T& value, std::ostream& out,
+                         const Node& node)
+    {
+        ResultList result = node.contains(value);
+        out << "Not - Child returned " << result
+            << ". Returning opposite.\n";
+        return !result;
+    }
+}
+
+ResultList
+Not::trace(const Context& context, std::ostream& out) const
+{
+    return traceNotValue(context, out, *_child);
+}
+
+
+} // select
+} // document
diff --git a/document/src/vespa/document/select/branch.h b/document/src/vespa/document/select/branch.h
new file mode 100644
index 00000000000..7d5de0e8d51
--- /dev/null
+++ b/document/src/vespa/document/select/branch.h
@@ -0,0 +1,102 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/*
+ * @class document::select::Branch
+ * @ingroup select
+ *
+ * @brief Base class for branch nodes in the document selection tree.
+ *
+ * @author H�kon Humberset
+ * @date 2005-06-07
+ * @version $Id$
+ */
+
+#pragma once
+
+#include <memory>
+#include <list>
+#include "node.h"
+
+namespace document {
+namespace select {
+
+class Branch : public Node
+{
+public:
+    Branch(const vespalib::stringref & name) : Node(name) {}
+
+    virtual bool isLeafNode() const { return false; }
+};
+
+class And : public Branch
+{
+    std::unique_ptr<Node> _left;
+    std::unique_ptr<Node> _right;
+public:
+    And(std::unique_ptr<Node> left, std::unique_ptr<Node> right,
+        const char* name = 0);
+
+    virtual ResultList contains(const Context& context) const
+        { return (_left->contains(context) && _right->contains(context)); }
+    virtual ResultList trace(const Context&, std::ostream& trace) const;
+    virtual void visit(Visitor &v) const;
+    virtual void print(std::ostream& out, bool verbose,
+                       const std::string& indent) const;
+
+    const Node& getLeft() const { return *_left; }
+    const Node& getRight() const { return *_right; }
+
+    Node::UP clone() const {
+        return wrapParens(new And(_left->clone(),
+                                  _right->clone(),
+                                  _name.c_str()));
+    }
+};
+
+class Or : public Branch
+{
+    std::unique_ptr<Node> _left;
+    std::unique_ptr<Node> _right;
+public:
+    Or(std::unique_ptr<Node> left, std::unique_ptr<Node> right,
+        const char* name = 0);
+
+    virtual ResultList contains(const Context& context) const
+        { return (_left->contains(context) || _right->contains(context)); }
+    virtual ResultList trace(const Context&, std::ostream& trace) const;
+    virtual void visit(Visitor &v) const;
+    virtual void print(std::ostream& out, bool verbose,
+                       const std::string& indent) const;
+
+    const Node& getLeft() const { return *_left; }
+    const Node& getRight() const { return *_right; }
+
+    Node::UP clone() const {
+        return wrapParens(new Or(_left->clone(),
+                                 _right->clone(),
+                                 _name.c_str()));
+    }
+};
+
+class Not : public Branch
+{
+    std::unique_ptr<Node> _child;
+public:
+    Not(std::unique_ptr<Node> child, const char* name = 0);
+
+    virtual ResultList contains(const Context& context) const
+        { return !_child->contains(context); }
+    virtual ResultList trace(const Context&, std::ostream& trace) const;
+    virtual void visit(Visitor &v) const;
+    virtual void print(std::ostream& out, bool verbose,
+                       const std::string& indent) const;
+
+    const Node& getChild() const { return *_child; }
+
+    Node::UP clone() const {
+        return wrapParens(new Not(_child->clone(), _name.c_str()));
+    }
+};
+
+} // select
+} // document
+
diff --git a/document/src/vespa/document/select/cloningvisitor.cpp b/document/src/vespa/document/select/cloningvisitor.cpp
new file mode 100644
index 00000000000..2c25efad466
--- /dev/null
+++ b/document/src/vespa/document/select/cloningvisitor.cpp
@@ -0,0 +1,380 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "cloningvisitor.h"
+#include "valuenode.h"
+#include "branch.h"
+#include "compare.h" 
+#include "constant.h"
+#include "invalidconstant.h"
+#include "doctype.h" 
+
+
+namespace document
+{
+
+namespace select
+{
+
+const int CloningVisitor::OrPriority;
+const int CloningVisitor::AndPriority;
+const int CloningVisitor::NotPriority;
+const int CloningVisitor::ComparePriority;
+const int CloningVisitor::AddPriority;
+const int CloningVisitor::SubPriority;
+const int CloningVisitor::MulPriority;
+const int CloningVisitor::DivPriority;
+const int CloningVisitor::ModPriority;
+const int CloningVisitor::DocumentTypePriority;
+const int CloningVisitor::FieldValuePriority;
+const int CloningVisitor::InvalidConstPriority;
+const int CloningVisitor::InvalidValPriority;
+const int CloningVisitor::ConstPriority;
+const int CloningVisitor::FuncPriority;
+const int CloningVisitor::VariablePriority;
+const int CloningVisitor::FloatPriority;
+const int CloningVisitor::IntegerPriority;
+const int CloningVisitor::CurrentTimePriority;
+const int CloningVisitor::StringPriority;
+const int CloningVisitor::NullValPriority;
+const int CloningVisitor::IdPriority;
+const int CloningVisitor::SearchColPriority;
+
+CloningVisitor::CloningVisitor(void)
+    : _node(),
+      _valueNode(),
+      _constVal(false),
+      _priority(-1),
+      _fieldNodes(0u),
+      _resultSet()
+{
+}
+
+
+CloningVisitor::~CloningVisitor(void)
+{
+}
+
+
+void
+CloningVisitor::visitAndBranch(const And &expr)
+{
+    int priority = AndPriority;
+    expr.getLeft().visit(*this);
+    bool lhsConstVal = _constVal;
+    ResultSet lhsSet(_resultSet);
+    setNodeParentheses(priority);
+    std::unique_ptr<Node> lhs(std::move(_node));
+    revisit();
+    expr.getRight().visit(*this);
+    _constVal &= lhsConstVal;
+    _resultSet.calcAnd(lhsSet);
+    setNodeParentheses(priority);
+    std::unique_ptr<Node> rhs(std::move(_node));
+    _priority = priority;
+    _node.reset(new And(std::move(lhs), std::move(rhs), "and"));
+};
+
+
+void
+CloningVisitor::visitOrBranch(const Or &expr)
+{
+    int priority = OrPriority;
+    expr.getLeft().visit(*this);
+    bool lhsConstVal = _constVal;
+    ResultSet lhsSet(_resultSet);
+    setNodeParentheses(priority);
+    std::unique_ptr<Node> lhs(std::move(_node));
+    revisit();
+    expr.getRight().visit(*this);
+    _constVal &= lhsConstVal;
+    _resultSet.calcOr(lhsSet);
+    setNodeParentheses(priority);
+    std::unique_ptr<Node> rhs(std::move(_node));
+    _priority = priority;
+    _node.reset(new Or(std::move(lhs), std::move(rhs), "or"));
+};
+
+
+void
+CloningVisitor::visitNotBranch(const Not &expr)
+{
+    int priority = ComparePriority;
+    expr.getChild().visit(*this);
+    setNodeParentheses(priority);
+    _resultSet.calcNot();
+    std::unique_ptr<Node> child(std::move(_node));
+    _priority = priority;
+    _node.reset(new Not(std::move(child), "not"));
+};
+
+
+void
+CloningVisitor::visitComparison(const Compare &expr)
+{
+    int priority = ComparePriority;
+    expr.getLeft().visit(*this);
+    bool lhsConstVal = _constVal;
+    setValueNodeParentheses(priority);
+    std::unique_ptr<ValueNode> lhs(std::move(_valueNode));
+    revisit();
+    expr.getRight().visit(*this);
+    _constVal &= lhsConstVal;
+    setValueNodeParentheses(priority);
+    std::unique_ptr<ValueNode> rhs(std::move(_valueNode));
+    const Operator &op(expr.getOperator());
+    _priority = priority;
+    _resultSet.fill(); // should be less if const
+    _node.reset(new Compare(std::move(lhs),
+                            op,
+                            std::move(rhs),
+                            expr.getBucketIdFactory()));
+};
+
+
+void
+CloningVisitor::visitArithmeticValueNode(const ArithmeticValueNode &expr)
+{
+    expr.getLeft().visit(*this);
+    bool lhsConstVal = _constVal;
+    int lhsPriority = _priority;
+    std::unique_ptr<ValueNode> lhs(std::move(_valueNode));
+    revisit();
+    expr.getRight().visit(*this);
+    bool rhsConstVal = _constVal;
+    int rhsPriority = _priority;
+    std::unique_ptr<ValueNode> rhs(std::move(_valueNode));
+    setArithmeticValueNode(expr,
+                           std::move(lhs), lhsPriority, lhsConstVal,
+                           std::move(rhs), rhsPriority, rhsConstVal);
+}
+
+
+void
+CloningVisitor::visitFunctionValueNode(const FunctionValueNode &expr)
+{
+    int priority = FuncPriority;
+    expr.getChild().visit(*this);
+    setValueNodeParentheses(priority);
+    std::unique_ptr<ValueNode> child(std::move(_valueNode));
+    _priority = priority;
+    _valueNode.reset(new FunctionValueNode(expr.getFunctionName(), std::move(child)));
+};
+
+
+void
+CloningVisitor::visitConstant(const Constant &expr)
+{
+    _constVal = true;
+    _priority = ConstPriority;
+    bool val = expr.getConstantValue();
+    _resultSet.add(val ? Result::True : Result::False);
+    _node.reset(new Constant(val ? "true" : "false"));
+}
+
+
+void
+CloningVisitor::visitInvalidConstant(const InvalidConstant &expr)
+{
+    (void) expr;
+    _constVal = true;
+    _priority = InvalidConstPriority;
+    _resultSet.add(Result::Invalid);
+    _node.reset(new InvalidConstant("invalid"));
+}
+
+
+void
+CloningVisitor::visitDocumentType(const DocType &expr)
+{
+    _constVal = false;
+    _priority = DocumentTypePriority;
+    _resultSet.add(Result::True);
+    _resultSet.add(Result::False);
+    _node = expr.clone();
+}
+
+
+void
+CloningVisitor::visitIdValueNode(const IdValueNode &expr)
+{
+    _constVal = false;
+    ++_fieldNodes; // needs document id, thus needs document
+    _valueNode = expr.clone();
+    _priority = IdPriority;
+}
+
+
+void
+CloningVisitor::visitSearchColumnValueNode(const SearchColumnValueNode &expr)
+{
+    _constVal = false;
+    ++_fieldNodes; // needs document id, thus needs document
+    _valueNode = expr.clone();
+    _priority = SearchColPriority;
+}
+
+
+void
+CloningVisitor::visitFieldValueNode(const FieldValueNode &expr)
+{
+    _constVal = false;
+    ++_fieldNodes; // needs document id, thus needs document
+    _valueNode = expr.clone();
+    _priority = FieldValuePriority;
+}
+
+
+void
+CloningVisitor::visitFloatValueNode(const FloatValueNode &expr)
+{
+    _constVal = true;
+    _valueNode = expr.clone();
+    _priority = FloatPriority;
+}
+
+
+void
+CloningVisitor::visitVariableValueNode(const VariableValueNode &expr)
+{
+    _valueNode.reset(new VariableValueNode(expr.getVariableName()));
+    _priority = VariablePriority;
+}
+
+
+void
+CloningVisitor::visitIntegerValueNode(const IntegerValueNode &expr)
+{
+    _constVal = true;
+    _valueNode = expr.clone();
+    _priority = IntegerPriority;
+}
+
+
+void
+CloningVisitor::visitCurrentTimeValueNode(const CurrentTimeValueNode &expr)
+{
+    _constVal = false;
+    _valueNode = expr.clone();
+    _priority = CurrentTimePriority;
+}
+
+
+void
+CloningVisitor::visitStringValueNode(const StringValueNode &expr)
+{
+    _constVal = true;
+    _valueNode = expr.clone();
+    _priority = StringPriority;
+}
+
+
+void
+CloningVisitor::visitNullValueNode(const NullValueNode &expr)
+{
+    _constVal = true;
+    _valueNode = expr.clone();
+    _priority = NullValPriority;
+}
+
+void
+CloningVisitor::visitInvalidValueNode(const InvalidValueNode &expr)
+{
+    _constVal = true;
+    _valueNode = expr.clone();
+    _priority = InvalidValPriority;
+}
+
+
+void
+CloningVisitor::setNodeParentheses(int priority)
+{
+    if (_priority < priority)
+        _node->setParentheses();
+}
+
+
+void
+CloningVisitor::setValueNodeParentheses(int priority)
+{
+    if (_priority < priority)
+        _valueNode->setParentheses();
+}
+
+
+void
+CloningVisitor::setArithmeticValueNode(const ArithmeticValueNode &expr,
+                                       std::unique_ptr<ValueNode> lhs,
+                                       int lhsPriority,
+                                       bool lhsConstVal,
+                                       std::unique_ptr<ValueNode> rhs,
+                                       int rhsPriority,
+                                       bool rhsConstVal)
+{
+    bool lassoc = false;
+    bool rassoc = false;
+    int priority = 0;
+    switch(expr.getOperator()) {
+    case ArithmeticValueNode::ADD:
+        priority = AddPriority;
+        lassoc = true;
+        rassoc = true;
+        break;
+    case ArithmeticValueNode::SUB:
+        priority = SubPriority;
+        lassoc = true;
+        break;
+    case ArithmeticValueNode::MUL:
+        priority = MulPriority;
+        lassoc = true;
+        rassoc = true;
+        break;
+    case ArithmeticValueNode::DIV:
+        priority = DivPriority;
+        lassoc = true;
+        break;
+    case ArithmeticValueNode::MOD:
+        priority = ModPriority;
+        lassoc = true;
+        break;
+    }
+    if (lhsPriority < priority ||
+        (lhsPriority == priority && !lassoc)) {
+        lhs->setParentheses();
+    }
+    _constVal = lhsConstVal && rhsConstVal;
+    if (rhsPriority < priority ||
+        (rhsPriority == priority && !rassoc)) {
+        rhs->setParentheses();
+    }
+    _priority = priority;
+    _valueNode.reset(new ArithmeticValueNode(std::move(lhs),
+                                             expr.getOperatorName(),
+                                             std::move(rhs)));
+}
+
+
+void
+CloningVisitor::swap(CloningVisitor &rhs)
+{
+    std::swap(_constVal, rhs._constVal);
+    std::swap(_priority, rhs._priority);
+    std::swap(_node, rhs._node);
+    std::swap(_valueNode, rhs._valueNode);
+    std::swap(_resultSet, rhs._resultSet);
+    std::swap(_fieldNodes, rhs._fieldNodes);
+}
+
+
+void
+CloningVisitor::revisit(void)
+{
+    _constVal = false;
+    _priority = -1;
+    _resultSet.clear();
+}
+
+
+}
+
+}
diff --git a/document/src/vespa/document/select/cloningvisitor.h b/document/src/vespa/document/select/cloningvisitor.h
new file mode 100644
index 00000000000..9a513eaa082
--- /dev/null
+++ b/document/src/vespa/document/select/cloningvisitor.h
@@ -0,0 +1,150 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include "visitor.h"
+#include "resultset.h"
+
+namespace document
+{
+
+namespace select
+{
+
+class Node;
+class ValueNode;
+
+class CloningVisitor : public Visitor
+{
+protected:
+    std::unique_ptr<Node> _node;
+    std::unique_ptr<ValueNode> _valueNode;
+    bool _constVal;
+    int _priority;
+    uint32_t _fieldNodes;
+    ResultSet _resultSet;
+
+    static const int OrPriority = 100;
+    static const int AndPriority = 200;
+    static const int NotPriority = 300;
+    static const int ComparePriority = 400;
+    static const int AddPriority = 500;
+    static const int SubPriority = 500;
+    static const int MulPriority = 600;
+    static const int DivPriority = 600;
+    static const int ModPriority = 700;
+    static const int DocumentTypePriority = 1000;
+    static const int FieldValuePriority = 1000;
+    static const int InvalidConstPriority = 1000;
+    static const int InvalidValPriority = 1000;
+    static const int ConstPriority = 1000;
+    static const int FuncPriority = 1000;
+    static const int VariablePriority = 1000;
+    static const int FloatPriority = 1000;
+    static const int IntegerPriority = 1000;
+    static const int CurrentTimePriority = 1000;
+    static const int StringPriority = 1000;
+    static const int NullValPriority = 1000;
+    static const int IdPriority = 1000;
+    static const int SearchColPriority = 1000;
+
+public:
+    CloningVisitor(void);
+
+    virtual
+    ~CloningVisitor(void);
+
+    virtual void
+    visitAndBranch(const And &expr);
+
+    virtual void
+    visitOrBranch(const Or &expr);
+
+    virtual void
+    visitNotBranch(const Not &expr);
+
+    virtual void
+    visitComparison(const Compare &expr);
+
+    virtual void
+    visitArithmeticValueNode(const ArithmeticValueNode &expr);
+
+    virtual void
+    visitFunctionValueNode(const FunctionValueNode &expr);
+
+    virtual void
+    visitConstant(const Constant &expr);
+
+    virtual void
+    visitInvalidConstant(const InvalidConstant &expr);
+
+    virtual void
+    visitDocumentType(const DocType &expr);
+
+    virtual void
+    visitIdValueNode(const IdValueNode &expr);
+
+    virtual void
+    visitSearchColumnValueNode(const SearchColumnValueNode &expr);
+
+    virtual void
+    visitFieldValueNode(const FieldValueNode &expr);
+
+    virtual void
+    visitFloatValueNode(const FloatValueNode &expr);
+
+    virtual void
+    visitVariableValueNode(const VariableValueNode &expr);
+
+    virtual void
+    visitIntegerValueNode(const IntegerValueNode &expr);
+
+    virtual void
+    visitCurrentTimeValueNode(const CurrentTimeValueNode &expr);
+
+    virtual void
+    visitStringValueNode(const StringValueNode &expr);
+
+    virtual void
+    visitNullValueNode(const NullValueNode &expr);
+
+    virtual void
+    visitInvalidValueNode(const InvalidValueNode &expr);
+
+    std::unique_ptr<Node> &
+    getNode(void)
+    {
+        return _node;
+    }
+    
+    std::unique_ptr<ValueNode> &
+    getValueNode(void)
+    {
+        return _valueNode;
+    }
+
+    void
+    setNodeParentheses(int priority);
+
+    void
+    setValueNodeParentheses(int priority);
+
+    void
+    setArithmeticValueNode(const ArithmeticValueNode &expr,
+                           std::unique_ptr<ValueNode> lhs,
+                           int lhsPriority,
+                           bool lhsConstVal,
+                           std::unique_ptr<ValueNode> rhs,
+                           int rhsPriority,
+                           bool rhsConstVal);
+
+    void
+    swap(CloningVisitor &rhs);
+
+    void
+    revisit(void);
+};
+
+}
+
+}
+
diff --git a/document/src/vespa/document/select/compare.cpp b/document/src/vespa/document/select/compare.cpp
new file mode 100644
index 00000000000..2df33e00d22
--- /dev/null
+++ b/document/src/vespa/document/select/compare.cpp
@@ -0,0 +1,160 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include "compare.h"
+
+#include <boost/scoped_array.hpp>
+#include <iomanip>
+#include <vespa/log/log.h>
+#include <sstream>
+#include <vespa/document/datatype/datatype.h>
+#include "valuenode.h"
+#include <vespa/document/fieldvalue/document.h>
+#include <vespa/document/util/stringutil.h>
+#include "compare.h"
+#include "visitor.h"
+
+LOG_SETUP(".document.select.compare");
+
+namespace document {
+namespace select {
+
+Compare::Compare(std::unique_ptr<ValueNode> left,
+                 const Operator& op,
+                 std::unique_ptr<ValueNode> right,
+                 const BucketIdFactory& bucketIdFactory)
+    : Node("Compare"),
+      _left(std::move(left)),
+      _right(std::move(right)),
+      _operator(op),
+      _bucketIdFactory(bucketIdFactory)
+{
+}
+
+Compare::~Compare()
+{
+}
+
+Node::UP Compare::clone() const
+{
+    return wrapParens(new Compare(_left->clone(),
+                                  _operator,
+                                  _right->clone(),
+                                  _bucketIdFactory));
+}
+
+namespace {
+
+    template<typename T>
+    ResultList containsValue(const T& value, const ValueNode& l, const ValueNode& r,
+                         const Operator& op)
+    {
+        std::unique_ptr<Value> left(l.getValue(value));
+        std::unique_ptr<Value> right(r.getValue(value));
+        if (left->getType() == Value::Bucket
+            || right->getType() == Value::Bucket)
+        {
+            Value& bVal(left->getType() == Value::Bucket ? *left : *right);
+            Value& nVal(left->getType() == Value::Bucket ? *right : *left);
+            if (nVal.getType() == Value::Integer
+                && (op == FunctionOperator::EQ || op == FunctionOperator::NE
+                    || op == GlobOperator::GLOB))
+            {
+                document::BucketId b(
+                        static_cast<IntegerValue&>(bVal).getValue());
+                document::BucketId s(
+                        static_cast<IntegerValue&>(nVal).getValue());
+
+                ResultList resultList(Result::get(s.contains(b)));
+
+                if (op == FunctionOperator::NE) {
+                    return !resultList;
+                }
+                return resultList;
+            } else {
+                return ResultList(Result::Invalid);
+            }
+        }
+        return op.compare(*left, *right);
+    }
+
+    template<typename T>
+    ResultList traceValue(const T& value, const ValueNode& l, const ValueNode& r,
+                      const Operator& op, std::ostream& out)
+    {
+        std::unique_ptr<Value> left(l.traceValue(value, out));
+        std::unique_ptr<Value> right(r.traceValue(value, out));
+        if (left->getType() == Value::Bucket
+            || right->getType() == Value::Bucket)
+        {
+            Value& bVal(left->getType() == Value::Bucket ? *left : *right);
+            Value& nVal(left->getType() == Value::Bucket ? *right : *left);
+            if (nVal.getType() == Value::Integer
+                && (op == FunctionOperator::EQ || op == FunctionOperator::NE
+                    || op == GlobOperator::GLOB))
+            {
+                document::BucketId b(
+                        static_cast<IntegerValue&>(bVal).getValue());
+                document::BucketId s(
+                        static_cast<IntegerValue&>(nVal).getValue());
+
+                ResultList resultList(Result::get(s.contains(b)));
+
+                if (op == FunctionOperator::NE) {
+                    resultList = !resultList;
+                }
+
+                out << "Checked if " << b << " is ";
+                if (op == FunctionOperator::NE) { out << "not "; }
+                out << "contained in " << s
+                    << ". Result was " << resultList << ".\n";
+                return resultList;
+            } else {
+                out << "Compare type " << left->getType() << " vs "
+                    << right->getType() << " - Result is thus invalid.\n";
+
+                return ResultList(Result::Invalid);
+            }
+        }
+        out << "Compare - Left value ";
+        left->print(out, false, "");
+        out << " " << op.getName() << " right value ";
+        right->print(out, false, "");
+        out << "\n";
+        ResultList result = op.trace(*left, *right, out);
+        out << "Result from compare was " << result << ".\n";
+        return result;
+    }
+}
+
+ResultList Compare::contains(const Context& context) const
+{
+    return containsValue<Context>(context, *_left, *_right, _operator);
+}
+
+ResultList Compare::trace(const Context& context, std::ostream& out) const
+{
+    return traceValue(context, *_left, *_right, _operator, out);
+}
+
+
+void
+Compare::visit(Visitor &v) const
+{
+    v.visitComparison(*this);
+}
+
+
+void
+Compare::print(std::ostream& out, bool verbose, const std::string& indent) const
+{
+    if (_parentheses) out << '(';
+    _left->print(out, verbose, indent);
+    out << " ";
+    _operator.print(out, verbose, indent);
+    out << " ";
+    _right->print(out, verbose, indent);
+    if (_parentheses) out << ')';
+}
+
+} // select
+} // document
diff --git a/document/src/vespa/document/select/compare.h b/document/src/vespa/document/select/compare.h
new file mode 100644
index 00000000000..63c93027759
--- /dev/null
+++ b/document/src/vespa/document/select/compare.h
@@ -0,0 +1,67 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * @class document::select::Compare
+ * @ingroup select
+ *
+ * @brief Node comparing two values
+ *
+ * @author H�kon Humberset
+ * @date 2007-04-20
+ * @version $Id$
+ */
+
+#pragma once
+
+#include <memory>
+#include "node.h"
+#include "operator.h"
+#include <vespa/document/bucket/bucketidfactory.h>
+
+namespace document {
+namespace select {
+
+class ValueNode;
+
+class Compare : public Node
+{
+private:
+    std::unique_ptr<ValueNode> _left;
+    std::unique_ptr<ValueNode> _right;
+    const Operator& _operator;
+    const BucketIdFactory& _bucketIdFactory;
+
+    bool isLeafNode() const { return false; }
+public:
+    Compare(std::unique_ptr<ValueNode> left, const Operator& op,
+            std::unique_ptr<ValueNode> right,
+            const BucketIdFactory& bucketidfactory);
+    virtual ~Compare();
+
+    virtual ResultList
+    contains(const Context &context) const;
+
+    virtual ResultList
+    trace(const Context &context,
+          std::ostream& trace) const;
+
+    virtual void print(std::ostream&, bool verbose,
+                       const std::string& indent) const;
+    virtual void visit(Visitor& v) const;
+
+    const Operator& getOperator() const { return _operator; }
+
+    const ValueNode& getLeft() const { return *_left; }
+    const ValueNode& getRight() const { return *_right; }
+
+    Node::UP clone() const;
+
+    const BucketIdFactory &
+    getBucketIdFactory(void) const
+    {
+        return _bucketIdFactory;
+    }
+};
+
+} // select
+} // document
+
diff --git a/document/src/vespa/document/select/constant.cpp b/document/src/vespa/document/select/constant.cpp
new file mode 100644
index 00000000000..f12e7bd6739
--- /dev/null
+++ b/document/src/vespa/document/select/constant.cpp
@@ -0,0 +1,58 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include "constant.h"
+#include "visitor.h"
+
+namespace document {
+namespace select {
+
+Constant::Constant(const vespalib::stringref & value)
+    : Node(value),
+      _value(false)
+{
+    if (value.size() == 4 &&
+        (value[0] & 0xdf) == 'T' &&
+        (value[1] & 0xdf) == 'R' &&
+        (value[2] & 0xdf) == 'U' &&
+        (value[3] & 0xdf) == 'E')
+    {
+        _value = true;
+    } else if (value.size() == 5 &&
+        (value[0] & 0xdf) == 'F' &&
+        (value[1] & 0xdf) == 'A' &&
+        (value[2] & 0xdf) == 'L' &&
+        (value[3] & 0xdf) == 'S' &&
+        (value[4] & 0xdf) == 'E')
+    {
+        _value = false;
+    } else {
+        assert(false);
+    }
+}
+
+ResultList
+Constant::trace(const Context&, std::ostream& ost) const
+{
+    ost << "Constant - " << Result::get(_value) << ".\n";
+    return ResultList(Result::get(_value));
+}
+
+
+void
+Constant::visit(Visitor &v) const
+{
+    v.visitConstant(*this);
+}
+
+
+void
+Constant::print(std::ostream& out, bool,
+                const std::string&) const
+{
+    if (_parentheses) out << '(';
+    out << _name;
+    if (_parentheses) out << ')';
+}
+
+} // select
+} // document
diff --git a/document/src/vespa/document/select/constant.h b/document/src/vespa/document/select/constant.h
new file mode 100644
index 00000000000..1fec81d2f49
--- /dev/null
+++ b/document/src/vespa/document/select/constant.h
@@ -0,0 +1,48 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * @class document::select::Constant
+ * @ingroup select
+ *
+ * @brief Class describing a constant in the select tree.
+ *
+ * @author H�kon Humberset
+ * @date 2005-06-07
+ * @version $Id$
+ */
+
+#pragma once
+
+#include "node.h"
+
+namespace document {
+namespace select {
+
+class Constant : public Node
+{
+private:
+    bool _value;
+
+public:
+    explicit Constant(const vespalib::stringref & value);
+
+    virtual ResultList
+    contains(const Context&) const
+    {
+        return ResultList(Result::get(_value));
+    }
+
+    virtual ResultList
+    trace(const Context&, std::ostream& trace) const;
+    virtual void print(std::ostream& out, bool verbose,
+                       const std::string& indent) const;
+    virtual void visit(Visitor& v) const;
+
+    bool getConstantValue() const { return _value; }
+
+    Node::UP clone() const { return wrapParens(new Constant(_name)); }
+
+};
+
+} // select
+} // document
+
diff --git a/document/src/vespa/document/select/context.cpp b/document/src/vespa/document/select/context.cpp
new file mode 100644
index 00000000000..abac8391054
--- /dev/null
+++ b/document/src/vespa/document/select/context.cpp
@@ -0,0 +1,12 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include "context.h"
+#include "value.h"
+
+namespace document
+{
+namespace select
+{
+
+} // select
+} // document
diff --git a/document/src/vespa/document/select/context.h b/document/src/vespa/document/select/context.h
new file mode 100644
index 00000000000..6462aa7c838
--- /dev/null
+++ b/document/src/vespa/document/select/context.h
@@ -0,0 +1,70 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/vespalib/stllike/string.h>
+#include <vespa/vespalib/stllike/hash_map.h>
+
+namespace document
+{
+
+class Document;
+class DocumentId;
+class DocumentUpdate;
+
+namespace select
+{
+
+class Value;
+
+class Context
+{
+public:
+    typedef vespalib::hash_map<vespalib::string, double> VariableMap;
+
+    Context(void)
+        : _doc(NULL),
+          _docId(NULL),
+          _docUpdate(NULL),
+          _variables()
+    {
+    }
+
+    Context(const Document& doc)
+        : _doc(&doc),
+          _docId(NULL),
+          _docUpdate(NULL),
+          _variables()
+    {
+    }
+    
+    Context(const DocumentId& docId)
+        : _doc(NULL),
+          _docId(&docId),
+          _docUpdate(NULL),
+          _variables()
+    {
+    }
+    
+    Context(const DocumentUpdate& docUpdate)
+        : _doc(NULL),
+          _docId(NULL),
+          _docUpdate(&docUpdate),
+          _variables()
+    {
+    }
+
+    virtual
+    ~Context(void)
+    {
+    }
+
+    const Document* _doc;
+    const DocumentId* _docId;
+    const DocumentUpdate* _docUpdate;
+    VariableMap _variables;
+};
+
+} // select
+} // document
+
+
diff --git a/document/src/vespa/document/select/doctype.cpp b/document/src/vespa/document/select/doctype.cpp
new file mode 100644
index 00000000000..b6683fe1d8f
--- /dev/null
+++ b/document/src/vespa/document/select/doctype.cpp
@@ -0,0 +1,92 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include "doctype.h"
+#include "visitor.h"
+
+#include <vespa/document/update/documentupdate.h>
+#include <vespa/document/fieldvalue/document.h>
+
+namespace document {
+namespace select {
+
+namespace {
+    bool documentTypeEqualsName(const DocumentType& type,
+                                const vespalib::stringref& name)
+    {
+        if (type.getName() == name) return true;
+        for (std::vector<const DocumentType *>::const_iterator it
+                = type.getInheritedTypes().begin();
+             it != type.getInheritedTypes().end(); ++it)
+        {
+            if (documentTypeEqualsName(**it, name)) return true;
+        }
+        return false;
+    }
+}
+
+DocType::DocType(const vespalib::stringref& doctype)
+    : Node("DocType"),
+      _doctype(doctype)
+{
+}
+
+ResultList
+DocType::contains(const Context &context) const
+{
+    if (context._doc != NULL) {
+        const Document &doc = *context._doc;
+        return
+            ResultList(Result::get(
+                documentTypeEqualsName(doc.getType(),
+                                       _doctype)));
+    }
+    if (context._docId != NULL) {
+        return ResultList(Result::False);
+    }
+    const DocumentUpdate &upd(*context._docUpdate);
+    return ResultList(Result::get(
+                documentTypeEqualsName(upd.getType(), _doctype)));
+}
+
+ResultList
+DocType::trace(const Context& context, std::ostream& out) const
+{
+    ResultList result = contains(context);
+    if (context._doc != NULL) {
+        const Document &doc = *context._doc;
+        out << "DocType - Doc is type " << doc.getType()
+            << ", wanted " << _doctype << ", returning "
+            << result << ".\n";
+    } else if (context._docId != NULL) {
+        out << "DocType - Doc is type (document id -- unknown type)"
+            << ", wanted " << _doctype << ", returning "
+            << result << ".\n";
+    } else {
+        const DocumentUpdate &update(*context._docUpdate);
+        out << "DocType - Doc is type " << update.getType()
+            << ", wanted " << _doctype << ", returning "
+            << result << ".\n";
+    }
+    return result;
+}
+
+
+void
+DocType::visit(Visitor &v) const
+{
+    v.visitDocumentType(*this);
+}
+
+
+void
+DocType::print(std::ostream& out, bool verbose,
+                const std::string& indent) const
+{
+    (void) verbose; (void) indent;
+    if (_parentheses) out << '(';
+    out << _doctype;
+    if (_parentheses) out << ')';
+}
+
+} // select
+} // document
diff --git a/document/src/vespa/document/select/doctype.h b/document/src/vespa/document/select/doctype.h
new file mode 100644
index 00000000000..d96c3719cde
--- /dev/null
+++ b/document/src/vespa/document/select/doctype.h
@@ -0,0 +1,41 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * @class document::select::DocType
+ * @ingroup select
+ *
+ * @brief Class matching whether a document is of given type or not.
+ *
+ * @author H�kon Humberset
+ * @date 2005-06-07
+ * @version $Id$
+ */
+
+#pragma once
+
+#include <vespa/document/datatype/documenttype.h>
+#include "node.h"
+
+namespace document {
+namespace select {
+
+class DocType : public Node
+{
+private:
+    vespalib::string _doctype;
+
+public:
+    DocType(const vespalib::stringref& doctype);
+
+    virtual ResultList contains(const Context&) const;
+    virtual ResultList trace(const Context&, std::ostream& trace) const;
+    virtual void print(std::ostream& out, bool verbose,
+                       const std::string& indent) const;
+    virtual void visit(Visitor& v) const;
+
+    Node::UP clone() const { return wrapParens(new DocType(_doctype)); }
+
+};
+
+} // select
+} // document
+
diff --git a/document/src/vespa/document/select/gid_filter.cpp b/document/src/vespa/document/select/gid_filter.cpp
new file mode 100644
index 00000000000..cb194452ead
--- /dev/null
+++ b/document/src/vespa/document/select/gid_filter.cpp
@@ -0,0 +1,160 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include "gid_filter.h"
+#include "node.h"
+#include "parser.h"
+#include "visitor.h"
+#include "valuenode.h"
+#include "compare.h"
+#include "branch.h"
+#include <vespa/document/base/idstring.h>
+
+namespace document {
+namespace select {
+
+namespace {
+
+struct NoOpVisitor : Visitor {
+    void visitAndBranch(const And&) override {}
+    void visitComparison(const Compare&) override {}
+    void visitConstant(const Constant&) override {}
+    void visitInvalidConstant(const InvalidConstant&) override {}
+    void visitDocumentType(const DocType&) override {}
+    void visitNotBranch(const Not&) override {}
+    void visitOrBranch(const Or&) override {}
+    void visitArithmeticValueNode(const ArithmeticValueNode&) override {}
+    void visitFunctionValueNode(const FunctionValueNode&) override {}
+    void visitIdValueNode(const IdValueNode&) override {}
+    void visitSearchColumnValueNode(const SearchColumnValueNode&) override {}
+    void visitFieldValueNode(const FieldValueNode&) override {}
+    void visitFloatValueNode(const FloatValueNode&) override {}
+    void visitVariableValueNode(const VariableValueNode&) override {}
+    void visitIntegerValueNode(const IntegerValueNode&) override {}
+    void visitCurrentTimeValueNode(const CurrentTimeValueNode&) override {}
+    void visitStringValueNode(const StringValueNode&) override {}
+    void visitNullValueNode(const NullValueNode&) override {}
+    void visitInvalidValueNode(const InvalidValueNode&) override {}
+};
+
+/**
+ * Used for identifying whether a given visited node is a comparison node for a
+ * location constraint, and if so, what the location constraint parameters
+ * actually are. Only visits the children of the provided node.
+ *
+ * Allows for child nodes to be in any order to allow order-invariant
+ * (commuting) comparisons (i.e. "a == b" is identical to "b == a")
+ */
+struct IdComparisonVisitor : NoOpVisitor {
+    const IdValueNode*      _id_user_node{nullptr};
+    const IdValueNode*      _id_group_node{nullptr};
+    const IntegerValueNode* _int_literal_node{nullptr};
+    const StringValueNode*  _string_literal_node{nullptr};
+
+    void visitIdValueNode(const IdValueNode& node) override {
+        const auto type = node.getType();
+        if (type == IdValueNode::USER) {
+            _id_user_node = &node;
+        } else if (type == IdValueNode::GROUP) {
+            _id_group_node = &node;
+        }
+    }
+
+    void visitIntegerValueNode(const IntegerValueNode& node) override {
+        _int_literal_node = &node;
+    }
+
+    void visitStringValueNode(const StringValueNode& node) override {
+        _string_literal_node = &node;
+    }
+
+    bool is_valid_location_sub_expression() const noexcept {
+        return ((_id_user_node && _int_literal_node)
+                || (_id_group_node && _string_literal_node));
+    }
+};
+
+/**
+ * Base visitor type invariant: it MUST NOT descend further down the tree by
+ * default for any inner node.
+ */
+class LocationConstraintVisitor : public NoOpVisitor {
+    GidFilter::OptionalLocation _location;
+public:
+    GidFilter::OptionalLocation location() const noexcept { return _location; }
+private:
+    void visitAndBranch(const And& node) override {
+        node.getLeft().visit(*this);
+        node.getRight().visit(*this);
+    }
+
+    /**
+     * We explicitly DO NOT visit OR/NOT branches here. This implicitly
+     * causes the DFS of the AST to terminate early and does not attempt to
+     * identify any location predicates further down the tree. This means that
+     * we only process location predicates that are _directly_ reachable from
+     * the root node via 0-n AND branches and therefore must be matched in
+     * order for the whole selection to match. The default behavior when
+     * we cannot find a location predicate is to assume all documents may match,
+     * which is the correct behavior in any other case, as we can no longer
+     * guarantee that not matching the GID will cause the selection itself to
+     * also mismatch.
+     */
+
+    void visitComparison(const Compare& cmp) override {
+        IdComparisonVisitor id_visitor;
+        cmp.getLeft().visit(id_visitor);
+        cmp.getRight().visit(id_visitor);
+        if (!id_visitor.is_valid_location_sub_expression()) {
+            return; // Don't bother visiting any subtrees.
+        }
+        extract_location_from_id_visitor(id_visitor);
+    }
+
+    uint32_t truncate_location(int64_t full_location) const noexcept {
+        return static_cast<uint32_t>(full_location);
+    }
+
+    uint32_t location_from_integer_literal_node(
+            const IntegerValueNode& node) const
+    {
+        Context ctx;
+        auto rhs = node.getValue(ctx);
+        auto full_location = static_cast<const IntegerValue&>(*rhs).getValue();
+        return truncate_location(full_location);
+    }
+
+    uint32_t location_from_string_literal_node(
+            const StringValueNode& node) const
+    {
+        auto full_location = GroupDocIdString::locationFromGroupName(
+                node.getValue());
+        return truncate_location(full_location);
+    }
+
+    void extract_location_from_id_visitor(const IdComparisonVisitor& visitor) {
+        uint32_t location;
+        if (visitor._int_literal_node) {
+            location = location_from_integer_literal_node(
+                    *visitor._int_literal_node);
+        } else {
+            location = location_from_string_literal_node(
+                    *visitor._string_literal_node);
+        }
+        _location = GidFilter::OptionalLocation(location);
+    }
+};
+
+GidFilter::OptionalLocation location_bits_from_selection(const Node& ast_root) {
+    LocationConstraintVisitor visitor;
+    ast_root.visit(visitor);
+    return visitor.location();
+}
+
+} // anon ns
+
+GidFilter::GidFilter(const Node& ast_root)
+    : _required_gid_location(location_bits_from_selection(ast_root))
+{
+}
+
+} // select
+} // document
diff --git a/document/src/vespa/document/select/gid_filter.h b/document/src/vespa/document/select/gid_filter.h
new file mode 100644
index 00000000000..de5903919c1
--- /dev/null
+++ b/document/src/vespa/document/select/gid_filter.h
@@ -0,0 +1,91 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/document/base/globalid.h>
+
+namespace document {
+namespace select {
+
+class Node;
+
+/**
+ * This class allows for very quickly and cheaply filtering away metadata
+ * entries that may not possibly match a document selection with a location
+ * predicate, based on nothing but the GIDs in the metadata. This avoids
+ * having to fetch the document IDs or whole documents themselves from
+ * potentially slow storage in order to evaluate the selection in full.
+ */
+class GidFilter {
+public:
+    // GCC <= 5.3 has a bug where copying a boost::optional at certain -O
+    // levels causes a erroneous compiler warning.
+    // Using our own poor-man's optional for now.
+    // TODO: replace with std::otional once on a C++17 stdlib.
+    struct OptionalLocation {
+        uint32_t _location;
+        bool _valid;
+
+        OptionalLocation() : _location(0), _valid(false) {}
+
+        explicit OptionalLocation(uint32_t location)
+            : _location(location),
+              _valid(true)
+        {
+        }
+    };
+private:
+    OptionalLocation _required_gid_location;
+
+    /**
+     * Lifetime of AST Node pointed to does not have to extend beyond the call
+     * to this constructor.
+     */
+    explicit GidFilter(const Node& ast_root);
+public:
+    /**
+     * No-op filter; everything matches always.
+     */
+    GidFilter()
+        : _required_gid_location()
+    {
+    }
+
+    /**
+     * A GidFilter instance may be safely and cheaply copied. No dependencies
+     * exist on the life time of the AST from which it was created.
+     */
+    GidFilter(const GidFilter&) = default;
+    GidFilter& operator=(const GidFilter&) = default;
+
+    /**
+     * Create a filter with a location inferred from the provided selection.
+     * If the selection does not contain a location predicate, the GidFilter
+     * will effectively act as a no-op which assumes every document may match.
+     *
+     * It is safe to use the resulting GidFilter even if the lifetime of the
+     * Node pointed to by ast_root does not extend beyond this call; the
+     * GidFilter does not store any implicit or explicit references to it.
+     */
+    static GidFilter for_selection_root_node(const Node& ast_root) {
+        return GidFilter(ast_root);
+    }
+
+    /**
+     * Returns false iff there exists no way that a document whose ID has the
+     * given GID can possibly match the selection. This currently only applies
+     * if the document selection contains a location-based predicate (i.e.
+     * id.user or id.group).
+     *
+     * As the name implies this is a probabilistic match; it's possible for
+     * this function to return true even if the document selection matched
+     * against the full document/documentid would return false.
+     */
+    bool gid_might_match_selection(const GlobalId& gid) const {
+        const uint32_t gid_location = gid.getLocationSpecificBits();
+        return (!_required_gid_location._valid
+                || (gid_location == _required_gid_location._location));
+    }
+};
+
+} // select
+} // document
diff --git a/document/src/vespa/document/select/invalidconstant.cpp b/document/src/vespa/document/select/invalidconstant.cpp
new file mode 100644
index 00000000000..3573f83541d
--- /dev/null
+++ b/document/src/vespa/document/select/invalidconstant.cpp
@@ -0,0 +1,39 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include "invalidconstant.h"
+#include "visitor.h"
+
+namespace document {
+namespace select {
+
+InvalidConstant::InvalidConstant(const vespalib::stringref & value)
+    : Node(value)
+{
+}
+
+ResultList
+InvalidConstant::trace(const Context&, std::ostream& ost) const
+{
+    ost << "InvalidConstant - " << Result::Invalid << ".\n";
+    return ResultList(Result::Invalid);
+}
+
+
+void
+InvalidConstant::visit(Visitor &v) const
+{
+    v.visitInvalidConstant(*this);
+}
+
+
+void
+InvalidConstant::print(std::ostream& out, bool,
+                       const std::string&) const
+{
+    if (_parentheses) out << '(';
+    out << _name;
+    if (_parentheses) out << ')';
+}
+
+} // select
+} // document
diff --git a/document/src/vespa/document/select/invalidconstant.h b/document/src/vespa/document/select/invalidconstant.h
new file mode 100644
index 00000000000..14bd681b97a
--- /dev/null
+++ b/document/src/vespa/document/select/invalidconstant.h
@@ -0,0 +1,38 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * @class document::select::InvalidConstant
+ * @ingroup select
+ *
+ * @brief Class describing an invalid constant in the select tree.
+ *
+ * @author H�kon Humberset
+ * @date 2005-06-07
+ * @version $Id$
+ */
+
+#pragma once
+
+#include "node.h"
+
+namespace document {
+namespace select {
+
+class InvalidConstant : public Node
+{
+public:
+    explicit InvalidConstant(const vespalib::stringref &value);
+
+    virtual ResultList contains(const Context&) const
+        { return ResultList(Result::Invalid); }
+    virtual ResultList trace(const Context&, std::ostream& trace) const;
+    virtual void print(std::ostream& out, bool verbose,
+                       const std::string& indent) const;
+    virtual void visit(Visitor& v) const;
+
+    Node::UP clone() const { return wrapParens(new InvalidConstant(_name)); }
+
+};
+
+} // select
+} // document
+
diff --git a/document/src/vespa/document/select/node.h b/document/src/vespa/document/select/node.h
new file mode 100644
index 00000000000..4a708022d8e
--- /dev/null
+++ b/document/src/vespa/document/select/node.h
@@ -0,0 +1,67 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * @class document::select::Node
+ * @ingroup select
+ *
+ * @brief Base class for all nodes in the document selection tree.
+ *
+ * @author H�kon Humberset
+ * @date 2005-06-07
+ * @version $Id$
+ */
+
+#pragma once
+
+#include <iostream>
+#include <string>
+#include "resultlist.h"
+#include "context.h"
+
+namespace document {
+
+namespace select {
+
+class Visitor;
+
+class Node : public Printable
+{
+protected:
+    vespalib::string _name;
+    bool _parentheses; // Set to true if parentheses was used around this part
+                       // Set such that we can recreate original query in print.
+public:
+    typedef std::unique_ptr<Node> UP;
+    typedef std::shared_ptr<Node> SP;
+
+    Node(const vespalib::stringref & name) : _name(name), _parentheses(false) {}
+    virtual ~Node() {}
+
+    void setParentheses() { _parentheses = true; }
+
+    void
+    clearParentheses()
+    {
+        _parentheses = false;
+    }
+
+    bool hadParentheses() const { return _parentheses; }
+
+    virtual ResultList contains(const Context&) const = 0;
+    virtual ResultList trace(const Context&, std::ostream& trace) const = 0;
+    virtual bool isLeafNode() const { return true; }
+    virtual void visit(Visitor&) const = 0;
+
+    virtual Node::UP clone() const = 0;
+protected:
+    Node::UP wrapParens(Node* node) const {
+        Node::UP ret(node);
+        if (_parentheses) {
+            ret->setParentheses();
+        }
+        return ret;
+    }
+};
+
+} // select
+} // document
+
diff --git a/document/src/vespa/document/select/operator.cpp b/document/src/vespa/document/select/operator.cpp
new file mode 100644
index 00000000000..111a7ed40a1
--- /dev/null
+++ b/document/src/vespa/document/select/operator.cpp
@@ -0,0 +1,233 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include "operator.h"
+
+#include <stdint.h>
+#include <vespa/vespalib/util/regexp.h>
+#include <vespa/vespalib/stllike/asciistream.h>
+
+namespace document {
+namespace select {
+
+Operator::OperatorMap Operator::_operators;
+
+Operator::Operator(const vespalib::stringref & name)
+    : _name(name)
+{
+    OperatorMap::iterator it = _operators.find(name);
+    if (it != _operators.end()) {
+        assert(false);
+    }
+    _operators[_name] = this;
+}
+
+const Operator&
+Operator::get(const vespalib::stringref & name)
+{
+    OperatorMap::iterator it = _operators.find(name);
+    if (it == _operators.end()) {
+        assert(false);
+    }
+    return *it->second;
+}
+
+void
+Operator::print(std::ostream& out, bool verbose,
+                const std::string& indent) const
+{
+    (void) verbose; (void) indent;
+    out << _name;
+}
+
+ResultList
+FunctionOperator::compare(const Value& a, const Value& b) const
+{
+    return (a.*_comparator)(b);
+}
+
+ResultList
+FunctionOperator::trace(const Value& a, const Value& b,
+                        std::ostream& out) const
+{
+    ResultList result = (a.*_comparator)(b);
+    out << "Operator(" << getName() << ") - Result was "
+        << result << ".\n";
+    return result;
+}
+
+const FunctionOperator
+FunctionOperator::GT(">", &Value::operator>);
+
+const FunctionOperator
+FunctionOperator::GEQ(">=", &Value::operator>=);
+
+const FunctionOperator
+FunctionOperator::EQ("==", &Value::operator==);
+
+const FunctionOperator
+FunctionOperator::LEQ("<=", &Value::operator<=);
+
+const FunctionOperator
+FunctionOperator::LT("<", &Value::operator<);
+
+const FunctionOperator
+FunctionOperator::NE("!=", &Value::operator!=);
+
+RegexOperator::RegexOperator(const vespalib::stringref & name)
+    : Operator(name)
+{
+}
+
+ResultList
+RegexOperator::compare(const Value& a, const Value& b) const
+{
+    return a.regexCompare(b);
+}
+
+ResultList
+RegexOperator::trace(const Value& a, const Value& b, std::ostream& out) const
+{
+    return a.regexTrace(b, out);
+}
+
+ResultList
+RegexOperator::compareImpl(const Value& a, const Value& b) const
+{
+    const StringValue* left(dynamic_cast<const StringValue*>(&a));
+    const StringValue* right(dynamic_cast<const StringValue*>(&b));
+    if (left == 0 || right == 0) return ResultList(Result::Invalid);
+    return match(left->getValue(), right->getValue());
+}
+
+ResultList
+RegexOperator::traceImpl(const Value& a, const Value& b, std::ostream& out) const
+{
+    const StringValue* left(dynamic_cast<const StringValue*>(&a));
+    const StringValue* right(dynamic_cast<const StringValue*>(&b));
+    if (left == 0) {
+        out << "Operator(" << getName() << ") - Left value not a string. "
+            << "Returning invalid.\n";
+        return ResultList(Result::Invalid);
+    }
+    if (right == 0) {
+        out << "Operator(" << getName() << ") - Right value not a string. "
+            << "Returning invalid.\n";
+        return ResultList(Result::Invalid);
+    }
+    ResultList result = match(left->getValue(), right->getValue());
+    out << "Operator(" << getName() << ")(" << left->getValue() << ", "
+        << right->getValue() << ") - Result was " << result << "\n";
+    return result;
+}
+
+ResultList
+RegexOperator::match(const vespalib::string& val, const vespalib::stringref & expr) const
+{
+        // Should we catch this in parsing?
+    if (expr.size() == 0) return ResultList(Result::True);
+    vespalib::Regexp expression(expr);
+    return ResultList(Result::get(expression.match(val)));
+}
+
+const RegexOperator RegexOperator::REGEX("=~");
+
+GlobOperator::GlobOperator(const vespalib::stringref & name)
+    : RegexOperator(name)
+{
+}
+
+ResultList
+GlobOperator::compare(const Value& a, const Value& b) const
+{
+    return a.globCompare(b);
+}
+
+ResultList
+GlobOperator::trace(const Value& a, const Value& b, std::ostream& out) const
+{
+    return a.globTrace(b, out);
+}
+
+ResultList
+GlobOperator::compareImpl(const Value& a, const Value& b) const
+{
+    const StringValue* right(dynamic_cast<const StringValue*>(&b));
+        // Fall back to operator== if it isn't string matching
+    if (right == 0) {
+        return FunctionOperator::EQ.compare(a, b);
+    }
+    const StringValue* left(dynamic_cast<const StringValue*>(&a));
+    if (left == 0) return ResultList(Result::Invalid);
+    vespalib::string regex(convertToRegex(right->getValue()));
+    return match(left->getValue(), regex);
+}
+
+ResultList
+GlobOperator::traceImpl(const Value& a, const Value& b, std::ostream& ost) const
+{
+    const StringValue* right(dynamic_cast<const StringValue*>(&b));
+        // Fall back to operator== if it isn't string matching
+    if (right == 0) {
+        ost << "Operator(" << getName() << ") - Right val not a string, "
+            << "falling back to == behavior.\n";
+        return FunctionOperator::EQ.trace(a, b, ost);
+    }
+    const StringValue* left(dynamic_cast<const StringValue*>(&a));
+    if (left == 0) {
+        ost << "Operator(" << getName() << ") - Left value is not a string, "
+            << "returning invalid.\n";
+        return ResultList(Result::Invalid);
+    }
+    vespalib::string regex(convertToRegex(right->getValue()));
+    ost << "Operator(" << getName() << ") - Converted glob expression '"
+        << right->getValue() << "' to regex '"  << regex << "'.\n";
+    return match(left->getValue(), regex);
+}
+
+vespalib::string
+GlobOperator::convertToRegex(const vespalib::stringref & globpattern) const
+{
+    vespalib::asciistream ost;
+    ost << '^';
+    for(uint32_t i=0, n=globpattern.size(); i<n; ++i) {
+        switch(globpattern[i]) {
+            case '*': ost << ".*";
+                      break;
+            case '?': ost << ".";
+                      break;
+            case '^':
+            case '$':
+            case '|':
+            case '{':
+            case '}':
+            case '(':
+            case ')':
+            case '[':
+            case ']':
+            case '\\':
+            case '+':
+            case '.': ost << '\\' << globpattern[i];
+                      break;
+                // Are there other regex special chars we need to escape?
+            default: ost << globpattern[i];
+        }
+    }
+    ost << '$';
+    return ost.str();
+}
+
+bool
+GlobOperator::containsVariables(const vespalib::stringref & expression)
+{
+    for (size_t i=0, n=expression.size(); i<n; ++i) {
+        if (expression[i] == '*' || expression[i] == '?') {
+            return true;
+        }
+    }
+    return false;
+}
+
+const GlobOperator GlobOperator::GLOB("=");
+
+} // select
+} // document
diff --git a/document/src/vespa/document/select/operator.h b/document/src/vespa/document/select/operator.h
new file mode 100644
index 00000000000..9a22929ded8
--- /dev/null
+++ b/document/src/vespa/document/select/operator.h
@@ -0,0 +1,107 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * @class document::select::Operator
+ * @ingroup select
+ *
+ * @brief An operator that can be used to compare values
+ *
+ * @author H�kon Humberset
+ * @date 2007-04-20
+ * @version $Id$
+ */
+
+#pragma once
+
+#include "value.h"
+#include <vespa/vespalib/stllike/hash_map.h>
+
+namespace document {
+namespace select {
+
+class Operator : public Printable {
+private:
+    typedef vespalib::hash_map<vespalib::string, const Operator*> OperatorMap;
+    static OperatorMap _operators;
+    vespalib::string _name;
+
+public:
+    Operator(const vespalib::stringref & name);
+    virtual ~Operator() {}
+
+    virtual ResultList compare(const Value&, const Value&) const = 0;
+    virtual ResultList trace(const Value&, const Value&,
+                         std::ostream& trace) const = 0;
+    const vespalib::string& getName() const { return _name; }
+
+    static const Operator& get(const vespalib::stringref & name);
+
+    bool operator==(const Operator& op) const
+        { return (_name == op._name); }
+    bool operator!=(const Operator& op) const
+        { return (_name != op._name); }
+
+    virtual void print(std::ostream&, bool verbose,
+                       const std::string& indent) const;
+};
+
+class FunctionOperator : public Operator {
+private:
+    ResultList (Value::*_comparator)(const Value&) const;
+
+public:
+    FunctionOperator(const vespalib::stringref & name,
+                ResultList (Value::*comparator)(const Value&) const)
+        : Operator(name), _comparator(comparator) {}
+
+    virtual ResultList compare(const Value& a, const Value& b) const;
+    virtual ResultList trace(const Value&, const Value&, std::ostream& trace) const;
+
+    static const FunctionOperator GT;
+    static const FunctionOperator GEQ;
+    static const FunctionOperator EQ;
+    static const FunctionOperator LEQ;
+    static const FunctionOperator LT;
+    static const FunctionOperator NE;
+};
+
+class RegexOperator : public Operator {
+public:
+    RegexOperator(const vespalib::stringref & name);
+
+    // Delegates to Value::regexCompare
+    virtual ResultList compare(const Value& a, const Value& b) const;
+    virtual ResultList trace(const Value&, const Value&, std::ostream& trace) const;
+    ResultList match(const vespalib::string & val, const vespalib::stringref & expr) const;
+
+    static const RegexOperator REGEX;
+
+private:
+    friend class Value;
+    friend class ArrayValue;
+    // Note: not virtual, must be called on known type
+    ResultList compareImpl(const Value& a, const Value& b) const;
+    ResultList traceImpl(const Value&, const Value&, std::ostream& trace) const;
+};
+
+class GlobOperator : public RegexOperator {
+public:
+    GlobOperator(const vespalib::stringref & name);
+
+    // Delegates to Value::globCompare
+    virtual ResultList compare(const Value& a, const Value& b) const;
+    virtual ResultList trace(const Value&, const Value&, std::ostream& trace) const;
+    vespalib::string convertToRegex(const vespalib::stringref & globpattern) const;
+    static bool containsVariables(const vespalib::stringref & expression);
+
+    static const GlobOperator GLOB;
+private:
+    friend class Value;
+    friend class ArrayValue;
+    // Note: not virtual, must be called on known type
+    ResultList compareImpl(const Value& a, const Value& b) const;
+    ResultList traceImpl(const Value&, const Value&, std::ostream& trace) const;
+};
+
+} // select
+} // document
+
diff --git a/document/src/vespa/document/select/orderingselector.cpp b/document/src/vespa/document/select/orderingselector.cpp
new file mode 100644
index 00000000000..6b46d5cc836
--- /dev/null
+++ b/document/src/vespa/document/select/orderingselector.cpp
@@ -0,0 +1,219 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include "orderingselector.h"
+
+#include <algorithm>
+#include <vespa/document/base/documentid.h>
+#include <vespa/document/base/idstring.h>
+#include "node.h"
+#include "valuenode.h"
+#include "visitor.h"
+#include "compare.h"
+#include "branch.h"
+#include <math.h>
+
+namespace document {
+
+using namespace document::select;
+
+namespace {
+    /**
+     * Visitor class that is used for visiting a node tree generated by a
+     * document selection expression.
+     *
+     * The visitor class contains the set of buckets expression can match.
+     */
+    struct OrderingVisitor : public document::select::Visitor
+    {
+        OrderingSpecification::UP _spec;
+        OrderingSpecification::Order _order;
+
+        OrderingVisitor(OrderingSpecification::Order order)
+            : _order(order) {
+        }
+
+        OrderingSpecification::UP pickOrdering(const OrderingSpecification& a, const OrderingSpecification& b, bool isAnd) {
+            if (a.getWidthBits() == b.getWidthBits() && a.getDivisionBits() == b.getDivisionBits() && a.getOrder() == b.getOrder()) {
+                if ((a.getOrder() == OrderingSpecification::ASCENDING && isAnd) ||
+                (a.getOrder() == OrderingSpecification::DESCENDING && !isAnd)) {
+                    return OrderingSpecification::UP(
+                            new OrderingSpecification(a.getOrder(), std::max(a.getOrderingStart(), b.getOrderingStart()), b.getWidthBits(), a.getDivisionBits()));
+                } else {
+                    return OrderingSpecification::UP(
+                            new OrderingSpecification(a.getOrder(), std::min(a.getOrderingStart(), b.getOrderingStart()), b.getWidthBits(), a.getDivisionBits()));
+                }
+            }
+            return OrderingSpecification::UP();
+        }
+
+        void visitAndBranch(const document::select::And& node) {
+            OrderingVisitor left(_order);
+            node.getLeft().visit(left);
+            node.getRight().visit(*this);
+
+            if (left._spec.get() == NULL) {
+                return;
+            }
+
+            // If only left part is known return that part.
+            if (_spec.get() == NULL) {
+                _spec = std::move(left._spec);
+                return;
+            }
+
+            // Both are set...
+            _spec = pickOrdering(*_spec, *left._spec, true);
+        }
+
+        void visitOrBranch(const document::select::Or& node) {
+            OrderingVisitor left(_order);
+            node.getLeft().visit(left);
+            node.getRight().visit(*this);
+
+                // If one part is unknown we have to keep unknown status
+            if (left._spec.get() == NULL || _spec.get() == NULL) {
+                return;
+            }
+
+            _spec = pickOrdering(*_spec, *left._spec, false);
+        }
+
+        void visitNotBranch(const document::select::Not&) {
+        }
+
+        void compare(const select::IdValueNode& node,
+                     const select::ValueNode& valnode,
+                     const select::Operator& op, OrderingSpecification::Order order)
+        {
+            if (node.getType() != IdValueNode::ORDER) {
+                return;
+            }
+
+            if (node.getWidthBits() == -1 || node.getDivisionBits() == -1) {
+                return;
+            }
+
+            const IntegerValueNode* val(
+                    dynamic_cast<const IntegerValueNode*>(&valnode));
+            if (!val) return;
+
+            if (op == document::select::FunctionOperator::EQ) {
+                _spec.reset(new OrderingSpecification(order, val->getValue(), node.getWidthBits(), node.getDivisionBits()));
+            }
+
+            if (order == OrderingSpecification::ASCENDING) {
+                if (op == document::select::FunctionOperator::LEQ) {
+                    _spec.reset(new OrderingSpecification(order, 0, node.getWidthBits(), node.getDivisionBits()));
+                }
+                if (op == document::select::FunctionOperator::GT) {
+                    _spec.reset(new OrderingSpecification(order, val->getValue() + 1, node.getWidthBits(), node.getDivisionBits()));
+                }
+                if (op == document::select::FunctionOperator::GEQ) {
+                    _spec.reset(new OrderingSpecification(order, val->getValue(), node.getWidthBits(), node.getDivisionBits()));
+                }
+            } else {
+                if (op == document::select::FunctionOperator::LT) {
+                    _spec.reset(new OrderingSpecification(order, val->getValue() - 1, node.getWidthBits(), node.getDivisionBits()));
+                }
+                if (op == document::select::FunctionOperator::LEQ) {
+                    _spec.reset(new OrderingSpecification(order, val->getValue(), node.getWidthBits(), node.getDivisionBits()));
+                }
+            }
+        }
+
+        void visitComparison(const document::select::Compare& node) {
+            const IdValueNode* lid(dynamic_cast<const IdValueNode*>(
+                        &node.getLeft()));
+            if (lid) {
+                compare(*lid, node.getRight(), node.getOperator(), _order);
+            } else {
+                const IdValueNode* rid(dynamic_cast<const IdValueNode*>(
+                            &node.getRight()));
+                if (rid) {
+                    compare(*rid, node.getLeft(), node.getOperator(), _order);
+                }
+            }
+        }
+
+        void visitConstant(const document::select::Constant&) {
+        }
+
+        virtual void
+        visitInvalidConstant(const document::select::InvalidConstant &)
+        {
+        }
+
+        void visitDocumentType(const document::select::DocType&) {
+        }
+
+        virtual void
+        visitArithmeticValueNode(const ArithmeticValueNode &)
+        {
+        }
+
+        virtual void
+        visitFunctionValueNode(const FunctionValueNode &)
+        {
+        }
+
+        virtual void
+        visitIdValueNode(const IdValueNode &)
+        {
+        }
+
+        virtual void
+        visitSearchColumnValueNode(const SearchColumnValueNode &)
+        {
+        }
+
+        virtual void
+        visitFieldValueNode(const FieldValueNode &)
+        {
+        }
+
+        virtual void
+        visitFloatValueNode(const FloatValueNode &)
+        {
+        }
+
+        virtual void
+        visitVariableValueNode(const VariableValueNode &)
+        {
+        }
+
+        virtual void
+        visitIntegerValueNode(const IntegerValueNode &)
+        {
+        }
+
+        virtual void
+        visitCurrentTimeValueNode(const CurrentTimeValueNode &)
+        {
+        }
+
+        virtual void
+        visitStringValueNode(const StringValueNode &)
+        {
+        }
+
+        virtual void
+        visitNullValueNode(const NullValueNode &)
+        {
+        }
+
+        virtual void
+        visitInvalidValueNode(const InvalidValueNode &)
+        {
+        }
+    };
+}
+
+OrderingSpecification::UP
+OrderingSelector::select(const document::select::Node& expression, OrderingSpecification::Order order) const
+{
+    OrderingVisitor v(order);
+    expression.visit(v);
+    return std::move(v._spec);
+}
+
+} // document
diff --git a/document/src/vespa/document/select/orderingselector.h b/document/src/vespa/document/select/orderingselector.h
new file mode 100644
index 00000000000..a8f6c02d45c
--- /dev/null
+++ b/document/src/vespa/document/select/orderingselector.h
@@ -0,0 +1,28 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vector>
+#include <utility>
+#include <memory>
+#include "orderingspecification.h"
+
+namespace document {
+namespace select {
+    class Node;
+}
+
+class OrderingSelector {
+public:
+    /**
+     * Return the ordering specification implied by this document selection expression.
+     *
+     * @param expression The document selection expression to parse.
+     * @param ordering The ordering the user has selected to visit (ASCENDING/DESCENDING)
+     */
+    OrderingSpecification::UP select(const select::Node& expression, OrderingSpecification::Order ordering) const;
+
+private:
+};
+
+} // document
+
diff --git a/document/src/vespa/document/select/orderingspecification.cpp b/document/src/vespa/document/select/orderingspecification.cpp
new file mode 100644
index 00000000000..f42c057de25
--- /dev/null
+++ b/document/src/vespa/document/select/orderingspecification.cpp
@@ -0,0 +1,20 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include "orderingspecification.h"
+#include <sstream>
+
+namespace document {
+
+bool
+OrderingSpecification::operator==(const OrderingSpecification& other) const {
+    return _order == other._order && _orderingStart == other._orderingStart && _widthBits == other._widthBits && _divisionBits == other._divisionBits;
+}
+
+std::string
+OrderingSpecification::toString() const {
+    std::ostringstream ost;
+    ost << (_order == ASCENDING ? "+" : "-") << "," << _widthBits << "," << _divisionBits << "," << _orderingStart;
+    return ost.str();
+}
+
+}
diff --git a/document/src/vespa/document/select/orderingspecification.h b/document/src/vespa/document/select/orderingspecification.h
new file mode 100644
index 00000000000..0fa64862fa0
--- /dev/null
+++ b/document/src/vespa/document/select/orderingspecification.h
@@ -0,0 +1,47 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/fastos/types.h>
+#include <string>
+
+namespace document {
+
+class OrderingSpecification {
+public:
+    typedef std::unique_ptr<OrderingSpecification> UP;
+
+    enum Order { ASCENDING = 0, DESCENDING };
+
+    OrderingSpecification()
+        : _order(ASCENDING), _orderingStart(0), _widthBits(0), _divisionBits(0) {};
+
+    OrderingSpecification(Order order)
+        : _order(order), _orderingStart(0), _widthBits(0), _divisionBits(0) {};
+
+    OrderingSpecification(Order order, uint64_t orderingStart, uint16_t widthBits, uint16_t divisionBits)
+        : _order(order), _orderingStart(orderingStart), _widthBits(widthBits), _divisionBits(divisionBits) {}
+
+    Order getOrder() const { return _order; }
+    uint64_t getOrderingStart() const { return _orderingStart; }
+    uint16_t getWidthBits() const { return _widthBits; }
+    uint16_t getDivisionBits() const { return _divisionBits; }
+
+    bool operator==(const OrderingSpecification& other) const;
+
+    std::string toString() const;
+
+private:
+    Order _order;
+    uint64_t _orderingStart;
+    uint16_t _widthBits;
+    uint16_t _divisionBits;
+};
+
+inline std::ostream&
+operator<<(std::ostream& out, const OrderingSpecification& o)
+{
+    out << o.toString();
+    return out;
+}
+}
+
diff --git a/document/src/vespa/document/select/parser.cpp b/document/src/vespa/document/select/parser.cpp
new file mode 100644
index 00000000000..0dc9c1f1376
--- /dev/null
+++ b/document/src/vespa/document/select/parser.cpp
@@ -0,0 +1,1493 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "parser.h"
+
+#include <boost/spirit/include/classic_chset.hpp>
+#include <boost/spirit/include/classic_core.hpp>
+#include <boost/spirit/include/classic_escape_char.hpp>
+#include <boost/spirit/include/classic_grammar_def.hpp>
+#include <boost/spirit/include/classic_parse_tree.hpp>
+#include <boost/spirit/include/classic_tree_to_xml.hpp>
+#include <vespa/document/repo/documenttyperepo.h>
+#include <iostream>
+#include <map>
+#include <sstream>
+#include <vespa/document/base/exceptions.h>
+#include "branch.h"
+#include "compare.h"
+#include "constant.h"
+#include "operator.h"
+#include "doctype.h"
+#include "valuenode.h"
+#include "simpleparser.h"
+#include <vespa/document/util/stringutil.h>
+#include <vespa/vespalib/stllike/asciistream.h>
+
+using boost::spirit::classic::tree_node;
+using document::DocumentTypeRepo;
+using std::unique_ptr;
+using std::cerr;
+using std::endl;
+using std::istringstream;
+using std::ostringstream;
+using vespalib::IllegalStateException;
+
+/*
+ * This cannot be part of a plugin.  boost contains constructs causing
+ * compiler to generate calls to atexit().
+ */
+
+#define parse_assert(a)
+
+namespace document {
+namespace select {
+
+VESPA_IMPLEMENT_EXCEPTION(ParsingFailedException, vespalib::Exception);
+
+Parser::Parser(const DocumentTypeRepo& repo,
+               const BucketIdFactory& bucketIdFactory)
+    : _repo(repo),
+      _bucketIdFactory(bucketIdFactory)
+{
+}
+
+namespace {
+
+/**
+ * Defines the grammar for the document selection text format.
+ */
+struct DocSelectionGrammar
+    : public boost::spirit::classic::grammar<DocSelectionGrammar>
+{
+    /** Node identifiers (value 0 should not be used) */
+    enum ids { id_nil=1, id_bool, id_number, id_string,
+               id_doctype, id_fieldname, id_function, id_idarg, id_searchcolumnarg,
+               id_operator, id_idspec, id_searchcolumnspec, id_fieldspec, id_value,
+               id_valuefuncadd, id_valuefuncmul, id_valuefuncmod,
+               id_valuegroup, id_arithmvalue,
+               id_comparison, id_leaf, id_not, id_and,
+               id_or, id_group, id_order, id_expression, id_variable };
+
+    const DocumentTypeRepo &_repo;
+    const BucketIdFactory& _bucketIdFactory;
+
+    DocSelectionGrammar(const DocumentTypeRepo& repo,
+                        const BucketIdFactory& bucketIdFactory)
+        : _repo(repo),
+          _bucketIdFactory(bucketIdFactory) {}
+
+    const BucketIdFactory& getBucketIdFactory() const
+    { return _bucketIdFactory; }
+
+    /** Grammar base types. To be able to retrieve different grammars. */
+    template <typename Scanner>
+    struct gram_base {
+        typedef typename boost::spirit::classic::rule<Scanner,
+                                             boost::spirit::classic::parser_tag<id_nil> > rule_nil;
+        typedef typename boost::spirit::classic::rule<Scanner,
+                                             boost::spirit::classic::parser_tag<id_bool> > rule_bool;
+        typedef typename boost::spirit::classic::rule<Scanner,
+                                             boost::spirit::classic::parser_tag<id_number> > rule_number;
+        typedef typename boost::spirit::classic::rule<Scanner,
+                                             boost::spirit::classic::parser_tag<id_string> > rule_string;
+        typedef typename boost::spirit::classic::rule<Scanner,
+                                             boost::spirit::classic::parser_tag<id_doctype> > rule_doctype;
+        typedef typename boost::spirit::classic::rule<Scanner,
+                                             boost::spirit::classic::parser_tag<id_fieldname> > rule_fieldname;
+        typedef typename boost::spirit::classic::rule<Scanner,
+                                             boost::spirit::classic::parser_tag<id_function> > rule_function;
+        typedef typename boost::spirit::classic::rule<Scanner,
+                                             boost::spirit::classic::parser_tag<id_idarg> > rule_idarg;
+        typedef typename boost::spirit::classic::rule<Scanner,
+                                             boost::spirit::classic::parser_tag<id_searchcolumnarg> > rule_searchcolumnarg;
+        typedef typename boost::spirit::classic::rule<Scanner,
+                                             boost::spirit::classic::parser_tag<id_operator> > rule_operator;
+        typedef typename boost::spirit::classic::rule<Scanner,
+                                             boost::spirit::classic::parser_tag<id_idspec> > rule_idspec;
+        typedef typename boost::spirit::classic::rule<Scanner,
+                                             boost::spirit::classic::parser_tag<id_searchcolumnspec> > rule_searchcolumnspec;
+        typedef typename boost::spirit::classic::rule<Scanner,
+                                             boost::spirit::classic::parser_tag<id_fieldspec> > rule_fieldspec;
+        typedef typename boost::spirit::classic::rule<Scanner,
+                                             boost::spirit::classic::parser_tag<id_value> > rule_value;
+        typedef typename boost::spirit::classic::rule<Scanner,
+                                             boost::spirit::classic::parser_tag<id_valuefuncadd> > rule_valuefuncadd;
+        typedef typename boost::spirit::classic::rule<Scanner,
+                                             boost::spirit::classic::parser_tag<id_valuefuncmul> > rule_valuefuncmul;
+        typedef typename boost::spirit::classic::rule<Scanner,
+                                             boost::spirit::classic::parser_tag<id_valuefuncmod> > rule_valuefuncmod;
+        typedef typename boost::spirit::classic::rule<Scanner,
+                                             boost::spirit::classic::parser_tag<id_valuegroup> > rule_valuegroup;
+        typedef typename boost::spirit::classic::rule<Scanner,
+                                             boost::spirit::classic::parser_tag<id_arithmvalue> > rule_arithmvalue;
+        typedef typename boost::spirit::classic::rule<Scanner,
+                                             boost::spirit::classic::parser_tag<id_comparison> > rule_comparison;
+        typedef typename boost::spirit::classic::rule<Scanner,
+                                             boost::spirit::classic::parser_tag<id_leaf> > rule_leaf;
+        typedef typename boost::spirit::classic::rule<Scanner,
+                                             boost::spirit::classic::parser_tag<id_not> > rule_not;
+        typedef typename boost::spirit::classic::rule<Scanner,
+                                             boost::spirit::classic::parser_tag<id_and> > rule_and;
+        typedef typename boost::spirit::classic::rule<Scanner,
+                                             boost::spirit::classic::parser_tag<id_or> > rule_or;
+        typedef typename boost::spirit::classic::rule<Scanner,
+                                             boost::spirit::classic::parser_tag<id_group> > rule_group;
+        typedef typename boost::spirit::classic::rule<Scanner,
+                                             boost::spirit::classic::parser_tag<id_order> > rule_order;
+        typedef typename boost::spirit::classic::rule<Scanner,
+                                             boost::spirit::classic::parser_tag<id_expression> > rule_expression;
+        typedef typename boost::spirit::classic::rule<Scanner,
+                                             boost::spirit::classic::parser_tag<id_variable> > rule_variable;
+        typedef boost::spirit::classic::grammar_def<rule_expression,
+                                           rule_leaf,
+                                           rule_arithmvalue> type;
+    };
+
+    template <typename Scanner>
+    struct definition : gram_base<Scanner>::type
+    {
+        typename gram_base<Scanner>::rule_nil _nil;
+        typename gram_base<Scanner>::rule_bool _bool;
+        typename gram_base<Scanner>::rule_number _number;
+        typename gram_base<Scanner>::rule_string _string;
+        typename gram_base<Scanner>::rule_doctype _doctype;
+        typename gram_base<Scanner>::rule_fieldname _fieldname;
+        typename gram_base<Scanner>::rule_function _function;
+        typename gram_base<Scanner>::rule_idarg _idarg;
+        typename gram_base<Scanner>::rule_searchcolumnarg _searchcolumnarg;
+        typename gram_base<Scanner>::rule_operator _operator;
+        typename gram_base<Scanner>::rule_idspec _idspec;
+        typename gram_base<Scanner>::rule_searchcolumnspec _searchcolumnspec;
+        typename gram_base<Scanner>::rule_fieldspec _fieldspec;
+        typename gram_base<Scanner>::rule_value _value;
+        typename gram_base<Scanner>::rule_valuefuncadd _valuefuncadd;
+        typename gram_base<Scanner>::rule_valuefuncmul _valuefuncmul;
+        typename gram_base<Scanner>::rule_valuefuncmod _valuefuncmod;
+        typename gram_base<Scanner>::rule_valuegroup _valuegroup;
+        typename gram_base<Scanner>::rule_arithmvalue _arithmvalue;
+        typename gram_base<Scanner>::rule_comparison _comparison;
+        typename gram_base<Scanner>::rule_leaf _leaf;
+        typename gram_base<Scanner>::rule_not _not;
+        typename gram_base<Scanner>::rule_and _and;
+        typename gram_base<Scanner>::rule_or _or;
+        typename gram_base<Scanner>::rule_group _group;
+        typename gram_base<Scanner>::rule_order _order;
+        typename gram_base<Scanner>::rule_expression _expression;
+        typename gram_base<Scanner>::rule_variable _variable;
+
+        definition(const DocSelectionGrammar&)
+            : _nil(),
+              _bool(),
+              _number(),
+              _string(),
+              _doctype(),
+              _fieldname(),
+              _function(),
+              _idarg(),
+              _operator(),
+              _idspec(),
+              _searchcolumnspec(),
+              _fieldspec(),
+              _value(),
+              _valuefuncadd(),
+              _valuefuncmul(),
+              _valuefuncmod(),
+              _valuegroup(),
+              _arithmvalue(),
+              _comparison(),
+              _leaf(),
+              _not(),
+              _and(),
+              _or(),
+              _group(),
+              _order(),
+              _expression(),
+              _variable()
+        {
+            using namespace boost::spirit::classic;
+
+            boost::spirit::classic::uint_parser<uint64_t, 16, 1, -1> hexvalue;
+
+            // Initialize primitives
+            _nil = lexeme_d[ as_lower_d["null"] ];
+            _bool = lexeme_d[ as_lower_d["true"] | as_lower_d["false"] ];
+            _number = lexeme_d[ str_p("0x") >> hexvalue ] | lexeme_d[ real_p ];
+            _string = ( lexeme_d[
+                                ( no_node_d[ ch_p('"') ] >>
+                                 token_node_d[ *( ~chset<>("\\\"\x00-\x1f\x7f-\xff") |
+                                                       ( '\\' >> ( ch_p('\\') | 't' | 'n' | 'f' | 'r' | '"' |
+                                                               (ch_p('x') >> xdigit_p >> xdigit_p) ) ) ) ] >>
+                                 no_node_d[ ch_p('"') ] ) |
+                                ( no_node_d[ ch_p('\'') ] >>
+                                 token_node_d[ *( ~chset<>("\\'\x00-\x1f\x7f-\xff") |
+                                                       ( '\\' >> ( ch_p('\\') | 't' | 'n' | 'f' | 'r' | '\'' |
+                                                               (ch_p('x') >> xdigit_p >> xdigit_p) ) ) ) ] >>
+                                 no_node_d[ ch_p('\'') ] )
+                        ] );
+            _doctype = lexeme_d[ token_node_d[ chset<>("_A-Za-z")
+                                         >> *(chset<>("_A-Za-z0-9")) ]];
+            _fieldname = lexeme_d[ token_node_d[chset<>("_A-Za-z")
+                                 >> *(chset<>("_A-Za-z0-9{}[]$"))
+                               ]];
+            _function = lexeme_d[ token_node_d[ chset<>("A-Za-z")
+                                                    >> *(chset<>("A-Za-z0-9")) ]
+                                 >> no_node_d[ str_p("()") ] ];
+
+            _order = as_lower_d["order"]
+                     >> no_node_d[ ch_p('(') ]
+                     >> _number
+                     >> no_node_d[ ch_p(',') ]
+                     >> _number
+                     >> no_node_d[ ch_p(')') ];
+
+            _idarg = (as_lower_d[ "scheme"]    | as_lower_d[ "namespace"] |
+                      as_lower_d[ "specific" ] | as_lower_d[ "user" ] |
+                      as_lower_d[ "group" ] | as_lower_d[ "bucket" ] | 
+                      as_lower_d[ "gid" ] | as_lower_d["type"] | _order);
+
+            _searchcolumnarg = lexeme_d[ token_node_d[ *(chset<>("_A-Za-z0-9")) ]];
+            _operator = (str_p(">=") | ">" | "==" | "=~" | "="
+                         | "<=" | "<" | "!=");
+            // Derived
+            _idspec = as_lower_d["id"]
+                      >> !(no_node_d[ ch_p('.') ] >> _idarg);
+            _searchcolumnspec = as_lower_d["searchcolumn"]
+                                >> !(no_node_d[ ch_p('.') ] >> _searchcolumnarg);
+            _fieldspec = _doctype
+                         >> +( no_node_d[ ch_p('.') ] >> (_function | _fieldname));
+            _variable = lexeme_d[ token_node_d[chset<>("$")
+                                 >> *(chset<>("A-Za-z0-9"))
+                               ]];
+            _value = (_valuegroup | _function | _nil | _number | _string
+                      | _idspec | _searchcolumnspec | _fieldspec | _variable)
+                    >> *(no_node_d[ ch_p('.') ] >> _function);
+            _valuefuncmod = (_valuegroup | _value)
+                            >> +( ch_p('%')
+                                 >> (_valuegroup | _value) );
+            _valuefuncmul = (_valuefuncmod | _valuegroup | _value)
+                            >> +( (ch_p('*') | ch_p('/'))
+                                 >> (_valuefuncmod | _valuegroup | _value));
+            _valuefuncadd
+                = (_valuefuncmul | _valuefuncmod | _valuegroup | _value)
+                >> +((ch_p('+') | ch_p('-'))
+                     >> (_valuefuncmul | _valuefuncmod | _valuegroup |
+                         _value));
+            _valuegroup = no_node_d[ ch_p('(') ] >> _arithmvalue
+                        >> no_node_d[ ch_p(')') ]
+                        >> *(no_node_d[ ch_p('.') ] >> _function);
+            _arithmvalue = (_valuefuncadd | _valuefuncmul | _valuefuncmod
+                            | _valuegroup | _value);
+            _comparison = _arithmvalue >> _operator >> _arithmvalue;
+            _leaf = _bool | _comparison | _fieldspec | _doctype;
+
+            _not = (as_lower_d["not"] >> _group)
+                   | (lexeme_d[ as_lower_d["not"] >> no_node_d[ space_p ] ] >> _leaf);
+            _and = (_not | _group | _leaf)
+                   >> as_lower_d["and"] >> (_and | _not | _group | _leaf);
+            _or  = (_and | _not | _group | _leaf)
+                   >> as_lower_d["or"] >> (_or | _and | _not | _group | _leaf);
+            _group =  no_node_d[ ch_p('(') ]
+                      >> (_or | _and | _not | _group | _leaf)
+                      >> no_node_d[ ch_p(')') ];
+
+            _expression = !(_or | _and | _not | _group | _leaf)   >> end_p;
+
+            this->start_parsers(_expression, _leaf, _arithmvalue);
+        }
+    };
+
+};
+
+template<typename T>
+std::unique_ptr<Node>
+parseTree(DocSelectionGrammar& grammar, tree_node<T>& root) {
+    return parseNode(grammar, root);
+}
+
+template<typename T>
+std::unique_ptr<Node>
+parseNode(DocSelectionGrammar& grammar, tree_node<T>& node) {
+    switch (node.value.id().to_long()) {
+    case DocSelectionGrammar::id_or:
+        return parseOr(grammar, node);
+    case DocSelectionGrammar::id_and:
+        return parseAnd(grammar, node);
+    case DocSelectionGrammar::id_not:
+        return parseNot(grammar, node);
+    case DocSelectionGrammar::id_group:
+    {
+        std::unique_ptr<Node> n(parseNode(grammar, node.children[0]));
+        n->setParentheses();
+        return n;
+    }
+    case DocSelectionGrammar::id_leaf:
+    case DocSelectionGrammar::id_value:
+        parse_assert(node.children.size() == 1);
+        return parseNode(grammar, node.children[0]);
+    case DocSelectionGrammar::id_expression:
+        if (node.children.size() == 1) {
+            return parseNode(grammar, node.children[0]);
+        }
+        parse_assert(node.children.size() == 0);
+        return std::unique_ptr<Node>(new Constant("true"));
+    case DocSelectionGrammar::id_bool:
+        return parseBool(grammar, node);
+    case DocSelectionGrammar::id_comparison:
+        return parseComparison(grammar, node);
+    case DocSelectionGrammar::id_fieldspec:
+        return parseFieldSpec(grammar, node);
+    case DocSelectionGrammar::id_doctype:
+        return parseDocType(grammar, node);
+    }
+    vespalib::asciistream ost;
+    ost << "Received unhandled nodetype "
+        << node.value.id().to_long() << " in parseNode()\n";
+    throw IllegalStateException(ost.str(), VESPA_STRLOC);
+}
+
+template<typename T>
+std::unique_ptr<Node>
+parseOr(DocSelectionGrammar& grammar, tree_node<T>& node) {
+    parse_assert(node.value.id().to_long() == grammar.id_or);
+    parse_assert(node.children.size() == 3);
+    vespalib::string op(node.children[1].value.begin(),
+                   node.children[1].value.end());
+    return std::unique_ptr<Node>(new Or(
+                                       parseNode(grammar, node.children[0]),
+                                       parseNode(grammar, node.children[2]),
+                                       op.c_str()));
+}
+
+template<typename T>
+std::unique_ptr<Node>
+parseAnd(DocSelectionGrammar& grammar, tree_node<T>& node) {
+    parse_assert(node.value.id().to_long() == grammar.id_and);
+    parse_assert(node.children.size() == 3);
+    vespalib::string op(node.children[1].value.begin(),
+                   node.children[1].value.end());
+    return std::unique_ptr<Node>(new And(
+                                       parseNode(grammar, node.children[0]),
+                                       parseNode(grammar, node.children[2]),
+                                       op.c_str()));
+}
+
+template<typename T>
+std::unique_ptr<Node>
+parseNot(DocSelectionGrammar& grammar, tree_node<T>& node) {
+    parse_assert(node.value.id().to_long() == grammar.id_not);
+    parse_assert(node.children.size() == 2);
+    vespalib::string op(node.children[0].value.begin(),
+                   node.children[0].value.end());
+    return std::unique_ptr<Node>(new Not(
+                                       parseNode(grammar, node.children[1]), op.c_str()));
+}
+
+template<typename T>
+std::unique_ptr<Node>
+parseBool(DocSelectionGrammar& grammar, tree_node<T>& node) {
+    (void) grammar;
+    parse_assert(node.value.id().to_long() == grammar.id_bool);
+    parse_assert(node.children.size() == 1);
+    parse_assert(node.children[0].value.id().to_long() == grammar.id_bool);
+    parse_assert(node.children[0].children.size() == 0);
+    vespalib::string s(node.children[0].value.begin(), node.children[0].value.end());
+    return std::unique_ptr<Node>(new Constant(s));
+}
+
+template<typename T>
+std::unique_ptr<Node>
+parseComparison(DocSelectionGrammar& grammar, tree_node<T>& node) {
+    parse_assert(node.value.id().to_long() == grammar.id_comparison);
+    parse_assert(node.children.size() == 3);
+    parse_assert(node.children[1].children.size() == 1);
+    vespalib::string op(node.children[1].children[0].value.begin(),
+                   node.children[1].children[0].value.end());
+    return std::unique_ptr<Node>(new Compare(
+                                       parseArithmValue(grammar, node.children[0]),
+                                       Operator::get(op),
+                                       parseArithmValue(grammar, node.children[2]),
+                                       grammar.getBucketIdFactory()));
+}
+
+template<typename T>
+std::unique_ptr<Node>
+parseFieldSpec(DocSelectionGrammar& grammar, tree_node<T>& node) {
+    parse_assert(node.value.id().to_long() == grammar.id_fieldspec);
+    return std::unique_ptr<Node>(new Compare(
+                                       parseFieldSpecValue(grammar, node),
+                                       Operator::get("!="),
+                                       std::unique_ptr<ValueNode>(new NullValueNode("null")),
+                                       grammar.getBucketIdFactory()));
+}
+
+template<typename T>
+std::unique_ptr<ValueNode>
+parseVariable(DocSelectionGrammar& grammar, tree_node<T>& node) {
+    (void) grammar;
+    parse_assert(node.value.id().to_long() == grammar.id_variable);
+    vespalib::string varName(node.children[0].value.begin(),
+                        node.children[0].value.end());
+    return std::unique_ptr<ValueNode>(new VariableValueNode(varName.substr(1)));
+}
+
+template<typename T>
+std::unique_ptr<ValueNode>
+parseGlobValueFunction(DocSelectionGrammar& grammar, tree_node<T>& node) {
+    (void) grammar;
+    parse_assert(node.value.id().to_long() == grammar.id_function);
+    vespalib::string varName(node.children[0].value.begin(),
+                             node.children[0].value.end());
+    if (varName == "now") {
+        return std::unique_ptr<ValueNode>(new CurrentTimeValueNode);
+    }
+    throw ParsingFailedException("Unexpected function name '" + varName
+            + "' found.", VESPA_STRLOC);
+}
+
+template<typename T>
+std::unique_ptr<Node>
+parseDocType(DocSelectionGrammar& grammar, tree_node<T>& node) {
+    parse_assert(node.value.id().to_long() == grammar.id_doctype);
+    parse_assert(node.children.size() == 1);
+    parse_assert(node.children[0].value.id().to_long() == grammar.id_doctype);
+    parse_assert(node.children[0].children.size() == 0);
+    vespalib::string doctype(node.children[0].value.begin(),
+                        node.children[0].value.end());
+    // Verify existance of any version of document
+    if (!grammar._repo.getDocumentType(doctype)) {
+        throw ParsingFailedException("Document type " + doctype + " not found",
+                                     VESPA_STRLOC);
+    }
+    return std::unique_ptr<Node>(new DocType(doctype));
+}
+
+template<typename T>
+std::unique_ptr<ValueNode>
+addFunctions(DocSelectionGrammar& grammar, tree_node<T>& node,
+             std::unique_ptr<ValueNode> src, uint32_t index)
+{
+    (void) grammar;
+    while (index < node.children.size()) {
+        parse_assert(node.children[index].value.id().to_long()
+               == grammar.id_function);
+        vespalib::string func(node.children[index].children[0].value.begin(),
+                         node.children[index].children[0].value.end());
+        std::unique_ptr<ValueNode> fnode(new FunctionValueNode(func, std::move(src)));
+        src = std::move(fnode);
+        ++index;
+    }
+    return std::move(src);
+}
+
+template<typename T>
+std::unique_ptr<ValueNode>
+parseArithmValue(DocSelectionGrammar& grammar, tree_node<T>& node) {
+    switch (node.value.id().to_long()) {
+    case DocSelectionGrammar::id_arithmvalue:
+        parse_assert(node.children.size() == 1);
+        return parseArithmValue(grammar, node.children[0]);
+    case DocSelectionGrammar::id_value:
+        return parseValue(grammar, node);
+    case DocSelectionGrammar::id_valuegroup:
+        return parseValueGroup(grammar, node);
+    case DocSelectionGrammar::id_valuefuncadd:
+    case DocSelectionGrammar::id_valuefuncmul:
+    case DocSelectionGrammar::id_valuefuncmod:
+        return parseValueArithmetics(grammar, node);
+    }
+    vespalib::asciistream ost;
+    ost << "Received unhandled nodetype "
+        << node.value.id().to_long()
+        << " in parseArithmValue()\n";
+    throw IllegalStateException(ost.str(), VESPA_STRLOC);
+}
+
+template<typename T>
+std::unique_ptr<ValueNode>
+parseValueArithmetics(DocSelectionGrammar& grammar, tree_node<T>& node) {
+    parse_assert(node.children.size() >= 3 && node.children.size() % 2 == 1);
+    std::unique_ptr<ValueNode> lhs(parseArithmValue(grammar, node.children[0]));
+    for (unsigned int i = 1; i < node.children.size(); i += 2) {
+        vespalib::string op(node.children[i].value.begin(),
+                            node.children[i].value.end());
+        std::unique_ptr<ValueNode> rhs(parseArithmValue(grammar,
+                                                      node.children[i + 1]));
+        std::unique_ptr<ValueNode> res(
+                new ArithmeticValueNode(std::move(lhs), op, std::move(rhs)));
+        lhs = std::move(res);
+    }
+    return lhs;
+}
+
+template<typename T>
+std::unique_ptr<ValueNode>
+parseValueGroup(DocSelectionGrammar& grammar, tree_node<T>& node) {
+    parse_assert(node.value.id().to_long() == grammar.id_valuegroup);
+    parse_assert(node.children.size() >= 1);
+    std::unique_ptr<ValueNode> result(
+            parseArithmValue(grammar, node.children[0]));
+    result->setParentheses();
+    return addFunctions(grammar, node, std::move(result), 1);
+}
+
+template<typename T>
+std::unique_ptr<ValueNode>
+parseValue(DocSelectionGrammar& grammar, tree_node<T>& node) {
+    parse_assert(node.value.id().to_long() == grammar.id_value);
+    parse_assert(node.children.size() >= 1);
+    std::unique_ptr<ValueNode> result;
+    switch (node.children[0].value.id().to_long()) {
+    case DocSelectionGrammar::id_nil:
+        result = parseNilValue(grammar, node.children[0]);
+        break;
+    case DocSelectionGrammar::id_idspec:
+        result = parseIdSpecValue(grammar, node.children[0]);
+        break;
+    case DocSelectionGrammar::id_searchcolumnspec:
+        result = parseSearchColumnSpecValue(grammar, node.children[0]);
+        break;
+    case DocSelectionGrammar::id_fieldspec:
+        result = parseFieldSpecValue(grammar, node.children[0]);
+        break;
+    case DocSelectionGrammar::id_number:
+        result = parseNumberValue(grammar, node.children[0]);
+        break;
+    case DocSelectionGrammar::id_string:
+        result = parseStringValue(grammar, node.children[0]);
+        break;
+    case DocSelectionGrammar::id_valuegroup:
+        result = parseValueGroup(grammar, node.children[0]);
+        break;
+    case DocSelectionGrammar::id_variable:
+        result = parseVariable(grammar, node.children[0]);
+        break;
+    case DocSelectionGrammar::id_function:
+        result = parseGlobValueFunction(grammar, node.children[0]);
+        break;
+    default:
+        vespalib::asciistream ost;
+        ost << "Received unhandled nodetype "
+            << node.children[0].value.id().to_long()
+            << " in parseValue(), from node of type "
+            << node.value.id().to_long() << "\n";
+        throw IllegalStateException(ost.str(), VESPA_STRLOC);
+    }
+    return addFunctions(grammar, node, std::move(result), 1);
+}
+
+template<typename T>
+std::unique_ptr<ValueNode>
+parseNilValue(DocSelectionGrammar& grammar, tree_node<T>& node) {
+    (void) grammar;
+    parse_assert(node.value.id().to_long() == grammar.id_nil);
+    parse_assert(node.children.size() == 1);
+    parse_assert(node.children[0].children.size() == 0);
+    vespalib::string op(node.children[0].value.begin(),
+                   node.children[0].value.end());
+    return std::unique_ptr<ValueNode>(new NullValueNode(op));
+}
+
+template<typename T>
+std::unique_ptr<ValueNode>
+parseIdSpecValue(DocSelectionGrammar& grammar, tree_node<T>& node) {
+    parse_assert(node.value.id().to_long() == grammar.id_idspec);
+    parse_assert(node.children.size() >= 1);
+    parse_assert(node.children[0].children.size() == 0);
+    vespalib::string id(node.children[0].value.begin(),
+                   node.children[0].value.end());
+    if (node.children.size() == 1) {
+        return std::unique_ptr<ValueNode>(
+                new IdValueNode(grammar.getBucketIdFactory(), id, ""));
+    }
+
+    vespalib::string type;
+
+    int widthBits = -1;
+    int divisionBits = -1;
+
+    if (node.children[1].children[0].value.id().to_long() == grammar.id_order) {
+        tree_node<T>& ordernode(node.children[1].children[0]);
+        type = vespalib::string(ordernode.children[0].value.begin(),
+                           ordernode.children[0].value.end());
+
+        vespalib::string val = vespalib::string(
+                ordernode.children[1].children[0].value.begin(),
+                ordernode.children[1].children[0].value.end());
+        widthBits = atoi(val.c_str());
+
+        val = vespalib::string(ordernode.children[2].children[0].value.begin(),
+                          ordernode.children[2].children[0].value.end());
+        divisionBits = atoi(val.c_str());
+    } else {
+        type = vespalib::string(node.children[1].children[0].value.begin(),
+                           node.children[1].children[0].value.end());
+    }
+
+    return std::unique_ptr<ValueNode>(
+            new IdValueNode(grammar.getBucketIdFactory(), id, type,
+                            widthBits, divisionBits));
+}
+
+template<typename T>
+std::unique_ptr<ValueNode>
+parseSearchColumnSpecValue(DocSelectionGrammar& grammar, tree_node<T>& node) {
+    parse_assert(node.value.id().to_long() == grammar.id_searchcolumnspec);
+    parse_assert(node.children.size() == 2);
+    parse_assert(node.children[0].children.size() == 0);
+    parse_assert(node.children[1].value.id().to_long() == grammar.id_searchcolumnarg);
+
+    vespalib::string id(node.children[0].value.begin(),
+                   node.children[0].value.end());
+    parse_assert(node.children.size() == 2);
+
+    vespalib::string val = vespalib::string(node.children[1].children[0].value.begin(),
+                                  node.children[1].children[0].value.end());
+    return std::unique_ptr<ValueNode>(new SearchColumnValueNode(
+                                            grammar.getBucketIdFactory(), id, atoi(val.c_str())));
+}
+
+template<typename T>
+std::unique_ptr<ValueNode>
+parseFieldSpecValue(DocSelectionGrammar& grammar, tree_node<T>& node) {
+    parse_assert(node.value.id().to_long() == grammar.id_fieldspec);
+    parse_assert(node.children.size() >= 2);
+    parse_assert(node.children[0].value.id().to_long() == grammar.id_doctype);
+    vespalib::string doctype(node.children[0].children[0].value.begin(),
+                        node.children[0].children[0].value.end());
+    // Verify that document type exist at any version
+    if (!grammar._repo.getDocumentType(doctype)) {
+        throw ParsingFailedException("Document type " + doctype + " not found",
+                                     VESPA_STRLOC);
+    }
+    std::unique_ptr<ValueNode> value;
+    uint32_t iterator = 2;
+
+    parse_assert(node.children[1].value.id().to_long() == grammar.id_fieldname);
+    vespalib::string field(node.children[1].children[0].value.begin(),
+                           node.children[1].children[0].value.end());
+    while (iterator < node.children.size()
+           && node.children[iterator].value.id().to_long() == grammar.id_fieldname)
+    {
+        field += "." + vespalib::string(
+                node.children[iterator].children[0].value.begin(),
+                node.children[iterator].children[0].value.end());
+        ++iterator;
+    }
+    value.reset(new FieldValueNode(doctype, field));
+
+    for (; iterator<node.children.size(); ++iterator) {
+        std::unique_ptr<ValueNode> child(std::move(value));
+        vespalib::string function(node.children[iterator].children[0].value.begin(),
+                             node.children[iterator].children[0].value.end());
+        parse_assert(node.children[iterator].value.id().to_long() == grammar.id_function);
+        value.reset(new FunctionValueNode(function, std::move(child)));
+    }
+    return value;
+}
+
+template<typename T>
+std::unique_ptr<ValueNode>
+parseNumberValue(DocSelectionGrammar& grammar, tree_node<T>& node) {
+    (void) grammar;
+    parse_assert(node.value.id().to_long() == grammar.id_number);
+    vespalib::string sval;
+    int base = 10;
+    if (node.children.size() == 2) {
+        base = 16;
+        sval = vespalib::string(node.children[1].value.begin(),
+                           node.children[1].value.end());
+        parse_assert(node.children[0].value.id().to_long() == grammar.id_number);
+        parse_assert(node.children[1].value.id().to_long() == grammar.id_number);
+    } else {
+        parse_assert(node.children.size() == 1);
+        sval = vespalib::string(node.children[0].value.begin(),
+                           node.children[0].value.end());
+        parse_assert(node.children[0].value.id().to_long() == grammar.id_number);
+    }
+    if (sval.find('.') != vespalib::string::npos) {
+        char* endptr;
+        double val = strtod(sval.c_str(), &endptr);
+        if (*endptr == '\0') {
+            return std::unique_ptr<ValueNode>(new FloatValueNode(val));
+        }
+    } else {
+        char* endptr;
+        int64_t val;
+        if (base == 16) {
+            val = strtoull(sval.c_str(), &endptr, base);
+        } else {
+            val = strtoll(sval.c_str(), &endptr, base);
+        }
+        if (*endptr == '\0') {
+            return std::unique_ptr<ValueNode>(new IntegerValueNode(val, false));
+        }
+    }
+    vespalib::string error = "'" + sval + "' is not a valid number.";
+    throw ParsingFailedException(error, VESPA_STRLOC);
+}
+
+template<typename T>
+std::unique_ptr<ValueNode>
+parseStringValue(DocSelectionGrammar& grammar, tree_node<T>& node) {
+    (void) grammar;
+    parse_assert(node.value.id().to_long() == grammar.id_string);
+    if (node.children.size() == 0) {
+        return std::unique_ptr<ValueNode>(new StringValueNode(""));
+    }
+    parse_assert(node.children.size() == 1);
+    parse_assert(node.children[0].value.id().to_long() == grammar.id_string);
+    vespalib::string val(node.children[0].value.begin(),
+                    node.children[0].value.end());
+    return std::unique_ptr<ValueNode>(new StringValueNode(StringUtil::unescape(val)));
+}
+
+template<typename Tree>
+void printSpiritTree(std::ostream& out, Tree tree, const vespalib::string& query,
+                     const DocSelectionGrammar& grammar) {
+    using boost::spirit::classic::parser_id;
+
+    std::map<parser_id, vespalib::string> names;
+    names[parser_id(grammar.id_bool)] = "bool";
+    names[parser_id(grammar.id_number)] = "number";
+    names[parser_id(grammar.id_string)] = "string";
+    names[parser_id(grammar.id_doctype)] = "doctype";
+    names[parser_id(grammar.id_fieldname)] = "fieldname";
+    names[parser_id(grammar.id_function)] = "function";
+    names[parser_id(grammar.id_idarg)] = "idarg";
+    names[parser_id(grammar.id_searchcolumnarg)] = "searchcolumnarg";
+    names[parser_id(grammar.id_operator)] = "operator";
+    names[parser_id(grammar.id_idspec)] = "idspec";
+    names[parser_id(grammar.id_searchcolumnspec)] = "searchcolumnspec";
+    names[parser_id(grammar.id_fieldspec)] = "fieldspec";
+    names[parser_id(grammar.id_value)] = "value";
+    names[parser_id(grammar.id_valuefuncadd)] = "valuefuncadd";
+    names[parser_id(grammar.id_valuefuncmul)] = "valuefuncmul";
+    names[parser_id(grammar.id_valuefuncmod)] = "valuefuncmod";
+    names[parser_id(grammar.id_valuegroup)] = "valuegroup";
+    names[parser_id(grammar.id_arithmvalue)] = "arithmvalue";
+    names[parser_id(grammar.id_comparison)] = "comparison";
+    names[parser_id(grammar.id_leaf)] = "leaf";
+    names[parser_id(grammar.id_not)] = "not";
+    names[parser_id(grammar.id_and)] = "and";
+    names[parser_id(grammar.id_or)] = "or";
+    names[parser_id(grammar.id_group)] = "group";
+    names[parser_id(grammar.id_expression)] = "expression";
+    tree_to_xml(out, tree, query.c_str(), names);
+}
+
+template<typename Parser>
+bool testExpr(const DocumentTypeRepo& repo,
+              const BucketIdFactory& factory,
+              const vespalib::string& expression, const Parser& parser,
+              const vespalib::string& result)
+{
+    //std::cerr << "Testing expression '" << expression << "'.\n";
+    using boost::spirit::classic::space_p;
+
+    DocSelectionGrammar grammar(repo, factory);
+    boost::spirit::classic::tree_parse_info<> info;
+    info = pt_parse(expression.c_str(), parser,
+                    space_p);
+    std::ostringstream ost;
+    printSpiritTree(ost, info.trees, expression, grammar);
+    if (!info.full) {
+        cerr << "Expression '" << expression
+             << "' wasn't completely parsed\n"
+             << ost.str() << "\n";
+        return false;
+    }
+    vespalib::string httpexpr = expression;
+    vespalib::string::size_type index;
+    while ((index = httpexpr.find('>')) != vespalib::string::npos) {
+        httpexpr = httpexpr.substr(0,index) + "&gt;"
+                   + httpexpr.substr(index+1);
+    }
+    vespalib::string fullresult = "<?xml version=\"1.0\" encoding=\"ISO-8859-1\"?>\n"
+                        "<!DOCTYPE parsetree SYSTEM \"parsetree.dtd\">\n"
+                        "<!-- " + httpexpr + " -->\n" + result;
+    //if (ost.str() != fullresult) {
+    if (fullresult != ost.str()) {
+        cerr << "Parsing expression '" << expression << "', expected\n"
+             << fullresult << "\nbut got\n" << ost.str() << "\n";
+        return false;
+    }
+    return true;
+}
+
+bool test(const DocumentTypeRepo& repo,
+          const BucketIdFactory& bucketIdFactory)
+{
+    //std::cerr << "\n\nTESTING DOCUMENT SELECT PARSER\n\n";
+    DocSelectionGrammar grammar(repo, bucketIdFactory);
+
+    using boost::spirit::classic::space_p;
+
+    // Parser two is the arithmvalue..
+    // idspec, fieldspec, number & stringval, + - * / % ()
+    testExpr(repo, bucketIdFactory, "3.14", grammar.use_parser<2>(),
+             "<parsetree version=\"1.0\">\n"
+             "    <parsenode rule=\"arithmvalue\">\n"
+             "        <parsenode rule=\"value\">\n"
+             "            <parsenode rule=\"number\">\n"
+             "                <parsenode rule=\"number\">\n"
+             "                    <value>3.14</value>\n"
+             "                </parsenode>\n"
+             "            </parsenode>\n"
+             "        </parsenode>\n"
+             "    </parsenode>\n"
+             "</parsetree>\n");
+    testExpr(repo, bucketIdFactory, "-999", grammar.use_parser<2>(),
+             "<parsetree version=\"1.0\">\n"
+             "    <parsenode rule=\"arithmvalue\">\n"
+             "        <parsenode rule=\"value\">\n"
+             "            <parsenode rule=\"number\">\n"
+             "                <parsenode rule=\"number\">\n"
+             "                    <value>-999</value>\n"
+             "                </parsenode>\n"
+             "            </parsenode>\n"
+             "        </parsenode>\n"
+             "    </parsenode>\n"
+             "</parsetree>\n");
+    testExpr(repo, bucketIdFactory, "15e4", grammar.use_parser<2>(),
+             "<parsetree version=\"1.0\">\n"
+             "    <parsenode rule=\"arithmvalue\">\n"
+             "        <parsenode rule=\"value\">\n"
+             "            <parsenode rule=\"number\">\n"
+             "                <parsenode rule=\"number\">\n"
+             "                    <value>15e4</value>\n"
+             "                </parsenode>\n"
+             "            </parsenode>\n"
+             "        </parsenode>\n"
+             "    </parsenode>\n"
+             "</parsetree>\n");
+    testExpr(repo, bucketIdFactory, "3.4e-4", grammar.use_parser<2>(),
+             "<parsetree version=\"1.0\">\n"
+             "    <parsenode rule=\"arithmvalue\">\n"
+             "        <parsenode rule=\"value\">\n"
+             "            <parsenode rule=\"number\">\n"
+             "                <parsenode rule=\"number\">\n"
+             "                    <value>3.4e-4</value>\n"
+             "                </parsenode>\n"
+             "            </parsenode>\n"
+             "        </parsenode>\n"
+             "    </parsenode>\n"
+             "</parsetree>\n");
+    testExpr(repo, bucketIdFactory, "\" Test \"", grammar.use_parser<2>(),
+             "<parsetree version=\"1.0\">\n"
+             "    <parsenode rule=\"arithmvalue\">\n"
+             "        <parsenode rule=\"value\">\n"
+             "            <parsenode rule=\"string\">\n"
+             "                <parsenode rule=\"string\">\n"
+             "                    <value> Test </value>\n"
+             "                </parsenode>\n"
+             "            </parsenode>\n"
+             "        </parsenode>\n"
+             "    </parsenode>\n"
+             "</parsetree>\n");
+    testExpr(repo, bucketIdFactory, "id", grammar.use_parser<2>(),
+             "<parsetree version=\"1.0\">\n"
+             "    <parsenode rule=\"arithmvalue\">\n"
+             "        <parsenode rule=\"value\">\n"
+             "            <parsenode rule=\"idspec\">\n"
+             "                <parsenode rule=\"idspec\">\n"
+             "                    <value>id</value>\n"
+             "                </parsenode>\n"
+             "            </parsenode>\n"
+             "        </parsenode>\n"
+             "    </parsenode>\n"
+             "</parsetree>\n");
+    testExpr(repo, bucketIdFactory, "id.namespace",
+             grammar.use_parser<2>(),
+             "<parsetree version=\"1.0\">\n"
+             "    <parsenode rule=\"arithmvalue\">\n"
+             "        <parsenode rule=\"value\">\n"
+             "            <parsenode rule=\"idspec\">\n"
+             "                <parsenode rule=\"idspec\">\n"
+             "                    <value>id</value>\n"
+             "                </parsenode>\n"
+             "                <parsenode rule=\"idarg\">\n"
+             "                    <parsenode rule=\"idarg\">\n"
+             "                        <value>namespace</value>\n"
+             "                    </parsenode>\n"
+             "                </parsenode>\n"
+             "            </parsenode>\n"
+             "        </parsenode>\n"
+             "    </parsenode>\n"
+             "</parsetree>\n");
+    testExpr(repo, bucketIdFactory, "id.hash()", grammar.use_parser<2>(),
+             "<parsetree version=\"1.0\">\n"
+             "    <parsenode rule=\"arithmvalue\">\n"
+             "        <parsenode rule=\"value\">\n"
+             "            <parsenode rule=\"idspec\">\n"
+             "                <parsenode rule=\"idspec\">\n"
+             "                    <value>id</value>\n"
+             "                </parsenode>\n"
+             "            </parsenode>\n"
+             "            <parsenode rule=\"function\">\n"
+             "                <parsenode rule=\"function\">\n"
+             "                    <value>hash</value>\n"
+             "                </parsenode>\n"
+             "            </parsenode>\n"
+             "        </parsenode>\n"
+             "    </parsenode>\n"
+             "</parsetree>\n");
+    testExpr(repo, bucketIdFactory,
+             "id.namespace.hash()", grammar.use_parser<2>(),
+             "<parsetree version=\"1.0\">\n"
+             "    <parsenode rule=\"arithmvalue\">\n"
+             "        <parsenode rule=\"value\">\n"
+             "            <parsenode rule=\"idspec\">\n"
+             "                <parsenode rule=\"idspec\">\n"
+             "                    <value>id</value>\n"
+             "                </parsenode>\n"
+             "                <parsenode rule=\"idarg\">\n"
+             "                    <parsenode rule=\"idarg\">\n"
+             "                        <value>namespace</value>\n"
+             "                    </parsenode>\n"
+             "                </parsenode>\n"
+             "            </parsenode>\n"
+             "            <parsenode rule=\"function\">\n"
+             "                <parsenode rule=\"function\">\n"
+             "                    <value>hash</value>\n"
+             "                </parsenode>\n"
+             "            </parsenode>\n"
+             "        </parsenode>\n"
+             "    </parsenode>\n"
+             "</parsetree>\n");
+    testExpr(repo, bucketIdFactory,
+             "music.artist", grammar.use_parser<2>(),
+             "<parsetree version=\"1.0\">\n"
+             "    <parsenode rule=\"arithmvalue\">\n"
+             "        <parsenode rule=\"value\">\n"
+             "            <parsenode rule=\"fieldspec\">\n"
+             "                <parsenode rule=\"doctype\">\n"
+             "                    <parsenode rule=\"doctype\">\n"
+             "                        <value>music</value>\n"
+             "                    </parsenode>\n"
+             "                </parsenode>\n"
+             "                <parsenode rule=\"fieldname\">\n"
+             "                    <parsenode rule=\"fieldname\">\n"
+             "                        <value>artist</value>\n"
+             "                    </parsenode>\n"
+             "                </parsenode>\n"
+             "            </parsenode>\n"
+             "        </parsenode>\n"
+             "    </parsenode>\n"
+             "</parsetree>\n");
+    testExpr(repo, bucketIdFactory,
+             "music.artist.lowercase()", grammar.use_parser<2>(),
+             "<parsetree version=\"1.0\">\n"
+             "    <parsenode rule=\"arithmvalue\">\n"
+             "        <parsenode rule=\"value\">\n"
+             "            <parsenode rule=\"fieldspec\">\n"
+             "                <parsenode rule=\"doctype\">\n"
+             "                    <parsenode rule=\"doctype\">\n"
+             "                        <value>music</value>\n"
+             "                    </parsenode>\n"
+             "                </parsenode>\n"
+             "                <parsenode rule=\"fieldname\">\n"
+             "                    <parsenode rule=\"fieldname\">\n"
+             "                        <value>artist</value>\n"
+             "                    </parsenode>\n"
+             "                </parsenode>\n"
+             "                <parsenode rule=\"function\">\n"
+             "                    <parsenode rule=\"function\">\n"
+             "                        <value>lowercase</value>\n"
+             "                    </parsenode>\n"
+             "                </parsenode>\n"
+             "            </parsenode>\n"
+             "        </parsenode>\n"
+             "    </parsenode>\n"
+             "</parsetree>\n");
+    testExpr(repo, bucketIdFactory, "(43)", grammar.use_parser<2>(),
+             "<parsetree version=\"1.0\">\n"
+             "    <parsenode rule=\"arithmvalue\">\n"
+             "        <parsenode rule=\"valuegroup\">\n"
+             "            <parsenode rule=\"arithmvalue\">\n"
+             "                <parsenode rule=\"value\">\n"
+             "                    <parsenode rule=\"number\">\n"
+             "                        <parsenode rule=\"number\">\n"
+             "                            <value>43</value>\n"
+             "                        </parsenode>\n"
+             "                    </parsenode>\n"
+             "                </parsenode>\n"
+             "            </parsenode>\n"
+             "        </parsenode>\n"
+             "    </parsenode>\n"
+             "</parsetree>\n");
+    testExpr(repo, bucketIdFactory,
+             "1 + 2 * 3 - 10 % 2 / 3", grammar.use_parser<2>(),
+             "<parsetree version=\"1.0\">\n"
+             "    <parsenode rule=\"arithmvalue\">\n"
+             "        <parsenode rule=\"valuefuncadd\">\n"
+             "            <parsenode rule=\"value\">\n"
+             "                <parsenode rule=\"number\">\n"
+             "                    <parsenode rule=\"number\">\n"
+             "                        <value>1</value>\n"
+             "                    </parsenode>\n"
+             "                </parsenode>\n"
+             "            </parsenode>\n"
+             "            <parsenode rule=\"valuefuncadd\">\n"
+             "                <value>+</value>\n"
+             "            </parsenode>\n"
+             "            <parsenode rule=\"valuefuncmul\">\n"
+             "                <parsenode rule=\"value\">\n"
+             "                    <parsenode rule=\"number\">\n"
+             "                        <parsenode rule=\"number\">\n"
+             "                            <value>2</value>\n"
+             "                        </parsenode>\n"
+             "                    </parsenode>\n"
+             "                </parsenode>\n"
+             "                <parsenode rule=\"valuefuncmul\">\n"
+             "                    <value>*</value>\n"
+             "                </parsenode>\n"
+             "                <parsenode rule=\"value\">\n"
+             "                    <parsenode rule=\"number\">\n"
+             "                        <parsenode rule=\"number\">\n"
+             "                            <value>3</value>\n"
+             "                        </parsenode>\n"
+             "                    </parsenode>\n"
+             "                </parsenode>\n"
+             "            </parsenode>\n"
+             "            <parsenode rule=\"valuefuncadd\">\n"
+             "                <value>-</value>\n"
+             "            </parsenode>\n"
+             "            <parsenode rule=\"valuefuncmul\">\n"
+             "                <parsenode rule=\"valuefuncmod\">\n"
+             "                    <parsenode rule=\"value\">\n"
+             "                        <parsenode rule=\"number\">\n"
+             "                            <parsenode rule=\"number\">\n"
+             "                                <value>10</value>\n"
+             "                            </parsenode>\n"
+             "                        </parsenode>\n"
+             "                    </parsenode>\n"
+             "                    <parsenode rule=\"valuefuncmod\">\n"
+             "                        <value>%</value>\n"
+             "                    </parsenode>\n"
+             "                    <parsenode rule=\"value\">\n"
+             "                        <parsenode rule=\"number\">\n"
+             "                            <parsenode rule=\"number\">\n"
+             "                                <value>2</value>\n"
+             "                            </parsenode>\n"
+             "                        </parsenode>\n"
+             "                    </parsenode>\n"
+             "                </parsenode>\n"
+             "                <parsenode rule=\"valuefuncmul\">\n"
+             "                    <value>/</value>\n"
+             "                </parsenode>\n"
+             "                <parsenode rule=\"value\">\n"
+             "                    <parsenode rule=\"number\">\n"
+             "                        <parsenode rule=\"number\">\n"
+             "                            <value>3</value>\n"
+             "                        </parsenode>\n"
+             "                    </parsenode>\n"
+             "                </parsenode>\n"
+             "            </parsenode>\n"
+             "        </parsenode>\n"
+             "    </parsenode>\n"
+             "</parsetree>\n");
+    testExpr(repo, bucketIdFactory, "(43 + 14) / 34",
+             grammar.use_parser<2>(),
+             "<parsetree version=\"1.0\">\n"
+             "    <parsenode rule=\"arithmvalue\">\n"
+             "        <parsenode rule=\"valuefuncmul\">\n"
+             "            <parsenode rule=\"valuegroup\">\n"
+             "                <parsenode rule=\"arithmvalue\">\n"
+             "                    <parsenode rule=\"valuefuncadd\">\n"
+             "                        <parsenode rule=\"value\">\n"
+             "                            <parsenode rule=\"number\">\n"
+             "                                <parsenode rule=\"number\">\n"
+             "                                    <value>43</value>\n"
+             "                                </parsenode>\n"
+             "                            </parsenode>\n"
+             "                        </parsenode>\n"
+             "                        <parsenode rule=\"valuefuncadd\">\n"
+             "                            <value>+</value>\n"
+             "                        </parsenode>\n"
+             "                        <parsenode rule=\"value\">\n"
+             "                            <parsenode rule=\"number\">\n"
+             "                                <parsenode rule=\"number\">\n"
+             "                                    <value>14</value>\n"
+             "                                </parsenode>\n"
+             "                            </parsenode>\n"
+             "                        </parsenode>\n"
+             "                    </parsenode>\n"
+             "                </parsenode>\n"
+             "            </parsenode>\n"
+             "            <parsenode rule=\"valuefuncmul\">\n"
+             "                <value>/</value>\n"
+             "            </parsenode>\n"
+             "            <parsenode rule=\"value\">\n"
+             "                <parsenode rule=\"number\">\n"
+             "                    <parsenode rule=\"number\">\n"
+             "                        <value>34</value>\n"
+             "                    </parsenode>\n"
+             "                </parsenode>\n"
+             "            </parsenode>\n"
+             "        </parsenode>\n"
+             "    </parsenode>\n"
+             "</parsetree>\n");
+    testExpr(repo, bucketIdFactory, "34 * (3 - 1) % 4",
+             grammar.use_parser<2>(),
+             "<parsetree version=\"1.0\">\n"
+             "    <parsenode rule=\"arithmvalue\">\n"
+             "        <parsenode rule=\"valuefuncmul\">\n"
+             "            <parsenode rule=\"value\">\n"
+             "                <parsenode rule=\"number\">\n"
+             "                    <parsenode rule=\"number\">\n"
+             "                        <value>34</value>\n"
+             "                    </parsenode>\n"
+             "                </parsenode>\n"
+             "            </parsenode>\n"
+             "            <parsenode rule=\"valuefuncmul\">\n"
+             "                <value>*</value>\n"
+             "            </parsenode>\n"
+             "            <parsenode rule=\"valuefuncmod\">\n"
+             "                <parsenode rule=\"valuegroup\">\n"
+             "                    <parsenode rule=\"arithmvalue\">\n"
+             "                        <parsenode rule=\"valuefuncadd\">\n"
+             "                            <parsenode rule=\"value\">\n"
+             "                                <parsenode rule=\"number\">\n"
+             "                                    <parsenode rule=\"number\">\n"
+             "                                        <value>3</value>\n"
+             "                                    </parsenode>\n"
+             "                                </parsenode>\n"
+             "                            </parsenode>\n"
+             "                            <parsenode rule=\"valuefuncadd\">\n"
+             "                                <value>-</value>\n"
+             "                            </parsenode>\n"
+             "                            <parsenode rule=\"value\">\n"
+             "                                <parsenode rule=\"number\">\n"
+             "                                    <parsenode rule=\"number\">\n"
+             "                                        <value>1</value>\n"
+             "                                    </parsenode>\n"
+             "                                </parsenode>\n"
+             "                            </parsenode>\n"
+             "                        </parsenode>\n"
+             "                    </parsenode>\n"
+             "                </parsenode>\n"
+             "                <parsenode rule=\"valuefuncmod\">\n"
+             "                    <value>%</value>\n"
+             "                </parsenode>\n"
+             "                <parsenode rule=\"value\">\n"
+             "                    <parsenode rule=\"number\">\n"
+             "                        <parsenode rule=\"number\">\n"
+             "                            <value>4</value>\n"
+             "                        </parsenode>\n"
+             "                    </parsenode>\n"
+             "                </parsenode>\n"
+             "            </parsenode>\n"
+             "        </parsenode>\n"
+             "    </parsenode>\n"
+             "</parsetree>\n");
+
+    // Parser 1 is a leaf. bool, comparison, fieldspec, doctype
+    testExpr(repo, bucketIdFactory, "true", grammar.use_parser<1>(),
+             "<parsetree version=\"1.0\">\n"
+             "    <parsenode rule=\"leaf\">\n"
+             "        <parsenode rule=\"bool\">\n"
+             "            <parsenode rule=\"bool\">\n"
+             "                <value>true</value>\n"
+             "            </parsenode>\n"
+             "        </parsenode>\n"
+             "    </parsenode>\n"
+             "</parsetree>\n");
+    testExpr(repo, bucketIdFactory, "false", grammar.use_parser<1>(),
+             "<parsetree version=\"1.0\">\n"
+             "    <parsenode rule=\"leaf\">\n"
+             "        <parsenode rule=\"bool\">\n"
+             "            <parsenode rule=\"bool\">\n"
+             "                <value>false</value>\n"
+             "            </parsenode>\n"
+             "        </parsenode>\n"
+             "    </parsenode>\n"
+             "</parsetree>\n");
+    testExpr(repo, bucketIdFactory, "music.test", grammar.use_parser<1>(),
+             "<parsetree version=\"1.0\">\n"
+             "    <parsenode rule=\"leaf\">\n"
+             "        <parsenode rule=\"fieldspec\">\n"
+             "            <parsenode rule=\"doctype\">\n"
+             "                <parsenode rule=\"doctype\">\n"
+             "                    <value>music</value>\n"
+             "                </parsenode>\n"
+             "            </parsenode>\n"
+             "            <parsenode rule=\"fieldname\">\n"
+             "                <parsenode rule=\"fieldname\">\n"
+             "                    <value>test</value>\n"
+             "                </parsenode>\n"
+             "            </parsenode>\n"
+             "        </parsenode>\n"
+             "    </parsenode>\n"
+             "</parsetree>\n");
+    testExpr(repo, bucketIdFactory, "music", grammar.use_parser<1>(),
+             "<parsetree version=\"1.0\">\n"
+             "    <parsenode rule=\"leaf\">\n"
+             "        <parsenode rule=\"doctype\">\n"
+             "            <parsenode rule=\"doctype\">\n"
+             "                <value>music</value>\n"
+             "            </parsenode>\n"
+             "        </parsenode>\n"
+             "    </parsenode>\n"
+             "</parsetree>\n");
+    testExpr(repo, bucketIdFactory,
+             "music.artist = \"*john*\"", grammar.use_parser<1>(),
+             "<parsetree version=\"1.0\">\n"
+             "    <parsenode rule=\"leaf\">\n"
+             "        <parsenode rule=\"comparison\">\n"
+             "            <parsenode rule=\"arithmvalue\">\n"
+             "                <parsenode rule=\"value\">\n"
+             "                    <parsenode rule=\"fieldspec\">\n"
+             "                        <parsenode rule=\"doctype\">\n"
+             "                            <parsenode rule=\"doctype\">\n"
+             "                                <value>music</value>\n"
+             "                            </parsenode>\n"
+             "                        </parsenode>\n"
+             "                        <parsenode rule=\"fieldname\">\n"
+             "                            <parsenode rule=\"fieldname\">\n"
+             "                                <value>artist</value>\n"
+             "                            </parsenode>\n"
+             "                        </parsenode>\n"
+             "                    </parsenode>\n"
+             "                </parsenode>\n"
+             "            </parsenode>\n"
+             "            <parsenode rule=\"operator\">\n"
+             "                <parsenode rule=\"operator\">\n"
+             "                    <value>=</value>\n"
+             "                </parsenode>\n"
+             "            </parsenode>\n"
+             "            <parsenode rule=\"arithmvalue\">\n"
+             "                <parsenode rule=\"value\">\n"
+             "                    <parsenode rule=\"string\">\n"
+             "                        <parsenode rule=\"string\">\n"
+             "                            <value>*john*</value>\n"
+             "                        </parsenode>\n"
+             "                    </parsenode>\n"
+             "                </parsenode>\n"
+             "            </parsenode>\n"
+             "        </parsenode>\n"
+             "    </parsenode>\n"
+             "</parsetree>\n");
+    testExpr(repo, bucketIdFactory,
+             "music.length >= 180", grammar.use_parser<1>(),
+             "<parsetree version=\"1.0\">\n"
+             "    <parsenode rule=\"leaf\">\n"
+             "        <parsenode rule=\"comparison\">\n"
+             "            <parsenode rule=\"arithmvalue\">\n"
+             "                <parsenode rule=\"value\">\n"
+             "                    <parsenode rule=\"fieldspec\">\n"
+             "                        <parsenode rule=\"doctype\">\n"
+             "                            <parsenode rule=\"doctype\">\n"
+             "                                <value>music</value>\n"
+             "                            </parsenode>\n"
+             "                        </parsenode>\n"
+             "                        <parsenode rule=\"fieldname\">\n"
+             "                            <parsenode rule=\"fieldname\">\n"
+             "                                <value>length</value>\n"
+             "                            </parsenode>\n"
+             "                        </parsenode>\n"
+             "                    </parsenode>\n"
+             "                </parsenode>\n"
+             "            </parsenode>\n"
+             "            <parsenode rule=\"operator\">\n"
+             "                <parsenode rule=\"operator\">\n"
+             "                    <value>&gt;=</value>\n"
+             "                </parsenode>\n"
+             "            </parsenode>\n"
+             "            <parsenode rule=\"arithmvalue\">\n"
+             "                <parsenode rule=\"value\">\n"
+             "                    <parsenode rule=\"number\">\n"
+             "                        <parsenode rule=\"number\">\n"
+             "                            <value>180</value>\n"
+             "                        </parsenode>\n"
+             "                    </parsenode>\n"
+             "                </parsenode>\n"
+             "            </parsenode>\n"
+             "        </parsenode>\n"
+             "    </parsenode>\n"
+             "</parsetree>\n");
+
+    // Parser 0 - The whole expression
+    testExpr(repo, bucketIdFactory,
+             "true oR nOt false And true", grammar.use_parser<0>(),
+             "<parsetree version=\"1.0\">\n"
+             "    <parsenode rule=\"expression\">\n"
+             "        <parsenode rule=\"or\">\n"
+             "            <parsenode rule=\"leaf\">\n"
+             "                <parsenode rule=\"bool\">\n"
+             "                    <parsenode rule=\"bool\">\n"
+             "                        <value>true</value>\n"
+             "                    </parsenode>\n"
+             "                </parsenode>\n"
+             "            </parsenode>\n"
+             "            <parsenode rule=\"or\">\n"
+             "                <value>oR</value>\n"
+             "            </parsenode>\n"
+             "            <parsenode rule=\"and\">\n"
+             "                <parsenode rule=\"not\">\n"
+             "                    <parsenode rule=\"not\">\n"
+             "                        <value>nOt</value>\n"
+             "                    </parsenode>\n"
+             "                    <parsenode rule=\"leaf\">\n"
+             "                        <parsenode rule=\"bool\">\n"
+             "                            <parsenode rule=\"bool\">\n"
+             "                                <value>false</value>\n"
+             "                            </parsenode>\n"
+             "                        </parsenode>\n"
+             "                    </parsenode>\n"
+             "                </parsenode>\n"
+             "                <parsenode rule=\"and\">\n"
+             "                    <value>And</value>\n"
+             "                </parsenode>\n"
+             "                <parsenode rule=\"leaf\">\n"
+             "                    <parsenode rule=\"bool\">\n"
+             "                        <parsenode rule=\"bool\">\n"
+             "                            <value>true</value>\n"
+             "                        </parsenode>\n"
+             "                    </parsenode>\n"
+             "                </parsenode>\n"
+             "            </parsenode>\n"
+             "        </parsenode>\n"
+             "    </parsenode>\n"
+             "</parsetree>\n");
+    testExpr(repo, bucketIdFactory,
+             "(true oR false) aNd true", grammar.use_parser<0>(),
+             "<parsetree version=\"1.0\">\n"
+             "    <parsenode rule=\"expression\">\n"
+             "        <parsenode rule=\"and\">\n"
+             "            <parsenode rule=\"group\">\n"
+             "                <parsenode rule=\"or\">\n"
+             "                    <parsenode rule=\"leaf\">\n"
+             "                        <parsenode rule=\"bool\">\n"
+             "                            <parsenode rule=\"bool\">\n"
+             "                                <value>true</value>\n"
+             "                            </parsenode>\n"
+             "                        </parsenode>\n"
+             "                    </parsenode>\n"
+             "                    <parsenode rule=\"or\">\n"
+             "                        <value>oR</value>\n"
+             "                    </parsenode>\n"
+             "                    <parsenode rule=\"leaf\">\n"
+             "                        <parsenode rule=\"bool\">\n"
+             "                            <parsenode rule=\"bool\">\n"
+             "                                <value>false</value>\n"
+             "                            </parsenode>\n"
+             "                        </parsenode>\n"
+             "                    </parsenode>\n"
+             "                </parsenode>\n"
+             "            </parsenode>\n"
+             "            <parsenode rule=\"and\">\n"
+             "                <value>aNd</value>\n"
+             "            </parsenode>\n"
+             "            <parsenode rule=\"leaf\">\n"
+             "                <parsenode rule=\"bool\">\n"
+             "                    <parsenode rule=\"bool\">\n"
+             "                        <value>true</value>\n"
+             "                    </parsenode>\n"
+             "                </parsenode>\n"
+             "            </parsenode>\n"
+             "        </parsenode>\n"
+             "    </parsenode>\n"
+             "</parsetree>\n");
+    testExpr(repo, bucketIdFactory,
+             "iddoc or not(notand and ornot)", grammar.use_parser<0>(),
+             "<parsetree version=\"1.0\">\n"
+             "    <parsenode rule=\"expression\">\n"
+             "        <parsenode rule=\"or\">\n"
+             "            <parsenode rule=\"leaf\">\n"
+             "                <parsenode rule=\"doctype\">\n"
+             "                    <parsenode rule=\"doctype\">\n"
+             "                        <value>iddoc</value>\n"
+             "                    </parsenode>\n"
+             "                </parsenode>\n"
+             "            </parsenode>\n"
+             "            <parsenode rule=\"or\">\n"
+             "                <value>or</value>\n"
+             "            </parsenode>\n"
+             "            <parsenode rule=\"not\">\n"
+             "                <parsenode rule=\"not\">\n"
+             "                    <value>not</value>\n"
+             "                </parsenode>\n"
+             "                <parsenode rule=\"group\">\n"
+             "                    <parsenode rule=\"and\">\n"
+             "                        <parsenode rule=\"leaf\">\n"
+             "                            <parsenode rule=\"doctype\">\n"
+             "                                <parsenode rule=\"doctype\">\n"
+             "                                    <value>notand</value>\n"
+             "                                </parsenode>\n"
+             "                            </parsenode>\n"
+             "                        </parsenode>\n"
+             "                        <parsenode rule=\"and\">\n"
+             "                            <value>and</value>\n"
+             "                        </parsenode>\n"
+             "                        <parsenode rule=\"leaf\">\n"
+             "                            <parsenode rule=\"doctype\">\n"
+             "                                <parsenode rule=\"doctype\">\n"
+             "                                    <value>ornot</value>\n"
+             "                                </parsenode>\n"
+             "                            </parsenode>\n"
+             "                        </parsenode>\n"
+             "                    </parsenode>\n"
+             "                </parsenode>\n"
+             "            </parsenode>\n"
+             "        </parsenode>\n"
+             "    </parsenode>\n"
+             "</parsetree>\n");
+    return true;
+}
+
+}
+
+vespalib::Lock Parser::_G_parseLock;
+
+unique_ptr<Node> Parser::parse(const vespalib::stringref & s)
+{
+
+    simple::SelectionParser simple(_bucketIdFactory);
+    if (simple.parse(s) && simple.getRemaining().empty()) {
+        Node::UP tmp(simple.getNode());
+        assert(tmp.get() != NULL);
+        return tmp;
+    } else {
+        return fullParse(s);
+    }
+}
+
+unique_ptr<Node> Parser::fullParse(const vespalib::stringref & s)
+{
+    static bool haveTested = test(_repo, _bucketIdFactory); if (haveTested) {}
+    try{
+        vespalib::LockGuard guard(_G_parseLock);
+        DocSelectionGrammar grammar(_repo, _bucketIdFactory);
+        boost::spirit::classic::tree_parse_info<> info
+            = pt_parse(&s[0], &s[0]+s.size(),
+                       grammar.use_parser<0>(), boost::spirit::classic::space_p);
+        if (!info.full) {
+            vespalib::string unexpected(info.stop);
+            unsigned int position = s.size() - unexpected.size();
+            if (unexpected.size() > 10) {
+                unexpected = unexpected.substr(0,10);
+            }
+            vespalib::asciistream ost;
+            ost << "Unexpected token at position " << position << " ('"
+                << unexpected << "') in query '" << s << "',";
+            throw ParsingFailedException(ost.str(), VESPA_STRLOC);
+        }
+        parse_assert(info.trees.size() == 1);
+        //printSpiritTree(std::cerr, info.trees, s, grammar);
+        return parseTree(grammar, info.trees[0]);
+    } catch (ParsingFailedException& e) {
+        throw;
+    } catch (vespalib::Exception& e) {
+        throw ParsingFailedException("Parsing failed. See cause exception.",
+                                     e, VESPA_STRLOC);
+    } catch (std::exception& e) {
+        cerr << "Parser::parse() internal error: "
+             << e.what() << endl;
+        throw; // Program will abort when this tries to go out..
+    }
+    return unique_ptr<Node>();
+}
+
+} // select
+} // document
diff --git a/document/src/vespa/document/select/parser.h b/document/src/vespa/document/select/parser.h
new file mode 100644
index 00000000000..61cc6e05876
--- /dev/null
+++ b/document/src/vespa/document/select/parser.h
@@ -0,0 +1,39 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <memory>
+#include <string>
+#include <vespa/document/bucket/bucketidfactory.h>
+#include "node.h"
+#include <vespa/vespalib/util/exception.h>
+#include <vespa/vespalib/util/sync.h>
+
+namespace document {
+class DocumentTypeRepo;
+
+namespace select {
+
+VESPA_DEFINE_EXCEPTION(ParsingFailedException, vespalib::Exception);
+
+class Parser {
+public:
+    Parser(const DocumentTypeRepo&, const BucketIdFactory& bucketIdFactory);
+
+    /**
+     * Returns a newly allocated AST root node representing the selection
+     * if parsing is successful. Otherwise, ParsingFailedException will be
+     * thrown.
+     */
+    std::unique_ptr<Node> parse(const vespalib::stringref& s);
+
+private:
+    std::unique_ptr<Node> fullParse(const vespalib::stringref& s);
+    static vespalib::Lock _G_parseLock;
+    const DocumentTypeRepo& _repo;
+    const BucketIdFactory& _bucketIdFactory;
+};
+
+} // select
+} // parser
+
diff --git a/document/src/vespa/document/select/result.cpp b/document/src/vespa/document/select/result.cpp
new file mode 100644
index 00000000000..da97e925272
--- /dev/null
+++ b/document/src/vespa/document/select/result.cpp
@@ -0,0 +1,53 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include "result.h"
+
+namespace document {
+namespace select {
+
+Result Result::Invalid;
+Result Result::False;
+Result Result::True;
+
+Result::Result()
+{
+}
+
+const Result&
+Result::operator!() const {
+    if (this == &Invalid) {
+        return Invalid;
+    }
+    return (this == &True ? False : True);
+}
+
+const Result&
+Result::operator&&(const Result& r) const {
+    if (this == &False || &r == &False) {
+        return False;
+    }
+    return (this == &True && &r == &True ? True : Invalid);
+}
+
+const Result&
+Result::operator||(const Result& r) const {
+    if (this == &True || &r == &True) {
+        return True;
+    }
+    if (this == &Invalid || &r == &Invalid) {
+        return Invalid;
+    }
+    return False;
+}
+
+void
+Result::print(std::ostream& out, bool,
+              const std::string&) const
+{
+    if (this == &Invalid) out << "Invalid";
+    else if (this == &True) out << "True";
+    else out << "False";
+}
+
+} // select
+} // document
diff --git a/document/src/vespa/document/select/result.h b/document/src/vespa/document/select/result.h
new file mode 100644
index 00000000000..98e6195e11f
--- /dev/null
+++ b/document/src/vespa/document/select/result.h
@@ -0,0 +1,86 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+/**
+ * @class document::select::Result
+ * @ingroup select
+ *
+ * @brief Represents a result of matching a document. Can be invalid.
+ *
+ * Using a bool to represent match or not proved inferior.
+ * 'music.artist < 10' should not match any documents as long as music.artist
+ * is a string field. However, we don't want 'not music.artist < 10' or
+ * 'music.artist > 10' to match all documents because of that. This type is
+ * thus used as it has 3 outcomes.. True, false & invalid.
+ *
+ * @author H�kon Humberset
+ * @date 2007-20-05
+ * @version $Id$
+ */
+
+#pragma once
+
+#include <string>
+#include <vespa/document/util/printable.h>
+
+namespace document {
+namespace select {
+
+class Result : public Printable {
+public:
+    static Result Invalid;
+    static Result False;
+    static Result True;
+
+    void print(std::ostream& out, bool verbose,
+               const std::string& indent) const;
+
+    bool operator==(const Result& o) const { return (&o == this); }
+    bool operator!=(const Result& o) const { return (&o != this); }
+
+    const Result& operator&&(const Result&) const;
+    const Result& operator||(const Result&) const;
+    const Result& operator!() const;
+
+    static const Result& get(bool b) { return (b ? True : False); }
+
+    static uint32_t
+    enumRange(void)
+    {
+        return 3u;
+    }
+
+    uint32_t
+    toEnum(void) const
+    {
+        if (this == &Result::Invalid)
+            return 0u;
+        if (this == &Result::False)
+            return 1u;
+        if (this == &Result::True)
+            return 2u;
+        abort();
+    }
+
+    static const Result &
+    fromEnum(uint32_t val)
+    {
+        if (val == 0u)
+            return Result::Invalid;
+        if (val == 1u)
+            return Result::False;
+        if (val == 2u)
+            return Result::True;
+        abort();
+    }
+
+private:
+    Result();
+
+        // Singletons are not copyable
+    Result(const Result&);
+    Result& operator=(const Result&);
+};
+
+} // select
+} // document
+
diff --git a/document/src/vespa/document/select/resultlist.cpp b/document/src/vespa/document/select/resultlist.cpp
new file mode 100644
index 00000000000..4f630f54863
--- /dev/null
+++ b/document/src/vespa/document/select/resultlist.cpp
@@ -0,0 +1,156 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include "resultlist.h"
+
+namespace document {
+namespace select {
+
+ResultList::ResultList(const Result& result) {
+    add(VariableMap(), result);
+}
+
+void
+ResultList::add(const FieldValue::IteratorHandler::VariableMap& variables,
+                const Result& result)
+{
+    _results.push_back(ResultPair(variables, &result));
+}
+
+void
+ResultList::print(std::ostream& out, bool, const std::string&) const
+{
+    out << "ResultList(";
+    for (uint32_t i = 0; i < _results.size(); i++) {
+        if (!_results[i].first.empty()) {
+            out << FieldValue::IteratorHandler::toString(_results[i].first)
+                << " => ";
+        }
+        out << _results[i].second->toString() << " ";
+    }
+    out << ")";
+}
+
+namespace {
+
+const Result &
+combineResultsLocal(const ResultList::Results & results)
+{
+    if (results.empty()) {
+        return Result::False;
+    }
+
+    bool foundFalse = false;
+
+    for (const auto & it : results) {
+        if (*it.second == Result::True) {
+            return Result::True;
+        } else if (*it.second == Result::False) {
+            foundFalse = true;
+        }
+    }
+
+    return (foundFalse) ? Result::False : Result::Invalid;
+}
+
+}
+
+const Result&
+ResultList::combineResults() const {
+    return combineResultsLocal(_results);
+}
+
+bool
+ResultList::combineVariables(
+        FieldValue::IteratorHandler::VariableMap& output,
+        const FieldValue::IteratorHandler::VariableMap& input) const
+{
+    // First, verify that all variables are overlapping
+    for (FieldValue::IteratorHandler::VariableMap::const_iterator iter
+            = output.begin(); iter != output.end(); iter++)
+    {
+        FieldValue::IteratorHandler::VariableMap::const_iterator found(
+                input.find(iter->first));
+
+        if (found != input.end()) {
+            if (!(found->second == iter->second)) {
+                return false;
+            }
+        }
+    }
+
+    for (FieldValue::IteratorHandler::VariableMap::const_iterator iter
+            = input.begin(); iter != input.end(); iter++)
+    {
+        FieldValue::IteratorHandler::VariableMap::const_iterator found(
+                output.find(iter->first));
+        if (found != output.end()) {
+            if (!(found->second == iter->second)) {
+                return false;
+            }
+        }
+    }
+    // Ok, variables are overlapping. Add all variables from input to output.
+    for (FieldValue::IteratorHandler::VariableMap::const_iterator iter
+            = input.begin(); iter != input.end(); iter++)
+    {
+        output[iter->first] = iter->second;
+    }
+
+    return true;
+}
+
+ResultList
+ResultList::operator&&(const ResultList& other) const
+{
+    ResultList result;
+
+    // TODO: optimize
+    for (const auto & it : _results) {
+        for (const auto & it2 : other._results) {
+            FieldValue::IteratorHandler::VariableMap vars = it.first;
+
+            if (combineVariables(vars, it2.first)) {
+                result.add(vars, *it.second && *it2.second);
+            }
+        }
+    }
+
+    return result;
+}
+
+ResultList
+ResultList::operator||(const ResultList& other) const
+{
+    ResultList result;
+
+    // TODO: optimize
+    for (const auto & it : _results) {
+        for (const auto & it2 : other._results) {
+            FieldValue::IteratorHandler::VariableMap vars = it.first;
+
+            if (combineVariables(vars, it2.first)) {
+                result.add(vars, *it.second || *it2.second);
+            }
+        }
+    }
+
+    return result;
+}
+
+ResultList
+ResultList::operator!() const {
+    ResultList result;
+
+    for (const auto & it : _results) {
+        result.add(it.first, !*it.second);
+    }
+
+    return result;
+}
+
+bool ResultList::operator==(const ResultList& other) const {
+    return (combineResultsLocal(_results) == combineResultsLocal(other._results));
+}
+
+} // select
+} // document
diff --git a/document/src/vespa/document/select/resultlist.h b/document/src/vespa/document/select/resultlist.h
new file mode 100644
index 00000000000..cd10f1ec2d8
--- /dev/null
+++ b/document/src/vespa/document/select/resultlist.h
@@ -0,0 +1,62 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vector>
+#include <vespa/document/fieldvalue/fieldvalue.h>
+#include "result.h"
+
+namespace document {
+namespace select {
+
+class ResultList : public Printable {
+public:
+    typedef FieldValue::IteratorHandler::VariableMap VariableMap;
+    typedef std::pair<VariableMap, const Result*> ResultPair;
+    typedef std::vector<ResultPair> Results;
+    typedef Results::iterator iterator;
+    typedef Results::const_iterator const_iterator;
+
+    ResultList() {};
+
+    /**
+       Creates a result list with one element with the given result type and no parameters.
+    */
+    explicit ResultList(const Result& result);
+
+    void add(const VariableMap& variables, const Result& result);
+
+    ResultList operator&&(const ResultList& other) const;
+    ResultList operator||(const ResultList& other) const;
+    ResultList operator!() const;
+
+    void print(std::ostream& out, bool verbose, const std::string& indent) const;
+
+    bool isEmpty() const { return _results.empty(); }
+
+    const Result& combineResults() const;
+
+    bool operator==(const ResultList& other) const;
+
+    const Results& getResults() const { return _results; }
+    Results& getResults() { return _results; }
+    const_iterator begin() const { return _results.begin(); }
+    const_iterator end() const { return _results.end(); }
+    iterator begin() { return _results.begin(); }
+    iterator end() { return _results.end(); }
+
+private:
+    Results _results;
+    bool combineVariables(VariableMap& output, const FieldValue::IteratorHandler::VariableMap& input) const;
+};
+
+inline bool operator==(const ResultList& list, const Result& other) {
+    return (list.combineResults() == other);
+}
+
+inline bool operator!=(const ResultList& list, const Result& other) {
+    return (list.combineResults() != other);
+}
+
+}
+}
+
diff --git a/document/src/vespa/document/select/resultset.cpp b/document/src/vespa/document/select/resultset.cpp
new file mode 100644
index 00000000000..7c8f3b91104
--- /dev/null
+++ b/document/src/vespa/document/select/resultset.cpp
@@ -0,0 +1,76 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include "resultset.h"
+
+namespace document
+{
+
+namespace select
+{
+
+std::vector<ResultSet> ResultSet::_ands;
+std::vector<ResultSet> ResultSet::_ors;
+std::vector<ResultSet> ResultSet::_nots;
+
+/*
+ * Precalculate possible outcomes of boolean operations, given possible
+ * inputs.
+ */
+void
+ResultSet::preCalc(void)
+{
+    uint32_t erange = Result::enumRange();
+    uint32_t range = illegalMask();
+    _ands.resize(range * range);
+    _ors.resize(range * range);
+    _nots.resize(range);
+    for (ResultSet lset; lset.pcvalid(); lset.pcnext()) {
+        for (ResultSet rset; rset.pcvalid(); rset.pcnext()) {
+            ResultSet myand;
+            ResultSet myor;
+            for (uint32_t lenum = 0; lenum < erange; ++lenum) {
+                if (!lset.hasEnum(lenum))
+                    continue;
+                const Result &lhs(Result::fromEnum(lenum));
+                for (uint32_t renum = 0; renum < erange; ++renum) {
+                    if (!rset.hasEnum(renum))
+                        continue;
+                    const Result &rhs(Result::fromEnum(renum));
+                    myand.add(lhs && rhs);
+                    myor.add(lhs || rhs);
+                }
+            }
+            _ands[(lset._val << erange) + rset._val] = myand;
+            _ors[(lset._val << erange) + rset._val] = myor;
+        }
+        ResultSet mynot;
+        for (uint32_t lenum = 0; lenum < erange; ++lenum) {
+            if (!lset.hasEnum(lenum))
+                continue;
+            const Result &lhs(Result::fromEnum(lenum));
+            mynot.add(!lhs);
+        }
+        _nots[lset._val] = mynot;
+    }
+}
+
+
+namespace
+{
+
+class Precalc
+{
+public:
+    Precalc(void)
+    {
+        ResultSet::preCalc();
+    }
+};
+
+
+Precalc precalc;
+
+}
+
+} // select
+} // document
diff --git a/document/src/vespa/document/select/resultset.h b/document/src/vespa/document/select/resultset.h
new file mode 100644
index 00000000000..91f3e1c34a9
--- /dev/null
+++ b/document/src/vespa/document/select/resultset.h
@@ -0,0 +1,125 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <string>
+#include "result.h"
+
+namespace document
+{
+namespace select
+{
+
+/*
+ * Contains possible values of operations returning a Result.
+ */
+
+class ResultSet
+{
+    // bitvector of the enum values for possible results
+    uint8_t _val; // range [0..7], cf. Result::toEnum()
+
+    // Precalculated results for boolean operations on Result.
+    static std::vector<ResultSet> _ands;
+    static std::vector<ResultSet> _ors;
+    static std::vector<ResultSet> _nots;
+public:
+    ResultSet(void)
+        : _val(0u)
+    {
+    }
+
+    static uint32_t
+    enumToMask(uint32_t rhs)
+    {
+        return 1u << rhs;
+    }
+
+    static uint32_t
+    illegalMask(void)
+    {
+        return (1u << Result::enumRange());
+    }
+
+    void
+    add(const Result &rhs)
+    {
+        _val |= enumToMask(rhs.toEnum());
+    }
+
+    bool
+    hasEnum(uint32_t rhs) const
+    {
+        return (_val & enumToMask(rhs)) != 0u;
+    }
+
+    bool
+    hasResult(const Result &rhs) const
+    {
+        return hasEnum(rhs.toEnum());
+    }
+
+    bool
+    operator==(const ResultSet &rhs) const
+    {
+        return _val == rhs._val;
+    }
+
+    bool
+    operator!=(const ResultSet &rhs) const
+    {
+        return _val != rhs._val;
+    }
+
+    // calculcate set of results emitted by document selection and operator.
+    ResultSet
+    calcAnd(const ResultSet &rhs) const
+    {
+        return _ands[(_val << Result::enumRange()) | rhs._val];
+    }
+
+    // calculcate set of results emitted by document selection or operator.
+    ResultSet
+    calcOr(const ResultSet &rhs) const
+    {
+        return _ors[(_val << Result::enumRange()) | rhs._val];
+    }
+
+    // calculcate set of results emitted by document selection not operator.
+    ResultSet
+    calcNot(void) const
+    {
+        return _nots[_val];
+    }
+
+    void
+    clear(void)
+    {
+        _val = 0;
+    }
+
+    void
+    fill(void)
+    {
+        _val = illegalMask() - 1;
+    }
+
+    static void
+    preCalc(void);
+private:
+    // precalc helper methods
+    void
+    pcnext(void)
+    {
+        ++_val;
+    }
+
+    bool
+    pcvalid(void) const
+    {
+        return _val < illegalMask();
+    }
+};
+
+} // select
+} // document
+
diff --git a/document/src/vespa/document/select/simpleparser.cpp b/document/src/vespa/document/select/simpleparser.cpp
new file mode 100644
index 00000000000..e4c6748ca25
--- /dev/null
+++ b/document/src/vespa/document/select/simpleparser.cpp
@@ -0,0 +1,203 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include "simpleparser.h"
+#include "compare.h"
+
+namespace document {
+
+namespace select {
+
+namespace simple {
+
+size_t eatWhite(const char * s, size_t len)
+{
+    size_t pos(0);
+    for (;(pos < len) && isspace(s[pos]); pos++);
+    return pos;
+}
+
+bool icmp(char c, char l)
+{
+    return tolower(c) == l;
+}
+
+bool IdSpecParser::parse(const vespalib::stringref & s)
+{
+    bool retval(false);
+    size_t pos(eatWhite(s.c_str(), s.size()));
+    if (pos+1 < s.size()) {
+        if (icmp(s[pos], 'i') && icmp(s[pos+1],'d')) {
+            pos += 2;
+            if (pos < s.size()) {
+                switch (s[pos]) {
+                case '.':
+                    {
+                        int widthBits(-1);
+                        int divisionBits(-1);
+                        size_t startPos(++pos);
+                        for (;(pos < s.size()) && (tolower(s[pos]) >= 'a') && (tolower(s[pos]) <= 'z'); pos++);
+                        size_t len(pos - startPos);
+                        if (((len == 4) && (strncasecmp(&s[startPos], "user", 4) == 0 ||
+                                            strncasecmp(&s[startPos], "type", 4) == 0)) ||
+                            ((len == 5) && (strncasecmp(&s[startPos], "group", 5) == 0)) ||
+                            ((len == 6) && (strncasecmp(&s[startPos], "scheme", 6) == 0)) ||
+                            ((len == 8) && (strncasecmp(&s[startPos], "specific", 8) == 0)) ||
+                            ((len == 9) && (strncasecmp(&s[startPos], "namespace", 9) == 0)))
+                        {
+                            retval = true;
+                            setValue(ValueNode::UP(new IdValueNode(_bucketIdFactory, "id", s.substr(startPos, len), widthBits, divisionBits)));
+                        } else {
+                            pos = startPos;
+                        }
+                    }
+                    break;
+                case '!':
+                case '<':
+                case '>':
+                case '=':
+                case '\t':
+                case '\n':
+                case '\r':
+                case ' ':
+                    {
+                        retval = true;
+                        setValue(ValueNode::UP(new IdValueNode(_bucketIdFactory, "id", "")));
+                    }
+                    break;
+                default:
+                    break;
+                }
+            }
+        }
+    }
+    setRemaining(s.substr(pos));
+    return retval;
+}
+
+bool OperatorParser::parse(const vespalib::stringref & s)
+{
+    bool retval(false);
+    size_t pos(eatWhite(s.c_str(), s.size()));
+    if (pos+1 < s.size()) {
+        retval = true;
+        if (s[pos] == '=') {
+            pos++;
+            if (s[pos] == '=') {
+                pos++;
+                _operator = & Operator::get("==");
+            } else if (s[pos] == '~') {
+                pos++;
+                _operator = & Operator::get("=~");
+            } else {
+                _operator = & Operator::get("=");
+            }
+        } else if (s[pos] == '>') {
+            pos++;
+            if (s[pos] == '=') {
+                _operator = & Operator::get(">=");
+                pos++;
+            } else {
+                _operator = & Operator::get(">");
+            }
+        } else if (s[pos] == '<') {
+            pos++;
+            if (s[pos] == '=') {
+                pos++;
+                _operator = & Operator::get("<=");
+            } else {
+                _operator = & Operator::get("<");
+            }
+        } else if ((s[pos] == '!') && (s[pos] == '=')) {
+            pos += 2;
+            _operator = & Operator::get("!=");
+        } else {
+            retval = false;
+        }
+    }
+    setRemaining(s.substr(pos));
+    return retval;
+}
+
+bool StringParser::parse(const vespalib::stringref & s)
+{
+    bool retval(false);
+    setRemaining(s);
+    size_t pos(eatWhite(s.c_str(), s.size()));
+    if (pos + 1 < s.size()) {
+        if (s[pos++] == '"') {
+            vespalib::string str;
+            for(;(pos < s.size()) && (s[pos] != '"');pos++) {
+                if ((pos < s.size()) && (s[pos] == '\\')) {
+                    pos++;
+                }
+                str += s[pos];
+            }
+            if (s[pos] == '"') {
+                pos++;
+                retval = true;
+                setValue(ValueNode::UP(new StringValueNode(str)));
+            }
+        }
+        setRemaining(s.substr(pos+1));
+    }
+    return retval;
+}
+
+bool IntegerParser::parse(const vespalib::stringref & s)
+{
+    bool retval(false);
+    size_t pos(eatWhite(s.c_str(), s.size()));
+    if (pos < s.size()) {
+        char * err(NULL);
+        errno = 0;
+        bool isHex((s.size() - pos) && (s[pos] == '0') && (s[pos+1] == 'x'));
+        int64_t v = isHex
+                    ? strtoul(&s[pos], &err, 0)
+                    : strtol(&s[pos], &err, 0);
+        long len = err - &s[pos];
+        if ((errno == 0) && (pos+len <= s.size())) {
+            retval = true;
+            pos += len;
+            setValue(ValueNode::UP(new IntegerValueNode(v, false)));
+        }
+    }
+    setRemaining(s.substr(pos));
+    return retval;
+}
+
+bool SelectionParser::parse(const vespalib::stringref & s)
+{
+    bool retval(false);
+    IdSpecParser id(_bucketIdFactory);
+    if (id.parse(s)) {
+        OperatorParser op;
+        if (op.parse(id.getRemaining())) {
+            if (id.isUserSpec()) {
+                IntegerParser v;
+                if (v.parse(op.getRemaining())) {
+                    setNode(Node::UP(new Compare(id.stealValue(), *op.getOperator(), v.stealValue(), _bucketIdFactory)));
+                    retval = true;
+                }
+                setRemaining(v.getRemaining());
+            } else {
+                StringParser v;
+                if (v.parse(op.getRemaining())) {
+                    setNode(Node::UP(new Compare(id.stealValue(), *op.getOperator(), v.stealValue(), _bucketIdFactory)));
+                    retval = true;
+                }
+                setRemaining(v.getRemaining());
+            }
+        } else {
+            setRemaining(op.getRemaining());
+        }
+    } else {
+        setRemaining(id.getRemaining());
+    }
+    return retval;
+}
+
+
+
+} // simple
+} // select
+} // parser
diff --git a/document/src/vespa/document/select/simpleparser.h b/document/src/vespa/document/select/simpleparser.h
new file mode 100644
index 00000000000..742a7638175
--- /dev/null
+++ b/document/src/vespa/document/select/simpleparser.h
@@ -0,0 +1,97 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/vespalib/stllike/string.h>
+#include "node.h"
+#include "valuenode.h"
+#include "operator.h"
+#include <vespa/document/bucket/bucketidfactory.h>
+
+namespace document {
+
+namespace select {
+
+namespace simple {
+
+class Parser {
+public:
+    virtual ~Parser() { }
+    virtual bool parse(const vespalib::stringref & s) = 0;
+    const vespalib::stringref & getRemaining() const { return _remaining; }
+protected:
+    void setRemaining(const vespalib::stringref & s) { _remaining = s; }
+private:
+    vespalib::stringref _remaining;
+};
+
+class NodeResult {
+public:
+    Node::UP getNode() { return std::move(_node); }
+protected:
+    void setNode(Node::UP node) { _node = std::move(node); }
+private:
+    Node::UP            _node;
+};
+
+class ValueResult {
+public:
+    ValueNode::UP stealValue() { return std::move(_value); }
+    const ValueNode & getValue() const { return *_value; }
+protected:
+    void setValue(ValueNode::UP node) { _value = std::move(node); }
+private:
+    ValueNode::UP            _value;
+};
+
+class IdSpecParser : public Parser, public ValueResult
+{
+public:
+    IdSpecParser(const BucketIdFactory& bucketIdFactory) :
+        _bucketIdFactory(bucketIdFactory)
+    {}
+    virtual bool parse(const vespalib::stringref & s);
+    const IdValueNode & getId() const { return static_cast<const IdValueNode &>(getValue()); }
+    bool isUserSpec() const { return getId().getType() == IdValueNode::USER; }
+private:
+    const BucketIdFactory & _bucketIdFactory;
+};
+
+class OperatorParser : public Parser
+{
+public:
+    virtual bool parse(const vespalib::stringref & s);
+    const Operator * getOperator() const { return _operator; }
+private:
+    const Operator *_operator;
+};
+
+class StringParser : public Parser, public ValueResult
+{
+public:
+    virtual bool parse(const vespalib::stringref & s);
+};
+
+class IntegerParser : public Parser, public ValueResult
+{
+public:
+    virtual bool parse(const vespalib::stringref & s);
+};
+
+class SelectionParser : public Parser, public NodeResult
+{
+public:
+    SelectionParser(const BucketIdFactory& bucketIdFactory) :
+        _bucketIdFactory(bucketIdFactory)
+    {}
+    virtual bool parse(const vespalib::stringref & s);
+private:
+    const BucketIdFactory & _bucketIdFactory;
+};
+
+
+
+} // simple
+} // select
+} // parser
+
diff --git a/document/src/vespa/document/select/traversingvisitor.cpp b/document/src/vespa/document/select/traversingvisitor.cpp
new file mode 100644
index 00000000000..ffd840bd9fa
--- /dev/null
+++ b/document/src/vespa/document/select/traversingvisitor.cpp
@@ -0,0 +1,134 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "traversingvisitor.h"
+#include "valuenode.h"
+#include "branch.h"
+#include "compare.h" 
+
+namespace document
+{
+
+namespace select
+{
+
+void
+TraversingVisitor::visitAndBranch(const And &expr)
+{
+    expr.getLeft().visit(*this); expr.getRight().visit(*this);
+};
+
+
+void
+TraversingVisitor::visitOrBranch(const Or &expr)
+{
+    expr.getLeft().visit(*this); expr.getRight().visit(*this);
+};
+
+
+void
+TraversingVisitor::visitNotBranch(const Not &expr)
+{
+    expr.getChild().visit(*this);
+};
+
+
+void
+TraversingVisitor::visitComparison(const Compare &expr)
+{
+    expr.getLeft().visit(*this); expr.getRight().visit(*this);
+};
+
+
+void
+TraversingVisitor::visitArithmeticValueNode(const ArithmeticValueNode &expr)
+{
+    expr.getLeft().visit(*this); expr.getRight().visit(*this);
+}
+
+
+void
+TraversingVisitor::visitFunctionValueNode(const FunctionValueNode &expr) {
+    expr.getChild().visit(*this);
+};
+
+
+void
+TraversingVisitor::visitConstant(const Constant &)
+{
+}
+
+
+void
+TraversingVisitor::visitInvalidConstant(const InvalidConstant &)
+{
+}
+
+
+void
+TraversingVisitor::visitDocumentType(const DocType &)
+{
+}
+
+
+void
+TraversingVisitor::visitIdValueNode(const IdValueNode &)
+{
+}
+
+
+void
+TraversingVisitor::visitSearchColumnValueNode(const SearchColumnValueNode &)
+{
+}
+
+
+void
+TraversingVisitor::visitFieldValueNode(const FieldValueNode &)
+{
+}
+
+
+void
+TraversingVisitor::visitFloatValueNode(const FloatValueNode &)
+{
+}
+
+
+void
+TraversingVisitor::visitVariableValueNode(const VariableValueNode &)
+{
+}
+
+
+void
+TraversingVisitor::visitIntegerValueNode(const IntegerValueNode &)
+{
+}
+
+
+void
+TraversingVisitor::visitCurrentTimeValueNode(const CurrentTimeValueNode &)
+{
+}
+
+
+void
+TraversingVisitor::visitStringValueNode(const StringValueNode &)
+{
+}
+
+
+void
+TraversingVisitor::visitNullValueNode(const NullValueNode &)
+{
+}
+
+void
+TraversingVisitor::visitInvalidValueNode(const InvalidValueNode &)
+{
+}
+
+}
+
+}
diff --git a/document/src/vespa/document/select/traversingvisitor.h b/document/src/vespa/document/select/traversingvisitor.h
new file mode 100644
index 00000000000..9b3712571a5
--- /dev/null
+++ b/document/src/vespa/document/select/traversingvisitor.h
@@ -0,0 +1,74 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include "visitor.h"
+
+namespace document {
+namespace select {
+
+class
+TraversingVisitor : public Visitor
+{
+public:
+    virtual void
+    visitAndBranch(const And &expr);
+
+    virtual void
+    visitOrBranch(const Or &expr);
+
+    virtual void
+    visitNotBranch(const Not &expr);
+
+    virtual void
+    visitComparison(const Compare &expr);
+
+    virtual void
+    visitArithmeticValueNode(const ArithmeticValueNode &expr);
+
+    virtual void
+    visitFunctionValueNode(const FunctionValueNode &expr);
+
+    virtual void
+    visitConstant(const Constant &);
+
+    virtual void
+    visitInvalidConstant(const InvalidConstant &);
+
+    virtual void
+    visitDocumentType(const DocType &);
+
+    virtual void
+    visitIdValueNode(const IdValueNode &);
+
+    virtual void
+    visitSearchColumnValueNode(const SearchColumnValueNode &);
+
+    virtual void
+    visitFieldValueNode(const FieldValueNode &);
+
+    virtual void
+    visitFloatValueNode(const FloatValueNode &);
+
+    virtual void
+    visitVariableValueNode(const VariableValueNode &);
+
+    virtual void
+    visitIntegerValueNode(const IntegerValueNode &);
+
+    virtual void
+    visitCurrentTimeValueNode(const CurrentTimeValueNode &);
+
+    virtual void
+    visitStringValueNode(const StringValueNode &);
+
+    virtual void
+    visitNullValueNode(const NullValueNode &);
+
+    virtual void
+    visitInvalidValueNode(const InvalidValueNode &);
+};
+
+}
+
+}
+
diff --git a/document/src/vespa/document/select/value.cpp b/document/src/vespa/document/select/value.cpp
new file mode 100644
index 00000000000..c140cf5d373
--- /dev/null
+++ b/document/src/vespa/document/select/value.cpp
@@ -0,0 +1,422 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "value.h"
+#include "operator.h"
+#include <stdint.h>
+
+namespace document {
+namespace select {
+
+ResultList
+Value::globCompare(const Value& value) const
+{
+    return GlobOperator::GLOB.compareImpl(*this, value);
+}
+
+ResultList
+Value::regexCompare(const Value& value) const
+{
+    return RegexOperator::REGEX.compareImpl(*this, value);
+}
+
+ResultList
+Value::globTrace(const Value& value, std::ostream& trace) const
+{
+    return GlobOperator::GLOB.traceImpl(*this, value, trace);
+}
+
+ResultList
+Value::regexTrace(const Value& value, std::ostream& trace) const
+{
+    return RegexOperator::REGEX.traceImpl(*this, value, trace);
+}
+
+ResultList
+InvalidValue::operator<(const Value&) const
+{
+    return ResultList(Result::Invalid);
+}
+
+ResultList
+InvalidValue::operator==(const Value&) const
+{
+    return ResultList(Result::Invalid);
+}
+
+void
+InvalidValue::print(std::ostream& out, bool verbose,
+                    const std::string& indent) const
+{
+    (void) verbose; (void) indent;
+    out << "invalid";
+}
+
+ResultList
+NullValue::operator<(const Value&) const
+{
+    return ResultList(Result::Invalid);
+}
+
+ResultList
+NullValue::operator==(const Value& value) const
+{
+    const NullValue* nval(dynamic_cast<const NullValue*>(&value));
+    if (nval != 0) return ResultList(Result::True);
+    const InvalidValue* ival(dynamic_cast<const InvalidValue*>(&value));
+    return ResultList(ival != 0 ? Result::Invalid : Result::False);
+}
+
+
+ResultList
+NullValue::operator>(const Value &) const
+{
+    return ResultList(Result::Invalid);
+}
+
+
+ResultList
+NullValue::operator>=(const Value &) const
+{
+    return ResultList(Result::Invalid);
+}
+
+
+ResultList
+NullValue::operator<=(const Value &) const
+{
+    return ResultList(Result::Invalid);
+}
+
+
+void
+NullValue::print(std::ostream& out, bool verbose,
+                    const std::string& indent) const
+{
+    (void) verbose; (void) indent;
+    out << "nil";
+}
+
+StringValue::StringValue(const vespalib::stringref & val)
+    : Value(String),
+      _value(val)
+{
+}
+
+ResultList
+StringValue::operator<(const Value& value) const
+{
+    const StringValue* val(dynamic_cast<const StringValue*>(&value));
+    if (val == 0) return ResultList(Result::Invalid);
+    return ResultList(Result::get(_value < val->_value));
+}
+
+ResultList
+StringValue::operator==(const Value& value) const
+{
+    const StringValue* val(dynamic_cast<const StringValue*>(&value));
+    if (val == 0) {
+        const NullValue* nval(dynamic_cast<const NullValue*>(&value));
+        return ResultList(nval == 0 ? Result::Invalid : Result::False);
+    }
+    return ResultList(Result::get(_value == val->_value));
+}
+
+void
+StringValue::print(std::ostream& out, bool verbose,
+                    const std::string& indent) const
+{
+    (void) verbose; (void) indent;
+    out << "\"" << _value << "\"";
+}
+
+IntegerValue::IntegerValue(int64_t val, bool isBucketValue)
+    : NumberValue(isBucketValue ? Bucket : Integer),
+      _value(val)
+{
+}
+
+ResultList
+IntegerValue::operator<(const Value& value) const
+{
+    const NumberValue* val(dynamic_cast<const NumberValue*>(&value));
+    if (val == 0) return ResultList(Result::Invalid);
+    return val->operator>(*this);
+}
+
+ResultList
+IntegerValue::operator==(const Value& value) const
+{
+    const NumberValue* val(dynamic_cast<const NumberValue*>(&value));
+    if (val == 0) {
+        const NullValue* nval(dynamic_cast<const NullValue*>(&value));
+        return ResultList(nval == 0 ? Result::Invalid : Result::False);
+    }
+    return val->operator==(*this);
+}
+
+void
+IntegerValue::print(std::ostream& out, bool verbose,
+                    const std::string& indent) const
+{
+    (void) verbose; (void) indent;
+    out << _value << 'i';
+}
+
+FloatValue::FloatValue(double val)
+    : NumberValue(Float),
+      _value(val)
+{
+}
+
+ResultList
+FloatValue::operator<(const Value& value) const
+{
+    const NumberValue* val(dynamic_cast<const NumberValue*>(&value));
+    if (val == 0) return ResultList(Result::Invalid);
+    return val->operator>(*this);
+}
+
+ResultList
+FloatValue::operator==(const Value& value) const
+{
+    const NumberValue* val(dynamic_cast<const NumberValue*>(&value));
+    if (val == 0) {
+        const NullValue* nval(dynamic_cast<const NullValue*>(&value));
+        return ResultList(nval == 0 ? Result::Invalid : Result::False);
+    }
+    return val->operator==(*this);
+}
+
+void
+FloatValue::print(std::ostream& out, bool verbose,
+                    const std::string& indent) const
+{
+    (void) verbose; (void) indent;
+    out << _value << 'f';
+}
+
+ArrayValue::ArrayValue(const std::vector<VariableValue>& values)
+    : Value(Array),
+      _values(values)
+{
+}
+
+struct ArrayValue::EqualsComparator {
+    bool operator()(std::size_t lhs, std::size_t rhs) const { return lhs == rhs; }
+    ResultList operator()(const Value& lhs, const Value& rhs) const { return lhs == rhs; }
+};
+
+struct ArrayValue::NotEqualsComparator {
+    bool operator()(std::size_t lhs, std::size_t rhs) const { return lhs != rhs; }
+    ResultList operator()(const Value& lhs, const Value& rhs) const { return lhs != rhs; }
+};
+
+struct ArrayValue::LessThanComparator {
+    bool operator()(std::size_t lhs, std::size_t rhs) const { return lhs < rhs; }
+    ResultList operator()(const Value& lhs, const Value& rhs) const { return lhs < rhs; }
+};
+
+struct ArrayValue::GreaterThanComparator {
+    bool operator()(std::size_t lhs, std::size_t rhs) const { return lhs > rhs; }
+    ResultList operator()(const Value& lhs, const Value& rhs) const { return lhs > rhs; }
+};
+
+struct ArrayValue::LessThanOrEqualsComparator {
+    bool operator()(std::size_t lhs, std::size_t rhs) const { return lhs <= rhs; }
+    ResultList operator()(const Value& lhs, const Value& rhs) const { return lhs <= rhs; }
+};
+
+struct ArrayValue::GreaterThanOrEqualsComparator {
+    bool operator()(std::size_t lhs, std::size_t rhs) const { return lhs >= rhs; }
+    ResultList operator()(const Value& lhs, const Value& rhs) const { return lhs >= rhs; }
+};
+
+struct ArrayValue::GlobComparator {
+    bool operator()(std::size_t lhs, std::size_t rhs) const { return lhs == rhs; }
+    ResultList operator()(const Value& lhs, const Value& rhs) const {
+        return GlobOperator::GLOB.compareImpl(lhs, rhs);
+    }
+};
+
+struct ArrayValue::RegexComparator {
+    bool operator()(std::size_t lhs, std::size_t rhs) const { return lhs == rhs; }
+    ResultList operator()(const Value& lhs, const Value& rhs) const {
+        return RegexOperator::REGEX.compareImpl(lhs, rhs);
+    }
+};
+
+
+ResultList
+ArrayValue::operator<(const Value& value) const
+{
+    return doCompare(value, LessThanComparator());
+}
+
+ResultList
+ArrayValue::operator==(const Value& value) const
+{
+    return doCompare(value, EqualsComparator());
+}
+
+ResultList
+ArrayValue::operator>(const Value& value) const
+{
+    return doCompare(value, GreaterThanComparator());
+}
+
+ResultList
+ArrayValue::operator>=(const Value& value) const
+{
+    return doCompare(value, GreaterThanOrEqualsComparator());
+}
+
+ResultList
+ArrayValue::operator<=(const Value& value) const
+{
+    return doCompare(value, LessThanOrEqualsComparator());
+}
+
+ResultList
+ArrayValue::operator!=(const Value& value) const
+{
+    return doCompare(value, NotEqualsComparator());
+}
+
+ResultList
+ArrayValue::globCompare(const Value& value) const
+{
+    return doCompare(value, GlobComparator());
+}
+ResultList
+ArrayValue::regexCompare(const Value& value) const
+{
+    return doCompare(value, RegexComparator());
+}
+
+ResultList
+ArrayValue::globTrace(const Value& value, std::ostream& trace) const
+{
+    trace << "Glob compare of lhs ArrayValue, rhs " << value << "\n";
+    // TODO: have trace be propagated down to comparison ops?
+    return doCompare(value, GlobComparator());
+}
+
+ResultList
+ArrayValue::regexTrace(const Value& value, std::ostream& trace) const
+{
+    trace << "Regex compare of lhs ArrayValue, rhs " << value << "\n";
+    // TODO: have trace be propagated down to comparison ops?
+    return doCompare(value, RegexComparator());
+}
+
+void
+ArrayValue::print(std::ostream& out, bool verbose,
+                    const std::string& indent) const
+{
+    (void) verbose; (void) indent;
+    out << "<no array representation in language yet>";
+}
+
+StructValue::StructValue(const ValueMap & values)
+    : Value(Struct),
+      _values(values)
+{
+}
+
+ResultList
+StructValue::operator<(const Value& value) const
+{
+    const StructValue* val(dynamic_cast<const StructValue*>(&value));
+    if (val == 0) {
+        return ResultList(Result::Invalid);
+    }
+    ValueMap::const_iterator it1 = _values.begin();
+    ValueMap::const_iterator it2 = val->_values.begin();
+    while (it1 != _values.end() && it2 != val->_values.end()) {
+        if (it1->first != it2->first) {
+            return ResultList(it1->first < it2->first ? Result::True : Result::False);
+        }
+        ResultList result = (*it1->second == *it2->second);
+        if (result == Result::True) {
+            ++it1;
+            ++it2;
+            continue;
+        }
+        result = (*it1->second < *it2->second);
+        return result;
+    }
+    if (it1 != _values.end() || it2 != val->_values.end()) {
+        return ResultList(it1 == _values.end() ? Result::True : Result::False);
+    }
+    return ResultList(Result::False);
+}
+
+ResultList
+StructValue::operator==(const Value& value) const
+{
+    const StructValue* val(dynamic_cast<const StructValue*>(&value));
+    if (val == 0) {
+        const NullValue* nval(dynamic_cast<const NullValue*>(&value));
+        return ResultList(nval == 0 ? Result::Invalid : Result::False);
+    }
+    ValueMap::const_iterator it1 = _values.begin();
+    ValueMap::const_iterator it2 = val->_values.begin();
+    while (it1 != _values.end() && it2 != val->_values.end()) {
+        if (it1->first != it2->first) {
+            return ResultList(Result::False);
+        }
+        ResultList result = (*it1->second == *it2->second);
+        if (result == Result::True) {
+            ++it1;
+            ++it2;
+            continue;
+        }
+        return ResultList(Result::False);
+    }
+    if (it1 != _values.end() || it2 != val->_values.end()) {
+        return ResultList(Result::False);
+    }
+    return ResultList(Result::True);
+}
+
+void
+StructValue::print(std::ostream& out, bool verbose,
+                    const std::string& indent) const
+{
+    (void) verbose; (void) indent;
+    out << "<no struct representation in language yet>";
+}
+
+template <typename Predicate>
+ResultList
+ArrayValue::doCompare(const Value& value, const Predicate& cmp) const
+{
+    if (value.getType() == Array) {
+        // If comparing with an array, must match all.
+        const ArrayValue* val(static_cast<const ArrayValue*>(&value));
+        if (_values.size() != val->_values.size()) {
+            return ResultList(cmp(_values.size(), val->_values.size())
+                              ? Result::True : Result::False);
+        }
+        for (uint32_t i=0; i<_values.size(); ++i) {
+            ResultList result = cmp(*_values[i].second, *val->_values[i].second);
+            if (result == Result::False || result == Result::Invalid) {
+                return result;
+            }
+        }
+        return ResultList(Result::True);
+    } else {
+        ResultList results;
+
+        // If comparing with other value, must match one.
+        for (uint32_t i=0; i<_values.size(); ++i) {
+            results.add(_values[i].first,
+                        cmp(*_values[i].second, value).combineResults());
+        }
+        return results;
+    }
+}
+
+} // select
+} // document
diff --git a/document/src/vespa/document/select/value.h b/document/src/vespa/document/select/value.h
new file mode 100644
index 00000000000..b4e35d1528d
--- /dev/null
+++ b/document/src/vespa/document/select/value.h
@@ -0,0 +1,265 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * @class document::select::Value
+ * @ingroup select
+ *
+ * @brief Values are entities that can be compared.
+ *
+ * To be able to cope with field specifications that can end up in values of
+ * multiple types we need an abstraction.
+ *
+ * @author H�kon Humberset
+ * @date 2007-20-05
+ * @version $Id$
+ */
+
+#pragma once
+
+#include <boost/operators.hpp>
+#include <memory>
+#include <map>
+#include <string>
+#include <vector>
+#include <iosfwd>
+#include "resultlist.h"
+
+namespace document {
+namespace select {
+
+class Value : public document::Printable
+{
+public:
+    typedef std::shared_ptr<Value> SP;
+    typedef std::unique_ptr<Value> UP;
+    enum Type { Invalid, Null, String, Integer, Float, Array, Struct, Bucket };
+
+    Value(Type t) : _type(t) {}
+    virtual ~Value() {}
+
+    Type getType() const { return _type; }
+
+    virtual ResultList operator<(const Value& value) const = 0;
+    virtual ResultList operator==(const Value& value) const = 0;
+
+    virtual UP clone() const = 0;
+
+    virtual ResultList operator!=(const Value& value) const
+        { return !(this->operator==(value)); }
+    virtual ResultList operator>(const Value& value) const
+        { return (!(this->operator<(value)) && !(this->operator==(value))); }
+    virtual ResultList operator>=(const Value& value) const
+        { return !(this->operator<(value)); }
+    virtual ResultList operator<=(const Value& value) const
+        { return ((this->operator<(value)) || (this->operator==(value))); }
+
+    virtual ResultList globCompare(const Value& value) const;
+    virtual ResultList regexCompare(const Value& value) const;
+    virtual ResultList globTrace(const Value& value, std::ostream& trace) const;
+    virtual ResultList regexTrace(const Value& value, std::ostream& trace) const;
+
+private:
+    Type _type;
+};
+
+class InvalidValue : public Value
+{
+public:
+    InvalidValue() : Value(Invalid) {}
+
+    virtual ResultList operator<(const Value&) const;
+    virtual ResultList operator==(const Value&) const;
+    virtual void print(std::ostream& out, bool verbose,
+                       const std::string& indent) const;
+
+    Value::UP clone() const { return Value::UP(new InvalidValue()); }
+
+};
+
+class NullValue : public Value
+{
+public:
+    NullValue() : Value(Null) {}
+
+    virtual ResultList operator<(const Value&) const;
+    virtual ResultList operator==(const Value&) const;
+
+    virtual ResultList
+    operator>(const Value &) const;
+
+    virtual ResultList
+    operator>=(const Value &) const;
+
+    virtual ResultList
+    operator<=(const Value &) const;
+
+    virtual void print(std::ostream& out, bool verbose,
+                       const std::string& indent) const;
+
+    Value::UP clone() const { return Value::UP(new NullValue()); }
+};
+
+class StringValue : public Value
+{
+    vespalib::string _value;
+
+public:
+    StringValue(const vespalib::stringref & val);
+
+    const vespalib::string& getValue() const { return _value; }
+
+    virtual ResultList operator<(const Value& value) const;
+    virtual ResultList operator==(const Value& value) const;
+    virtual void print(std::ostream& out, bool verbose,
+                       const std::string& indent) const;
+
+    Value::UP clone() const { return Value::UP(new StringValue(_value)); }
+};
+
+class IntegerValue;
+class FloatValue;
+
+class NumberValue : public Value
+{
+public:
+    typedef double CommonValueType;
+
+    NumberValue(Type t) : Value(t) {}
+
+    virtual CommonValueType getCommonValue() const = 0;
+
+    virtual ResultList operator<(const Value& value) const = 0;
+    virtual ResultList operator>(const IntegerValue& value) const = 0;
+    virtual ResultList operator>(const FloatValue& value) const = 0;
+    virtual ResultList operator==(const Value& value) const = 0;
+    virtual ResultList operator==(const IntegerValue& value) const = 0;
+    virtual ResultList operator==(const FloatValue& value) const = 0;
+};
+
+class IntegerValue : public NumberValue
+{
+public:
+    typedef int64_t ValueType;
+
+    IntegerValue(ValueType value, bool isBucketValue);
+
+    ValueType getValue() const { return _value; }
+    CommonValueType getCommonValue() const { return _value; }
+
+    virtual ResultList operator<(const Value& value) const;
+    virtual ResultList operator==(const Value& value) const;
+
+    virtual ResultList operator>(const IntegerValue& value) const;
+    virtual ResultList operator>(const FloatValue& value) const;
+    virtual ResultList operator==(const IntegerValue& value) const;
+    virtual ResultList operator==(const FloatValue& value) const;
+    virtual void print(std::ostream& out, bool verbose,
+                       const std::string& indent) const;
+
+    Value::UP clone() const {
+        return Value::UP(new IntegerValue(_value, getType() == Value::Bucket));
+    }
+private:
+    ValueType _value;
+};
+
+class FloatValue : public NumberValue
+{
+public:
+    typedef double ValueType;
+
+    FloatValue(ValueType val);
+
+    ValueType getValue() const { return _value; }
+    CommonValueType getCommonValue() const { return _value; }
+
+    virtual ResultList operator<(const Value& value) const;
+    virtual ResultList operator==(const Value& value) const;
+
+    virtual ResultList operator>(const IntegerValue& value) const;
+    virtual ResultList operator>(const FloatValue& value) const;
+    virtual ResultList operator==(const IntegerValue& value) const;
+    virtual ResultList operator==(const FloatValue& value) const;
+    virtual void print(std::ostream& out, bool verbose,
+                       const std::string& indent) const;
+
+    Value::UP clone() const { return Value::UP(new FloatValue(_value)); }
+private:
+    ValueType _value;
+};
+
+inline ResultList IntegerValue::operator>(const IntegerValue& value) const
+    { return ResultList(Result::get(_value > value.getValue())); }
+inline ResultList IntegerValue::operator>(const FloatValue& value) const
+    { return ResultList(Result::get(_value > value.getValue())); }
+inline ResultList IntegerValue::operator==(const IntegerValue& value) const
+    { return ResultList(Result::get(_value == value.getValue())); }
+inline ResultList IntegerValue::operator==(const FloatValue& value) const
+    { return ResultList(Result::get(_value == value.getValue())); }
+
+inline ResultList FloatValue::operator>(const IntegerValue& value) const
+    { return ResultList(Result::get(_value > value.getValue())); }
+inline ResultList FloatValue::operator>(const FloatValue& value) const
+    { return ResultList(Result::get(_value > value.getValue())); }
+inline ResultList FloatValue::operator==(const IntegerValue& value) const
+    { return ResultList(Result::get(_value == value.getValue())); }
+inline ResultList FloatValue::operator==(const FloatValue& value) const
+    { return ResultList(Result::get(_value == value.getValue())); }
+
+class ArrayValue : public Value
+{
+public:
+    typedef std::pair<FieldValue::IteratorHandler::VariableMap, Value::SP> VariableValue;
+
+    ArrayValue(const std::vector<VariableValue>& values);
+
+    virtual ResultList operator<(const Value& value) const;
+    virtual ResultList operator>(const Value& value) const;
+    virtual ResultList operator==(const Value& value) const;
+    virtual ResultList operator!=(const Value& value) const;
+    virtual ResultList operator>=(const Value& value) const;
+    virtual ResultList operator<=(const Value& value) const;
+    virtual void print(std::ostream& out, bool verbose,
+                       const std::string& indent) const;
+
+    virtual ResultList globCompare(const Value& value) const;
+    virtual ResultList regexCompare(const Value& value) const;
+    virtual ResultList globTrace(const Value& value, std::ostream& trace) const;
+    virtual ResultList regexTrace(const Value& value, std::ostream& trace) const;
+
+    template <typename Predicate>
+    ResultList doCompare(const Value& value, const Predicate& cmp) const;
+
+    Value::UP clone() const { return Value::UP(new ArrayValue(_values)); }
+
+private:
+    struct EqualsComparator;
+    struct NotEqualsComparator;
+    struct LessThanComparator;
+    struct GreaterThanComparator;
+    struct LessThanOrEqualsComparator;
+    struct GreaterThanOrEqualsComparator;
+    struct GlobComparator;
+    struct RegexComparator;
+
+    std::vector<VariableValue> _values;
+};
+
+class StructValue : public Value
+{
+public:
+    typedef std::map<vespalib::string, Value::SP> ValueMap;
+    StructValue(const ValueMap & values);
+
+    virtual ResultList operator<(const Value& value) const;
+    virtual ResultList operator==(const Value& value) const;
+    virtual void print(std::ostream& out, bool verbose,
+                       const std::string& indent) const;
+
+    Value::UP clone() const { return Value::UP(new StructValue(_values)); }
+private:
+    ValueMap _values;
+};
+
+} // select
+} // document
+
diff --git a/document/src/vespa/document/select/valuenode.cpp b/document/src/vespa/document/select/valuenode.cpp
new file mode 100644
index 00000000000..94be21a1657
--- /dev/null
+++ b/document/src/vespa/document/select/valuenode.cpp
@@ -0,0 +1,1166 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include "valuenode.h"
+#include "visitor.h"
+
+#include <iomanip>
+#include <vespa/log/log.h>
+#include <vespa/document/base/exceptions.h>
+#include "parser.h"
+#include <vespa/document/fieldvalue/fieldvalues.h>
+#include <vespa/vespalib/util/md5.h>
+#include <vespa/document/util/stringutil.h>
+#include <sys/time.h>
+#include <vespa/vespalib/text/lowercase.h>
+
+LOG_SETUP(".document.select.valuenode");
+
+namespace document {
+namespace select {
+
+namespace {
+    bool documentTypeEqualsName(const DocumentType& type,
+                                const vespalib::stringref& name)
+    {
+        if (type.getName() == name) return true;
+        for (std::vector<const DocumentType *>::const_iterator it
+                = type.getInheritedTypes().begin();
+             it != type.getInheritedTypes().end(); ++it)
+        {
+            if (documentTypeEqualsName(**it, name)) return true;
+        }
+        return false;
+    }
+}
+
+InvalidValueNode::InvalidValueNode(const vespalib::stringref & name)
+    : _name(name)
+{
+}
+
+
+void
+InvalidValueNode::visit(Visitor &visitor) const
+{
+    visitor.visitInvalidValueNode(*this);
+}
+
+
+void
+InvalidValueNode::print(std::ostream& out, bool verbose,
+                        const std::string& indent) const
+{
+    (void) verbose; (void) indent;
+    if (hadParentheses()) out << '(';
+    out << _name;
+    if (hadParentheses()) out << ')';
+}
+
+NullValueNode::NullValueNode(const vespalib::stringref & name)
+    : _name(name)
+{
+}
+
+
+void
+NullValueNode::visit(Visitor &visitor) const
+{
+    visitor.visitNullValueNode(*this);
+}
+
+
+void
+NullValueNode::print(std::ostream& out, bool verbose,
+                        const std::string& indent) const
+{
+    (void) verbose; (void) indent;
+    if (hadParentheses()) out << '(';
+    out << _name;
+    if (hadParentheses()) out << ')';
+}
+
+StringValueNode::StringValueNode(const vespalib::stringref & val)
+    : _value(val)
+{
+}
+
+
+void
+StringValueNode::visit(Visitor &visitor) const
+{
+    visitor.visitStringValueNode(*this);
+}
+
+
+void
+StringValueNode::print(std::ostream& out, bool verbose,
+                       const std::string& indent) const
+{
+    (void) verbose; (void) indent;
+    if (hadParentheses()) out << '(';
+    out << "\"" << StringUtil::escape(_value) << "\"";
+    if (hadParentheses()) out << ')';
+}
+
+
+void
+IntegerValueNode::visit(Visitor &visitor) const
+{
+    visitor.visitIntegerValueNode(*this);
+}
+
+
+void
+IntegerValueNode::print(std::ostream& out, bool verbose,
+                       const std::string& indent) const
+{
+    (void) verbose; (void) indent;
+    if (hadParentheses()) out << '(';
+    out << _value;
+    if (hadParentheses()) out << ')';
+}
+
+int64_t
+CurrentTimeValueNode::getValue() const
+{
+    struct timeval mytime;
+    gettimeofday(&mytime, 0);
+    return mytime.tv_sec;
+}
+
+
+void
+CurrentTimeValueNode::visit(Visitor &visitor) const
+{
+    visitor.visitCurrentTimeValueNode(*this);
+}
+
+
+void
+CurrentTimeValueNode::print(std::ostream& out, bool verbose,
+                            const std::string& indent) const
+{
+    (void) verbose; (void) indent;
+    out << "now()";
+}
+
+std::unique_ptr<Value>
+VariableValueNode::getValue(const Context& context) const {
+    VariableMap::const_iterator iter = context._variables.find(_value);
+
+    if (iter != context._variables.end()) {
+        return std::unique_ptr<Value>(new FloatValue(iter->second));
+    } else {
+        return std::unique_ptr<Value>(new FloatValue(0.0));
+    }
+}
+
+
+void
+VariableValueNode::visit(Visitor &visitor) const
+{
+    visitor.visitVariableValueNode(*this);
+}
+
+
+void
+VariableValueNode::print(std::ostream& out, bool verbose,
+                       const std::string& indent) const
+{
+    (void) verbose; (void) indent;
+    if (hadParentheses()) out << '(';
+    out << "$" << _value;
+    if (hadParentheses()) out << ')';
+}
+
+
+void
+FloatValueNode::visit(Visitor &visitor) const
+{
+    visitor.visitFloatValueNode(*this);
+}
+
+
+void
+FloatValueNode::print(std::ostream& out, bool verbose,
+                       const std::string& indent) const
+{
+    (void) verbose; (void) indent;
+    if (hadParentheses()) out << '(';
+    out << _value;
+    if (hadParentheses()) out << ')';
+}
+
+FieldValueNode::FieldValueNode(const vespalib::string& doctype,
+                               const vespalib::string& field)
+    : _doctype(doctype),
+      _field(field)
+{
+}
+
+void
+FieldValueNode::initFieldPath(const DocumentType& type) const {
+    if (_fieldPath.size() == 0) {
+        FieldPath::UP path(type.buildFieldPath(_field));
+        if (!path.get()) {
+            throw FieldNotFoundException(
+                    vespalib::make_string("Could not create field path for doc "
+                                          "type: '%s' field: '%s'",
+                                          type.toString().c_str(),
+                                          _field.c_str()),
+                    VESPA_STRLOC);
+        }
+        _fieldPath = *path;
+    }
+}
+
+std::unique_ptr<Value>
+FieldValueNode::getValue(const Context& context) const
+{
+    if (context._doc == NULL) {
+        return std::unique_ptr<Value>(new InvalidValue());
+    }
+
+    const Document& doc = *context._doc;
+
+    if (!documentTypeEqualsName(doc.getType(), _doctype)) {
+        return std::unique_ptr<Value>(new InvalidValue());
+    }
+    try{
+        initFieldPath(doc.getType());
+
+        IteratorHandler handler;
+        doc.iterateNested(_fieldPath.begin(), _fieldPath.end(), handler);
+
+        if (handler.hasSingleValue()) {
+            return handler.getSingleValue();
+        } else {
+            const std::vector<ArrayValue::VariableValue>& values = handler.getValues();
+
+            if (values.size() == 0) {
+                return std::unique_ptr<Value>(new NullValue());
+            } else {
+                return std::unique_ptr<Value>(new ArrayValue(handler.getValues()));
+            }
+        }
+    } catch (vespalib::IllegalArgumentException& e) {
+        LOG(warning, "Caught exception while fetching field from document: %s", e.what());
+        return std::unique_ptr<Value>(new InvalidValue());
+    } catch (FieldNotFoundException& e) {
+        LOG(warning, "Tried to compare to field %s, not found in document type",
+                     _field.c_str());
+        return std::unique_ptr<Value>(new InvalidValue());
+    }
+}
+
+bool
+FieldValueNode::IteratorHandler::hasSingleValue() const {
+    return _firstValue.get() && (_values.size() == 0);
+}
+
+std::unique_ptr<Value>
+FieldValueNode::IteratorHandler::getSingleValue() {
+    return std::move(_firstValue);
+}
+
+const std::vector<ArrayValue::VariableValue>&
+FieldValueNode::IteratorHandler::getValues() {
+    if (_firstValue.get()) {
+        _values.insert(_values.begin(), ArrayValue::VariableValue(FieldValue::IteratorHandler::VariableMap(), Value::SP(_firstValue.release())));
+    }
+
+    return _values;
+}
+
+void
+FieldValueNode::IteratorHandler::onPrimitive(const Content& fv) {
+    if (!_firstValue.get() && getVariables().empty()) {
+        _firstValue = getInternalValue(fv.getValue());
+    } else {
+        _values.push_back(ArrayValue::VariableValue(
+                                  getVariables(),
+                                  Value::SP(getInternalValue(fv.getValue()).release())));
+    }
+}
+
+std::unique_ptr<Value>
+FieldValueNode::IteratorHandler::getInternalValue(const FieldValue& fval) const
+{
+    switch(fval.getClass().id()) {
+        case document::IntFieldValue::classId:
+        {
+            const IntFieldValue& val(dynamic_cast<const IntFieldValue&>(fval));
+            return std::unique_ptr<Value>(
+                    new IntegerValue(val.getAsInt(), false));
+        }
+        case document::ByteFieldValue::classId:
+        {
+            const ByteFieldValue& val(dynamic_cast<const ByteFieldValue&>(fval));
+            return std::unique_ptr<Value>(
+                    new IntegerValue(val.getAsByte(), false));
+        }
+        case LongFieldValue::classId:
+        {
+            const LongFieldValue& val(
+                    dynamic_cast<const LongFieldValue&>(fval));
+            return std::unique_ptr<Value>(
+                    new IntegerValue(val.getAsLong(), false));
+        }
+        case FloatFieldValue::classId:
+        {
+            const FloatFieldValue& val(
+                    dynamic_cast<const FloatFieldValue&>(fval));
+            return std::unique_ptr<Value>(new FloatValue(val.getAsFloat()));
+        }
+        case DoubleFieldValue::classId:
+
+       {
+            const DoubleFieldValue& val(
+                    dynamic_cast<const DoubleFieldValue&>(fval));
+            return std::unique_ptr<Value>(new FloatValue(val.getAsDouble()));
+        }
+        case StringFieldValue::classId:
+        {
+            const StringFieldValue& val(
+                    dynamic_cast<const StringFieldValue&>(fval));
+            return std::unique_ptr<Value>(new StringValue(val.getAsString()));
+        }
+        case ArrayFieldValue::classId:
+        {
+            const ArrayFieldValue& val(
+                    dynamic_cast<const ArrayFieldValue&>(fval));
+            if (val.size() == 0) {
+                return std::unique_ptr<Value>(new NullValue());
+            } else {
+                std::vector<ArrayValue::VariableValue> values;
+                // TODO: Array comparison.
+                return std::unique_ptr<Value>(new ArrayValue(values));
+            }
+        }
+        case StructFieldValue::classId:
+        {
+            const StructFieldValue& val(
+                    dynamic_cast<const StructFieldValue&>(fval));
+            if (val.empty()) {
+                return std::unique_ptr<Value>(new NullValue());
+            } else {
+                StructValue::ValueMap values;
+                for (StructFieldValue::const_iterator it(val.begin());
+                     it != val.end(); ++it)
+                {
+                    FieldValue::UP fv(val.getValue(it.field()));
+                    values[it.field().getName()] = Value::SP(getInternalValue(*fv).release());
+                }
+                return std::unique_ptr<Value>(new StructValue(values));
+            }
+        }
+        case MapFieldValue::classId:
+        {
+            const MapFieldValue& val(
+                    static_cast<const MapFieldValue&>(fval));
+            if (val.isEmpty()) {
+                return std::unique_ptr<Value>(new NullValue());
+            } else {
+                std::vector<ArrayValue::VariableValue> values;
+                // TODO: Map comparison
+                return std::unique_ptr<Value>(new ArrayValue(values));
+            }
+        }
+    }
+    LOG(warning, "Tried to use unsupported datatype %s in field comparison",
+                 fval.getDataType()->toString().c_str());
+    return std::unique_ptr<Value>(new InvalidValue());
+}
+
+
+void
+FieldValueNode::visit(Visitor &visitor) const
+{
+    visitor.visitFieldValueNode(*this);
+}
+
+
+void
+FieldValueNode::print(std::ostream& out, bool verbose,
+                       const std::string& indent) const
+{
+    (void) verbose; (void) indent;
+    if (hadParentheses()) out << '(';
+    out << _doctype << "." << _field;
+    if (hadParentheses()) out << ')';
+}
+
+
+std::unique_ptr<Value>
+FieldValueNode::traceValue(const Context &context, std::ostream& out) const
+{
+    if (context._doc == NULL) {
+        return defaultTrace(getValue(context), out);
+    }
+    const Document &doc(*context._doc);
+    if (!documentTypeEqualsName(doc.getType(), _doctype)) {
+        out << "Document is of type " << doc.getType() << " which isn't a "
+            << _doctype << " document, thus resolving invalid.\n";
+        return std::unique_ptr<Value>(new InvalidValue());
+    }
+    try{
+        initFieldPath(doc.getType());
+
+        IteratorHandler handler;
+        doc.iterateNested(_fieldPath.begin(), _fieldPath.end(), handler);
+
+        if (handler.hasSingleValue()) {
+            return handler.getSingleValue();
+        } else {
+            const std::vector<ArrayValue::VariableValue>& values = handler.getValues();
+
+            if (values.size() == 0) {
+                return std::unique_ptr<Value>(new NullValue());
+            } else {
+                return std::unique_ptr<Value>(new ArrayValue(handler.getValues()));
+            }
+        }
+    } catch (FieldNotFoundException& e) {
+        LOG(warning, "Tried to compare to field %s, not found in document type",
+                     _field.c_str());
+        out << "Field not found in document type " << doc.getType()
+            << ". Returning invalid.\n";
+        return std::unique_ptr<Value>(new InvalidValue());
+    }
+}
+
+IdValueNode::IdValueNode(const BucketIdFactory& bucketIdFactory,
+                         const vespalib::stringref & name, const vespalib::stringref & type,
+                         int widthBits, int divisionBits)
+    : _bucketIdFactory(bucketIdFactory),
+      _id(name),
+      _typestring(type),
+      _type(ALL),
+      _widthBits(widthBits),
+      _divisionBits(divisionBits)
+{
+    if (type.length() > 2) switch (type[0]) {
+    case 'b': _type = BUCKET;
+        break;
+    case 'n': _type = NS;
+        break;
+    case 'g':
+        if (type[1] == 'r') {
+            _type = GROUP;
+        } else if (type[1] == 'i') {
+            _type = GID;
+        }
+        break;
+    case 's': {
+        if (type[1] == 'c') { _type = SCHEME; } else { _type = SPEC; }
+        break;
+    }
+    case 't':
+        _type = TYPE;
+        break;
+    case 'u':
+        _type = USER;
+        break;
+    case 'o':
+        _type = ORDER;
+        break;
+    }
+}
+
+
+std::unique_ptr<Value>
+IdValueNode::getValue(const Context& context) const
+{
+    if (context._doc != NULL) {
+        return getValue(context._doc->getId());
+    } else if (context._docId != NULL) {
+        return getValue(*context._docId);
+    } else {
+        return getValue(context._docUpdate->getId());
+    }
+}
+
+
+std::unique_ptr<Value>
+IdValueNode::getValue(const DocumentId& id) const
+{
+    vespalib::string value;
+    switch (_type) {
+    case BUCKET:
+        return std::unique_ptr<Value>(
+            new IntegerValue(
+                _bucketIdFactory.getBucketId(id).getId(), true));
+    case NS:
+        value = id.getScheme().getNamespace(); break;
+    case SCHEME:
+        value = id.getScheme().getTypeName(id.getScheme().getType());
+        break;
+    case TYPE:
+        if (id.getScheme().hasDocType()) {
+            value = id.getScheme().getDocType();
+        } else {
+            return std::unique_ptr<Value>(new InvalidValue);
+        }
+        break;
+    case SPEC:
+        value = id.getScheme().getNamespaceSpecific();
+        break;
+    case ALL:
+        value = id.getScheme().toString();
+        break;
+    case GROUP:
+        if (id.getScheme().hasGroup()) {
+            value = id.getScheme().getGroup();
+        } else {
+            fprintf(stderr, "***** Returning invalid value for %s\n",
+                    id.toString().c_str());
+            return std::unique_ptr<Value>(new InvalidValue);
+        }
+        break;
+    case GID:
+        value = id.getGlobalId().toString();
+        break;
+    case ORDER:
+        if (id.getScheme().getType() == IdString::ORDERDOC) {
+            const OrderDocIdString& ods(
+                    static_cast<const OrderDocIdString&>(id.getScheme()));
+            if (ods.getWidthBits() == _widthBits
+                && ods.getDivisionBits() == _divisionBits)
+            {
+                return std::unique_ptr<Value>(new IntegerValue(
+                        static_cast<const OrderDocIdString&>(id.getScheme())
+                            .getOrdering(), false));
+            }
+        }
+        return std::unique_ptr<Value>(new InvalidValue());
+    case USER:
+        if (id.getScheme().hasNumber()) {
+            return std::unique_ptr<Value>(
+                    new IntegerValue(id.getScheme().getNumber(), false));
+        } else {
+            return std::unique_ptr<Value>(new InvalidValue);
+        }
+    }
+
+    return std::unique_ptr<Value>(new StringValue(value));
+}
+
+
+std::unique_ptr<Value>
+IdValueNode::traceValue(const Context& context,
+                        std::ostream &out) const
+{
+    if (context._doc != NULL) {
+        return traceValue(context._doc->getId(), out);
+    } else if (context._docId != NULL) {
+        return traceValue(*context._docId, out);
+    } else {
+        return traceValue(context._docUpdate->getId(), out);
+    }
+}
+
+
+std::unique_ptr<Value>
+IdValueNode::traceValue(const DocumentId& id, std::ostream& out) const
+{
+    vespalib::string value;
+    switch (_type) {
+    case BUCKET:
+        {
+            document::BucketId bucket(_bucketIdFactory.getBucketId(id));
+            std::unique_ptr<Value> result(
+                new IntegerValue(bucket.getId(), true));
+            out << "Found id.bucket specification. Resolved to "
+                << bucket << ".\n";
+            return result;
+        }
+    case NS:
+        value = id.getScheme().getNamespace();
+        out << "Resolved id.namespace to value\"" << value << "\".\n";
+        break;
+    case SCHEME:
+        value = id.getScheme().getTypeName(id.getScheme().getType());
+        out << "Resolved id.scheme to value\"" << value << "\".\n";
+        break;
+    case TYPE:
+        if (id.getScheme().hasDocType()) {
+            value = id.getScheme().getDocType();
+            out << "Resolved id.type to value\"" << value << "\".\n";
+        } else {
+            out << "Could not resolve type of doc " << id << ".\n";
+            return std::unique_ptr<Value>(new InvalidValue);
+        }
+        break;
+    case SPEC:
+        value = id.getScheme().getNamespaceSpecific();
+        out << "Resolved id.specific to value\"" << value << "\".\n";
+        break;
+    case ALL:
+        value = id.getScheme().toString();
+        out << "Resolved id to \"" << value << "\".\n";
+        break;
+    case GROUP:
+        if (id.getScheme().hasGroup()) {
+            value = id.getScheme().getGroup();
+            out << "Resolved group of doc (type " << id.getScheme().getType()
+                << ") to \"" << value << "\".\n";
+        } else {
+            out << "Can't resolve group of doc \"" << id << "\".\n";
+            return std::unique_ptr<Value>(new InvalidValue);
+        }
+        break;
+    case GID:
+        value = id.getGlobalId().toString();
+        out << "Resolved gid to \"" << value << "\".\n";
+        break;
+    case ORDER:
+        if (id.getScheme().getType() == IdString::ORDERDOC) {
+            const OrderDocIdString& ods(
+                    static_cast<const OrderDocIdString&>(id.getScheme()));
+            if (ods.getWidthBits() == _widthBits
+                && ods.getDivisionBits() == _divisionBits)
+            {
+                std::unique_ptr<Value> result(new IntegerValue(
+                        static_cast<const OrderDocIdString&>(id.getScheme())
+                            .getOrdering(), false));
+                out << "Resolved id.order to int " << *result << "\n";
+                return result;
+            }
+        }
+        out << "Could not resolve id.order(" << _widthBits << ", "
+            << _divisionBits << ") of doc " << id << ".\n";
+        return std::unique_ptr<Value>(new InvalidValue());
+    case USER:
+        if (id.getScheme().hasNumber()) {
+            std::unique_ptr<Value> result(
+                    new IntegerValue(id.getScheme().getNumber(), false));
+            out << "Resolved user of doc type " << id.getScheme().getType()
+                << " to " << *result << ".\n";
+            return result;
+        } else {
+            out << "Could not resolve user of doc " << id << ".\n";
+            return std::unique_ptr<Value>(new InvalidValue);
+        }
+    }
+
+    return std::unique_ptr<Value>(new StringValue(value));
+}
+
+
+void
+IdValueNode::visit(Visitor &visitor) const
+{
+    visitor.visitIdValueNode(*this);
+}
+
+
+void
+IdValueNode::print(std::ostream& out, bool verbose,
+                   const std::string& indent) const
+{
+    (void) verbose; (void) indent;
+    if (hadParentheses()) out << '(';
+    out << _id;
+    if (_type != ALL) {
+        out << '.' << _typestring;
+    }
+    if (_type == ORDER) {
+        out << "(" << _widthBits << "," << _divisionBits << ")";
+    }
+    if (hadParentheses()) out << ')';
+}
+
+SearchColumnValueNode::SearchColumnValueNode(
+        const BucketIdFactory& bucketIdFactory,
+        const vespalib::stringref & name, int numColumns)
+    : _bucketIdFactory(bucketIdFactory),
+      _id(name),
+      _numColumns(numColumns),
+      _distribution(_numColumns, 16)
+{
+}
+
+int64_t
+SearchColumnValueNode::getValue(const BucketId& id) const
+{
+    return _distribution.getColumn(id);
+}
+
+
+std::unique_ptr<Value>
+SearchColumnValueNode::getValue(const Context& context) const
+{
+    if (context._doc != NULL) {
+        return getValue(context._doc->getId());
+    } else if (context._docId != NULL) {
+        return getValue(*context._docId);
+    } else {
+        return getValue(context._docUpdate->getId());
+    }
+}
+
+
+std::unique_ptr<Value>
+SearchColumnValueNode::getValue(const DocumentId& id) const
+{
+    return std::unique_ptr<Value>(new IntegerValue(
+                getValue(_bucketIdFactory.getBucketId(id)), false));
+}
+
+
+std::unique_ptr<Value>
+SearchColumnValueNode::traceValue(const Context& context,
+                                  std::ostream &out) const
+{
+    if (context._doc != NULL) {
+        return traceValue(context._doc->getId(), out);
+    } else if (context._docId != NULL) {
+        return traceValue(*context._docId, out);
+    } else {
+        return traceValue(context._docUpdate->getId(), out);
+    }
+}
+
+
+std::unique_ptr<Value>
+SearchColumnValueNode::traceValue(const DocumentId& id,
+                                  std::ostream& out) const
+{
+    std::unique_ptr<Value> result(new IntegerValue(
+                getValue(_bucketIdFactory.getBucketId(id)), false));
+    out << "Resolved search column of doc \"" << id << "\" to " << *result
+        << "\n";
+    return result;
+}
+
+
+void
+SearchColumnValueNode::visit(Visitor &visitor) const
+{
+    visitor.visitSearchColumnValueNode(*this);
+}
+
+
+void
+SearchColumnValueNode::print(std::ostream& out, bool verbose,
+                   const std::string& indent) const
+{
+    (void) verbose; (void) indent;
+    if (hadParentheses()) out << '(';
+    out << _id;
+    out << '.' << _numColumns;
+    if (hadParentheses()) out << ')';
+}
+
+namespace {
+    union HashUnion {
+        unsigned char _key[16];
+        int64_t       _hash[2];
+    };
+    int64_t hash(const void* data, uint32_t len) {
+        HashUnion hash;
+        fastc_md5sum((const unsigned char*) data, len, hash._key);
+        return hash._hash[0];
+    }
+}
+
+FunctionValueNode::FunctionValueNode(const vespalib::stringref & name,
+                                     std::unique_ptr<ValueNode> src)
+    : _function(),
+      _funcname(name),
+      _source(std::move(src))
+{
+    if (name == "lowercase") {
+        _function = LOWERCASE;
+    } else if (name == "hash") {
+        _function = HASH;
+    } else if (name == "abs") {
+        _function = ABS;
+    } else {
+        throw ParsingFailedException("No function '"+name+"' exist.",
+                                     VESPA_STRLOC);
+    }
+}
+
+std::unique_ptr<Value>
+FunctionValueNode::getValue(std::unique_ptr<Value> val) const
+{
+    switch (val->getType()) {
+        case Value::String:
+        {
+            StringValue& sval(static_cast<StringValue&>(*val));
+            if (_function == LOWERCASE) {
+                return std::unique_ptr<Value>(new StringValue(
+                    vespalib::LowerCase::convert(sval.getValue())));
+            } else if (_function == HASH) {
+                return std::unique_ptr<Value>(new IntegerValue(
+                    hash(sval.getValue().c_str(), sval.getValue().size()),
+                    false));
+            }
+            break;
+        }
+        case Value::Float:
+        {
+            FloatValue& fval(static_cast<FloatValue&>(*val));
+            if (_function == HASH) {
+                FloatValue::ValueType ffval = fval.getValue();
+                return std::unique_ptr<Value>(new IntegerValue(
+                    hash(&ffval, sizeof(ffval)), false));
+            } else if (_function == ABS) {
+                FloatValue::ValueType ffval = fval.getValue();
+                if (ffval < 0) ffval *= -1;
+                return std::unique_ptr<Value>(new FloatValue(ffval));
+            }
+            break;
+        }
+        case Value::Integer:
+        {
+            IntegerValue& ival(static_cast<IntegerValue&>(*val));
+            if (_function == HASH) {
+                IntegerValue::ValueType iival = ival.getValue();
+                return std::unique_ptr<Value>(new IntegerValue(
+                    hash(&iival, sizeof(iival)), false));
+            } else if (_function == ABS) {
+                IntegerValue::ValueType iival = ival.getValue();
+                if (iival < 0) iival *= -1;
+                return std::unique_ptr<Value>(new IntegerValue(iival, false));
+            }
+            break;
+        }
+        case Value::Bucket:
+        {
+            throw ParsingFailedException(
+                    "No functioncalls are allowed on value of type bucket",
+                    VESPA_STRLOC);
+            break;
+        }
+
+        case Value::Array: break;
+        case Value::Struct: break;
+        case Value::Invalid: break;
+        case Value::Null: break;
+    }
+    return std::unique_ptr<Value>(new InvalidValue);
+}
+
+std::unique_ptr<Value>
+FunctionValueNode::traceValue(std::unique_ptr<Value> val,
+                              std::ostream& out) const
+{
+    switch (val->getType()) {
+        case Value::String:
+        {
+            StringValue& sval(static_cast<StringValue&>(*val));
+            if (_function == LOWERCASE) {
+                std::unique_ptr<Value> result(new StringValue(
+                    vespalib::LowerCase::convert(sval.getValue())));
+                out << "Performed lowercase function on '" << sval
+                    << "' => '" << *result << "'.\n";
+                return result;
+            } else if (_function == HASH) {
+                std::unique_ptr<Value> result(new IntegerValue(
+                    hash(sval.getValue().c_str(), sval.getValue().size()),
+                    false));
+                out << "Performed hash on string '" << sval << "' -> "
+                    << *result << "\n";
+                return result;
+            }
+            break;
+        }
+        case Value::Float:
+        {
+            FloatValue& fval(static_cast<FloatValue&>(*val));
+            if (_function == HASH) {
+                FloatValue::ValueType ffval = fval.getValue();
+                std::unique_ptr<Value> result(new IntegerValue(
+                    hash(&ffval, sizeof(ffval)), false));
+                out << "Performed hash on float " << ffval << " -> " << *result
+                    << "\n";
+                return result;
+            } else if (_function == ABS) {
+                FloatValue::ValueType ffval = fval.getValue();
+                if (ffval < 0) ffval *= -1;
+                out << "Performed abs on float " << fval.getValue() << " -> "
+                    << ffval << "\n";
+                return std::unique_ptr<Value>(new FloatValue(ffval));
+            }
+            break;
+        }
+        case Value::Integer:
+        {
+            IntegerValue& ival(static_cast<IntegerValue&>(*val));
+            if (_function == HASH) {
+                IntegerValue::ValueType iival = ival.getValue();
+                std::unique_ptr<Value> result(new IntegerValue(
+                    hash(&iival, sizeof(iival)), false));
+                out << "Performed hash on float " << iival << " -> " << *result
+                    << "\n";
+                return result;
+            } else if (_function == ABS) {
+                IntegerValue::ValueType iival = ival.getValue();
+                if (iival < 0) iival *= -1;
+                out << "Performed abs on integer " << ival.getValue() << " -> "
+                    << iival << "\n";
+                return std::unique_ptr<Value>(new IntegerValue(iival, false));
+            }
+            break;
+        }
+        case Value::Bucket: break;
+        case Value::Array: break;
+        case Value::Struct: break;
+        case Value::Invalid: break;
+        case Value::Null: break;
+    }
+    out << "Cannot use function " << _function << " on a value of type "
+        << val->getType() << ". Resolving invalid.\n";
+    return std::unique_ptr<Value>(new InvalidValue);
+}
+
+
+void
+FunctionValueNode::visit(Visitor &visitor) const
+{
+    visitor.visitFunctionValueNode(*this);
+}
+
+
+void
+FunctionValueNode::print(std::ostream& out, bool verbose,
+                         const std::string& indent) const
+{
+    if (hadParentheses()) out << '(';
+    _source->print(out, verbose, indent);
+    out << '.' << _funcname << "()";
+    if (hadParentheses()) out << ')';
+}
+
+ArithmeticValueNode::ArithmeticValueNode(
+        std::unique_ptr<ValueNode> left, const vespalib::stringref & op,
+        std::unique_ptr<ValueNode> right)
+    : _operator(),
+      _left(std::move(left)),
+      _right(std::move(right))
+{
+    if (op.size() == 1) switch (op[0]) {
+        case '+': _operator = ADD; return;
+        case '-': _operator = SUB; return;
+        case '*': _operator = MUL; return;
+        case '/': _operator = DIV; return;
+        case '%': _operator = MOD; return;
+    }
+    throw ParsingFailedException(
+            "Arithmetic operator '"+op+"' does not exist.", VESPA_STRLOC);
+}
+
+const char*
+ArithmeticValueNode::getOperatorName() const
+{
+    switch (_operator) {
+        case ADD: return "+";
+        case SUB: return "-";
+        case MUL: return "*";
+        case DIV: return "/";
+        case MOD: return "%";
+    }
+    return "UNKNOWN";
+}
+
+
+
+std::unique_ptr<Value>
+ArithmeticValueNode::getValue(std::unique_ptr<Value> lval,
+                              std::unique_ptr<Value> rval) const
+{
+    switch (_operator) {
+        case ADD:
+        {
+            if (lval->getType() == Value::String &&
+                rval->getType() == Value::String)
+            {
+                StringValue& slval(static_cast<StringValue&>(*lval));
+                StringValue& srval(static_cast<StringValue&>(*rval));
+                return std::unique_ptr<Value>(new StringValue(
+                            slval.getValue() + srval.getValue()));
+            }
+        }
+        case SUB:
+        case MUL:
+        case DIV:
+        {
+            if (lval->getType() == Value::Integer &&
+                rval->getType() == Value::Integer)
+            {
+                IntegerValue& ilval(static_cast<IntegerValue&>(*lval));
+                IntegerValue& irval(static_cast<IntegerValue&>(*rval));
+                IntegerValue::ValueType res = 0;
+                switch (_operator) {
+                    case ADD: res = ilval.getValue() + irval.getValue(); break;
+                    case SUB: res = ilval.getValue() - irval.getValue(); break;
+                    case MUL: res = ilval.getValue() * irval.getValue(); break;
+                    case DIV:
+                        if (irval.getValue() != 0) {
+                            res = ilval.getValue() / irval.getValue();
+                        } else {
+                            throw vespalib::IllegalArgumentException("Division by zero");
+                        }
+                        break;
+                    case MOD: assert(0);
+                }
+                return std::unique_ptr<Value>(new IntegerValue(res, false));
+            }
+            NumberValue* nlval(dynamic_cast<NumberValue*>(lval.get()));
+            NumberValue* nrval(dynamic_cast<NumberValue*>(rval.get()));
+            if (nlval != 0 && nrval != 0) {
+                NumberValue::CommonValueType res = 0;
+                switch (_operator) {
+                    case ADD: res = nlval->getCommonValue()
+                                  + nrval->getCommonValue(); break;
+                    case SUB: res = nlval->getCommonValue()
+                                  - nrval->getCommonValue(); break;
+                    case MUL: res = nlval->getCommonValue()
+                                  * nrval->getCommonValue(); break;
+                    case DIV:
+                        if (nrval->getCommonValue() != 0) {
+                            res = nlval->getCommonValue()
+                                  / nrval->getCommonValue();
+                        } else {
+                            throw vespalib::IllegalArgumentException("Division by zero");
+                        }
+                        break;
+                    case MOD: assert(0);
+                }
+                return std::unique_ptr<Value>(new FloatValue(res));
+            }
+        }
+        case MOD:
+        {
+            if (lval->getType() == Value::Integer &&
+                rval->getType() == Value::Integer)
+            {
+                IntegerValue& ilval(static_cast<IntegerValue&>(*lval));
+                IntegerValue& irval(static_cast<IntegerValue&>(*rval));
+                if (irval.getValue() != 0) {
+                    return std::unique_ptr<Value>(new IntegerValue(ilval.getValue() % irval.getValue(), false));
+                } else {
+                    throw vespalib::IllegalArgumentException("Division by zero");
+                }
+            }
+        }
+    }
+    return std::unique_ptr<Value>(new InvalidValue);
+}
+
+std::unique_ptr<Value>
+ArithmeticValueNode::traceValue(std::unique_ptr<Value> lval,
+                                std::unique_ptr<Value> rval,
+                                std::ostream& out) const
+{
+    switch (_operator) {
+        case ADD:
+        {
+            if (lval->getType() == Value::String &&
+                rval->getType() == Value::String)
+            {
+                StringValue& slval(static_cast<StringValue&>(*lval));
+                StringValue& srval(static_cast<StringValue&>(*rval));
+                std::unique_ptr<Value> result(new StringValue(
+                            slval.getValue() + srval.getValue()));
+                out << "Appended strings '" << slval << "' + '" << srval
+                    << "' -> '" << *result << "'.\n";
+                return result;
+            }
+        }
+        case SUB:
+        case MUL:
+        case DIV:
+        {
+            if (lval->getType() == Value::Integer &&
+                rval->getType() == Value::Integer)
+            {
+                IntegerValue& ilval(static_cast<IntegerValue&>(*lval));
+                IntegerValue& irval(static_cast<IntegerValue&>(*rval));
+                IntegerValue::ValueType res = 0;
+                switch (_operator) {
+                    case ADD: res = ilval.getValue() + irval.getValue(); break;
+                    case SUB: res = ilval.getValue() - irval.getValue(); break;
+                    case MUL: res = ilval.getValue() * irval.getValue(); break;
+                    case DIV: res = ilval.getValue() / irval.getValue(); break;
+                    case MOD: assert(0);
+                }
+                std::unique_ptr<Value> result(new IntegerValue(res, false));
+                out << "Performed integer operation " << ilval << " "
+                    << getOperatorName() << " " << irval << " = " << *result
+                    << "\n";
+                return result;
+            }
+            NumberValue* nlval(dynamic_cast<NumberValue*>(lval.get()));
+            NumberValue* nrval(dynamic_cast<NumberValue*>(lval.get()));
+            if (nlval != 0 && nrval != 0) {
+                NumberValue::CommonValueType res = 0;
+                switch (_operator) {
+                    case ADD: res = nlval->getCommonValue()
+                                  + nrval->getCommonValue(); break;
+                    case SUB: res = nlval->getCommonValue()
+                                  - nrval->getCommonValue(); break;
+                    case MUL: res = nlval->getCommonValue()
+                                  * nrval->getCommonValue(); break;
+                    case DIV: res = nlval->getCommonValue()
+                                  / nrval->getCommonValue(); break;
+                    case MOD: assert(0);
+                }
+                std::unique_ptr<Value> result(new FloatValue(res));
+                out << "Performed float operation " << nlval << " "
+                    << getOperatorName() << " " << nrval << " = " << *result
+                    << "\n";
+                return result;
+            }
+        }
+        case MOD:
+        {
+            if (lval->getType() == Value::Integer &&
+                rval->getType() == Value::Integer)
+            {
+                IntegerValue& ilval(static_cast<IntegerValue&>(*lval));
+                IntegerValue& irval(static_cast<IntegerValue&>(*rval));
+                std::unique_ptr<Value> result(new IntegerValue(
+                            ilval.getValue() % irval.getValue(), false));
+                out << "Performed integer operation " << ilval << " "
+                    << getOperatorName() << " " << irval << " = " << *result
+                    << "\n";
+                return result;
+            }
+        }
+    }
+    out << "Failed to do operation " << getOperatorName()
+        << " on values of type " << lval->getType() << " and "
+        << rval->getType() << ". Resolving invalid.\n";
+    return std::unique_ptr<Value>(new InvalidValue);
+}
+
+
+void
+ArithmeticValueNode::visit(Visitor &visitor) const
+{
+    visitor.visitArithmeticValueNode(*this);
+}
+
+
+void
+ArithmeticValueNode::print(std::ostream& out, bool verbose,
+                           const std::string& indent) const
+{
+    if (hadParentheses()) out << '(';
+    _left->print(out, verbose, indent);
+    switch (_operator) {
+        case ADD: out << " + "; break;
+        case SUB: out << " - "; break;
+        case MUL: out << " * "; break;
+        case DIV: out << " / "; break;
+        case MOD: out << " % "; break;
+    }
+    _right->print(out, verbose, indent);
+    if (hadParentheses()) out << ')';
+}
+
+} // select
+} // document
diff --git a/document/src/vespa/document/select/valuenode.h b/document/src/vespa/document/select/valuenode.h
new file mode 100644
index 00000000000..37ccc26e3a3
--- /dev/null
+++ b/document/src/vespa/document/select/valuenode.h
@@ -0,0 +1,479 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * @class document::select::ValueNode
+ * @ingroup select
+ *
+ * @brief Node representing a value in the tree
+ *
+ * @author H�kon Humberset
+ * @date 2007-04-20
+ * @version $Id$
+ */
+
+#pragma once
+
+#include <memory>
+#include <vespa/document/bucket/bucketidfactory.h>
+#include "value.h"
+#include <vespa/document/fieldvalue/document.h>
+#include <vespa/document/update/documentupdate.h>
+#include <vespa/document/bucket/bucketdistribution.h>
+#include <vespa/document/fieldvalue/fieldvalue.h>
+#include "context.h"
+
+namespace document {
+
+class Document;
+class Field;
+
+namespace select {
+
+class ValueNode : public Printable
+{
+public:
+    typedef std::unique_ptr<ValueNode> UP;
+    typedef Context::VariableMap VariableMap;
+
+    ValueNode() : _parentheses(false) {}
+    virtual ~ValueNode() {}
+
+    void setParentheses() { _parentheses = true; }
+
+    void clearParentheses()
+    {
+        _parentheses = false;
+    }
+
+    bool hadParentheses() const { return _parentheses; }
+
+    virtual std::unique_ptr<Value>
+    getValue(const Context& context) const  = 0;
+
+    virtual std::unique_ptr<Value>
+    traceValue(const Context &context,
+               std::ostream &out) const {
+        return defaultTrace(getValue(context), out);
+    }
+
+    virtual void
+    print(std::ostream& out, bool verbose,
+          const std::string& indent) const = 0;
+
+    virtual void visit(Visitor&) const = 0;
+
+    virtual ValueNode::UP clone() const = 0;
+private:
+    bool _parentheses; // Set to true if parentheses was used around this part
+                       // Set such that we can recreate original query in print.
+protected:
+    ValueNode::UP wrapParens(ValueNode* node) const {
+        ValueNode::UP ret(node);
+        if (_parentheses) {
+            ret->setParentheses();
+        }
+        return ret;
+    }
+
+    std::unique_ptr<Value> defaultTrace(std::unique_ptr<Value> val,
+                                      std::ostream& out) const
+    {
+        out << "Returning value " << *val << ".\n";
+        return std::move(val);
+    }
+};
+
+class InvalidValueNode : public ValueNode
+{
+    vespalib::string _name;
+
+public:
+    InvalidValueNode(const vespalib::stringref & name);
+
+    virtual std::unique_ptr<Value>
+    getValue(const Context&) const
+    {
+        return std::unique_ptr<Value>(new InvalidValue());
+    }
+
+    virtual void
+    print(std::ostream& out, bool verbose,
+          const std::string& indent) const;
+
+    virtual void visit(Visitor& visitor) const;
+
+    ValueNode::UP clone() const {
+        return wrapParens(new InvalidValueNode(_name));
+    }
+};
+
+class NullValueNode : public ValueNode
+{
+    vespalib::string _name;
+
+public:
+    NullValueNode(const vespalib::stringref & name);
+
+    virtual std::unique_ptr<Value> getValue(const Context&) const
+        { return std::unique_ptr<Value>(new NullValue()); }
+
+    virtual void print(std::ostream& out, bool verbose,
+                       const std::string& indent) const;
+
+    virtual void visit(Visitor& visitor) const;
+
+    ValueNode::UP clone() const {
+        return wrapParens(new NullValueNode(_name));
+    }
+};
+
+class StringValueNode : public ValueNode
+{
+    vespalib::string _value;
+public:
+    StringValueNode(const vespalib::stringref & val);
+
+    const vespalib::string& getValue() const { return _value; }
+
+    virtual std::unique_ptr<Value> getValue(const Context&) const
+        { return std::unique_ptr<Value>(new StringValue(_value)); }
+
+    virtual void print(std::ostream& out, bool verbose,
+                       const std::string& indent) const;
+
+    virtual void visit(Visitor& visitor) const;
+
+    ValueNode::UP clone() const {
+        return wrapParens(new StringValueNode(_value));
+    }
+};
+
+class IntegerValueNode : public ValueNode
+{
+    int64_t _value;
+    bool _isBucketValue;
+public:
+    IntegerValueNode(int64_t val, bool isBucketValue)
+        : _value(val), _isBucketValue(isBucketValue) {}
+
+    int64_t getValue() const { return _value; }
+
+    virtual std::unique_ptr<Value> getValue(const Context&) const {
+        return std::unique_ptr<Value>(new IntegerValue(_value, _isBucketValue));
+    }
+
+    virtual void print(std::ostream& out, bool verbose,
+                       const std::string& indent) const;
+
+    virtual void visit(Visitor& visitor) const;
+
+    ValueNode::UP clone() const {
+        return wrapParens(new IntegerValueNode(_value,
+                                               _isBucketValue));
+    }
+};
+
+class CurrentTimeValueNode : public ValueNode
+{
+public:
+    int64_t getValue() const;
+
+    virtual std::unique_ptr<Value> getValue(const Context&) const {
+        return std::unique_ptr<Value>(new IntegerValue(getValue(), false));
+    }
+
+    virtual void print(std::ostream& out, bool verbose,
+                       const std::string& indent) const;
+
+    virtual void visit(Visitor& visitor) const;
+
+    ValueNode::UP clone() const {
+        return wrapParens(new CurrentTimeValueNode);
+    }
+};
+
+class VariableValueNode : public ValueNode
+{
+    vespalib::string _value;
+public:
+    VariableValueNode(const vespalib::string & variableName) : _value(variableName) {}
+
+    const vespalib::string& getVariableName() const { return _value; }
+
+    virtual std::unique_ptr<Value> getValue(const Context& context) const;
+
+    virtual void print(std::ostream& out, bool verbose,
+                       const std::string& indent) const;
+
+    virtual void visit(Visitor& visitor) const;
+
+    ValueNode::UP clone() const {
+        return wrapParens(new VariableValueNode(_value));
+    }
+};
+
+class FloatValueNode : public ValueNode
+{
+    double _value;
+public:
+    FloatValueNode(double val) : _value(val) {}
+
+    double getValue() const { return _value; }
+
+    virtual std::unique_ptr<Value> getValue(const Context&) const
+        { return std::unique_ptr<Value>(new FloatValue(_value)); }
+
+    virtual void print(std::ostream& out, bool verbose,
+                       const std::string& indent) const;
+
+    virtual void visit(Visitor& visitor) const;
+
+    ValueNode::UP clone() const {
+        return wrapParens(new FloatValueNode(_value));
+    }
+};
+
+class FieldValueNode : public ValueNode
+{
+    vespalib::string _doctype;
+    vespalib::string _field;
+    mutable FieldPath _fieldPath;
+
+public:
+    FieldValueNode(const vespalib::string& doctype,
+                   const vespalib::string& field);
+
+    const vespalib::string& getDocType() const { return _doctype; }
+
+    const vespalib::string& getFieldName() const { return _field; }
+
+    virtual std::unique_ptr<Value> getValue(const Context& context) const;
+
+    virtual std::unique_ptr<Value> traceValue(const Context &context,
+                                            std::ostream& out) const;
+
+    virtual void print(std::ostream& out, bool verbose,
+                       const std::string& indent) const;
+
+    virtual void visit(Visitor& visitor) const;
+
+    ValueNode::UP clone() const {
+        return wrapParens(new FieldValueNode(_doctype, _field));
+    }
+
+private:
+    class IteratorHandler : public FieldValue::IteratorHandler
+    {
+    public:
+        bool hasSingleValue() const;
+
+        std::unique_ptr<Value> getSingleValue();
+        const std::vector<ArrayValue::VariableValue>& getValues();
+
+    private:
+        std::unique_ptr<Value> _firstValue;
+        std::vector<ArrayValue::VariableValue> _values;
+
+        virtual void onPrimitive(const Content & fv);
+        std::unique_ptr<Value> getInternalValue(const FieldValue& fval) const;
+    };
+
+    void initFieldPath(const DocumentType&) const;
+};
+
+class IdValueNode : public ValueNode
+{
+public:
+    enum Type { SCHEME, NS, TYPE, USER, GROUP, GID, SPEC, BUCKET, ORDER, ALL };
+
+    IdValueNode(const BucketIdFactory& bucketIdFactory,
+                const vespalib::stringref & name, const vespalib::stringref & type,
+                int widthBits = -1, int divisionBits = -1);
+
+    Type getType() const { return _type; }
+
+    virtual std::unique_ptr<Value>
+    getValue(const Context& context) const;
+
+    std::unique_ptr<Value>
+    getValue(const DocumentId& id) const;
+
+    virtual std::unique_ptr<Value>
+    traceValue(const Context& context,
+               std::ostream &out) const;
+
+    std::unique_ptr<Value>
+    traceValue(const DocumentId& val,
+               std::ostream& out) const;
+
+    virtual void print(std::ostream& out, bool verbose,
+                       const std::string& indent) const;
+
+    virtual void visit(Visitor& visitor) const;
+
+    ValueNode::UP clone() const {
+        return wrapParens(new IdValueNode(_bucketIdFactory,
+                                          _id,
+                                          _typestring,
+                                          _widthBits,
+                                          _divisionBits));
+    }
+
+    int getWidthBits() const { return _widthBits; }
+    int getDivisionBits() const { return _divisionBits; }
+
+private:
+    const BucketIdFactory& _bucketIdFactory;
+    vespalib::string _id;
+    vespalib::string _typestring;
+    Type _type;
+    int _widthBits;
+    int _divisionBits;
+};
+
+class SearchColumnValueNode : public ValueNode
+{
+public:
+    SearchColumnValueNode(const BucketIdFactory& bucketIdFactory,
+                          const vespalib::stringref & name,
+                          int numColumns);
+
+    int getColumns() { return _numColumns; }
+
+    virtual std::unique_ptr<Value>
+    getValue(const Context& context) const;
+    
+    std::unique_ptr<Value>
+    getValue(const DocumentId& id) const;
+
+    virtual std::unique_ptr<Value>
+    traceValue(const Context& context,
+               std::ostream &out) const;
+
+    std::unique_ptr<Value>
+    traceValue(const DocumentId& val,
+               std::ostream& out) const;
+    
+    int64_t getValue(const BucketId& bucketId) const;
+
+    virtual void print(std::ostream& out, bool verbose,
+                       const std::string& indent) const;
+
+    virtual void visit(Visitor& visitor) const;
+
+    ValueNode::UP clone() const {
+        return wrapParens(new SearchColumnValueNode(_bucketIdFactory,
+                                                    _id,
+                                                    _numColumns));
+}
+
+private:
+    const BucketIdFactory& _bucketIdFactory;
+    vespalib::string _id;
+    int _numColumns;
+    BucketDistribution _distribution;
+};
+
+class FunctionValueNode : public ValueNode
+{
+public:
+    enum Function { LOWERCASE, HASH, ABS };
+
+    FunctionValueNode(const vespalib::stringref & name, std::unique_ptr<ValueNode> src);
+
+    Function getFunction() const { return _function; }
+
+    const vespalib::string &
+    getFunctionName(void) const
+    {
+        return _funcname;
+    }
+
+    virtual std::unique_ptr<Value>
+    getValue(const Context& context) const
+    {
+        return getValue(_source->getValue(context));
+    }
+
+    virtual std::unique_ptr<Value> traceValue(const Context &context,
+                                            std::ostream& out) const
+    {
+        return traceValue(_source->getValue(context), out);
+    }
+
+    virtual void print(std::ostream& out, bool verbose,
+                       const std::string& indent) const;
+
+    virtual void visit(Visitor& visitor) const;
+
+    ValueNode::UP clone() const {
+        return wrapParens(new FunctionValueNode(_funcname,
+                                                _source->clone()));
+    }
+
+    const ValueNode& getChild() const { return *_source; }
+
+private:
+    Function _function;
+    vespalib::string _funcname;
+    std::unique_ptr<ValueNode> _source;
+
+    virtual std::unique_ptr<Value> getValue(std::unique_ptr<Value> val) const;
+    virtual std::unique_ptr<Value> traceValue(std::unique_ptr<Value> val,
+                                            std::ostream& out) const;
+};
+
+class ArithmeticValueNode : public ValueNode
+{
+public:
+    enum Operator { ADD, SUB, MUL, DIV, MOD };
+
+    ArithmeticValueNode(std::unique_ptr<ValueNode> left,
+                        const vespalib::stringref & op,
+                        std::unique_ptr<ValueNode> right);
+
+    Operator getOperator() const { return _operator; }
+    const char* getOperatorName() const;
+
+    virtual std::unique_ptr<Value>
+    getValue(const Context& context) const
+    {
+        return getValue(_left->getValue(context), _right->getValue(context));
+    }
+    
+    virtual std::unique_ptr<Value>
+    traceValue(const Context &context,
+               std::ostream& out) const
+    {
+        return traceValue(_left->getValue(context),
+                          _right->getValue(context), out);
+    }
+
+    virtual void print(std::ostream& out, bool verbose,
+                       const std::string& indent) const;
+
+    virtual void visit(Visitor& visitor) const;
+
+    ValueNode::UP clone() const {
+        return wrapParens(new ArithmeticValueNode(_left->clone(),
+                                                  getOperatorName(),
+                                                  _right->clone()));
+    }
+
+    const ValueNode& getLeft() const { return *_left; }
+    const ValueNode& getRight() const { return *_right; }
+
+private:
+    Operator _operator;
+    std::unique_ptr<ValueNode> _left;
+    std::unique_ptr<ValueNode> _right;
+
+    virtual std::unique_ptr<Value> getValue(std::unique_ptr<Value> lval,
+                                          std::unique_ptr<Value> rval) const;
+    virtual std::unique_ptr<Value> traceValue(std::unique_ptr<Value> lval,
+                                            std::unique_ptr<Value> rval,
+                                            std::ostream&) const;
+};
+
+} // select
+} // document
+
diff --git a/document/src/vespa/document/select/visitor.h b/document/src/vespa/document/select/visitor.h
new file mode 100644
index 00000000000..2922a55d721
--- /dev/null
+++ b/document/src/vespa/document/select/visitor.h
@@ -0,0 +1,102 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * @class document::select::Visitor
+ * @ingroup select
+ *
+ * @brief Visitor class for going through the select tree.
+ *
+ * @author H�kon Humberset
+ * @date 2005-06-07
+ * @version $Id$
+ */
+
+#pragma once
+
+namespace document {
+namespace select {
+
+class And;
+class Compare;
+class Constant;
+class DocType;
+class Not;
+class Or;
+class ArithmeticValueNode;
+class FunctionValueNode;
+class IdValueNode;
+class InvalidConstant;
+class SearchColumnValueNode;
+class FieldValueNode;
+class FloatValueNode;
+class IntegerValueNode;
+class CurrentTimeValueNode;
+class StringValueNode;
+class NullValueNode;
+class InvalidValueNode;
+class VariableValueNode;
+
+class Visitor {
+public:
+    virtual ~Visitor() {}
+
+    virtual void
+    visitAndBranch(const And &) = 0;
+
+    virtual void
+    visitComparison(const Compare &) = 0;
+
+    virtual void
+    visitConstant(const Constant &) = 0;
+
+    virtual void
+    visitInvalidConstant(const InvalidConstant &) = 0;
+
+    virtual void
+    visitDocumentType(const DocType &) = 0;
+
+    virtual void
+    visitNotBranch(const Not &) = 0;
+
+    virtual void
+    visitOrBranch(const Or &) = 0;
+
+    virtual void
+    visitArithmeticValueNode(const ArithmeticValueNode &) = 0;
+
+    virtual void
+    visitFunctionValueNode(const FunctionValueNode &) = 0;
+
+    virtual void
+    visitIdValueNode(const IdValueNode &) = 0;
+
+    virtual void
+    visitSearchColumnValueNode(const SearchColumnValueNode &) = 0;
+
+    virtual void
+    visitFieldValueNode(const FieldValueNode &) = 0;
+
+    virtual void
+    visitFloatValueNode(const FloatValueNode &) = 0;
+
+    virtual void
+    visitVariableValueNode(const VariableValueNode &) = 0;
+
+    virtual void
+    visitIntegerValueNode(const IntegerValueNode &) = 0;
+
+    virtual void
+    visitCurrentTimeValueNode(const CurrentTimeValueNode &) = 0;
+
+    virtual void
+    visitStringValueNode(const StringValueNode &) = 0;
+
+    virtual void
+    visitNullValueNode(const NullValueNode &) = 0;
+
+    virtual void
+    visitInvalidValueNode(const InvalidValueNode &) = 0;
+};
+
+} // select
+} // document
+
diff --git a/document/src/vespa/document/serialization/.gitignore b/document/src/vespa/document/serialization/.gitignore
new file mode 100644
index 00000000000..583460ae288
--- /dev/null
+++ b/document/src/vespa/document/serialization/.gitignore
@@ -0,0 +1,3 @@
+*.So
+.depend
+Makefile
diff --git a/document/src/vespa/document/serialization/CMakeLists.txt b/document/src/vespa/document/serialization/CMakeLists.txt
new file mode 100644
index 00000000000..e17e4a735e4
--- /dev/null
+++ b/document/src/vespa/document/serialization/CMakeLists.txt
@@ -0,0 +1,11 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_library(document_serialization OBJECT
+    SOURCES
+    annotationdeserializer.cpp
+    annotationserializer.cpp
+    vespadocumentserializer.cpp
+    vespadocumentdeserializer.cpp
+    DEPENDS
+    AFTER
+    document_documentconfig
+)
diff --git a/document/src/vespa/document/serialization/annotationdeserializer.cpp b/document/src/vespa/document/serialization/annotationdeserializer.cpp
new file mode 100644
index 00000000000..1bc88684799
--- /dev/null
+++ b/document/src/vespa/document/serialization/annotationdeserializer.cpp
@@ -0,0 +1,149 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".annotationdeserializer");
+
+#include "annotationdeserializer.h"
+
+#include "vespadocumentdeserializer.h"
+#include <vespa/document/annotation/alternatespanlist.h>
+#include <vespa/document/annotation/annotation.h>
+#include <vespa/document/annotation/spanlist.h>
+#include <vespa/document/annotation/spantree.h>
+#include <vespa/document/base/exceptions.h>
+#include <vespa/document/fieldvalue/stringfieldvalue.h>
+#include <vespa/document/repo/fixedtyperepo.h>
+
+using std::unique_ptr;
+
+namespace document {
+
+AnnotationDeserializer::AnnotationDeserializer(const FixedTypeRepo &repo,
+                                               vespalib::nbostream &stream,
+                                               uint16_t version)
+    : _repo(repo),
+      _stream(stream),
+      _version(version),
+      _nodes() {
+}
+
+unique_ptr<SpanTree> AnnotationDeserializer::readSpanTree() {
+    VespaDocumentDeserializer deserializer(_repo, _stream, _version);
+
+    StringFieldValue tree_name;
+    deserializer.read(tree_name);
+    _nodes.clear();
+    SpanNode::UP root = readSpanNode();
+    unique_ptr<SpanTree> span_tree(new SpanTree(tree_name.getValue(), std::move(root)));
+
+    uint32_t annotation_count = getInt1_2_4Bytes(_stream);
+    span_tree->reserveAnnotations(annotation_count);
+    for (uint32_t i = 0; i < annotation_count; ++i) {
+        readAnnotation(span_tree->annotation(i));
+    }
+
+    return span_tree;
+}
+
+unique_ptr<SpanNode> AnnotationDeserializer::readSpanNode() {
+    uint8_t type = readValue<uint8_t>(_stream);
+    unique_ptr<SpanNode> node;
+    size_t node_index = _nodes.size();
+    _nodes.push_back(0);
+    if (type == 1) {  // Span.ID
+        Span * span = new Span();
+        node.reset(span);
+        readSpan(*span);
+    } else if (type == 2) {  // SpanList.ID
+        node = readSimpleSpanList();
+        if (node.get() == nullptr) {
+            node = readSpanList();
+        }
+    } else if (type == 4) {  // AlternateSpanList.ID
+        node = readAlternateSpanList();
+    } else {
+        LOG(warning, "Cannot read SpanNode of type %u.", type);
+    }
+    _nodes[node_index] = node.get();
+    return node;
+}
+
+unique_ptr<SpanList> AnnotationDeserializer::readSpanList() {
+    uint32_t child_count = getInt1_2_4Bytes(_stream);
+    unique_ptr<SpanList> span_list(new SpanList);
+    span_list->reserve(child_count);
+    _nodes.reserve(vespalib::roundUp2inN(_nodes.size() + child_count));
+    for (uint32_t i = 0; i < child_count; ++i) {
+        span_list->add(readSpanNode());
+    }
+    return span_list;
+}
+
+unique_ptr<SimpleSpanList> AnnotationDeserializer::readSimpleSpanList() {
+    size_t pos = _stream.rp();
+    uint32_t child_count = getInt1_2_4Bytes(_stream);
+    unique_ptr<SimpleSpanList> span_list(new SimpleSpanList(child_count));
+    _nodes.reserve(vespalib::roundUp2inN(_nodes.size() + child_count));
+    for (uint32_t i = 0; i < child_count; ++i) {
+        uint8_t type = readValue<uint8_t>(_stream);
+        if (type != 1) {
+            _stream.rp(pos);
+            return unique_ptr<SimpleSpanList>();
+        }
+        readSpan((*span_list)[i]);
+    }
+    for (uint32_t i = 0; i < child_count; ++i) {
+        _nodes.push_back(&(*span_list)[i]);
+    }
+    return span_list;
+}
+
+void AnnotationDeserializer::readAnnotation(Annotation & annotation) {
+    uint32_t type_id = readValue<uint32_t>(_stream);
+    uint8_t features = readValue<uint8_t>(_stream);
+    uint32_t size = getInt1_2_4Bytes(_stream);
+
+    const AnnotationType *type = _repo.getAnnotationType(type_id);
+    if (!type) {
+//        LOG(warning, "Skipping unknown annotation of type %u", type_id);
+        _stream.adjustReadPos(size);
+        return;
+    }
+    annotation.setType(type);
+
+    SpanNode *span_node = 0;
+    if (features & 1) {  // has span node
+        uint32_t span_node_id = getInt1_2_4Bytes(_stream);
+        span_node = _nodes[span_node_id];
+    }
+    if (features & 2) {  // has value
+        uint32_t data_type_id = readValue<uint32_t>(_stream);
+
+        const DataType *data_type = _repo.getDataType(data_type_id);
+        if (!data_type) {
+            LOG(warning, "Unknown data type %d", data_type_id);
+            _stream.adjustReadPos(size - sizeof(uint32_t));
+        } else {
+            FieldValue::UP value(data_type->createFieldValue());
+            VespaDocumentDeserializer deserializer(_repo, _stream, _version);
+            deserializer.read(*value);
+            annotation.setFieldValue(std::move(value));
+        }
+    }
+    if (span_node) {
+        annotation.setSpanNode(*span_node);
+    }
+}
+
+unique_ptr<AlternateSpanList> AnnotationDeserializer::readAlternateSpanList() {
+    unique_ptr<AlternateSpanList> span_list(new AlternateSpanList);
+    uint32_t tree_count = getInt1_2_4Bytes(_stream);
+    for (uint32_t i = 0; i < tree_count; ++i) {
+        span_list->setProbability(i, readValue<double>(_stream));
+        span_list->setSubtree(i, readSpanList());
+    }
+    return span_list;
+}
+
+}  // namespace document
diff --git a/document/src/vespa/document/serialization/annotationdeserializer.h b/document/src/vespa/document/serialization/annotationdeserializer.h
new file mode 100644
index 00000000000..450dc972126
--- /dev/null
+++ b/document/src/vespa/document/serialization/annotationdeserializer.h
@@ -0,0 +1,40 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/vespalib/objects/nbostream.h>
+#include <vespa/document/annotation/span.h>
+#include <vespa/document/serialization/util.h>
+
+namespace document {
+class AlternateSpanList;
+class Annotation;
+class FixedTypeRepo;
+class SpanList;
+class SpanTree;
+
+class AnnotationDeserializer {
+public:
+    AnnotationDeserializer(const FixedTypeRepo &repo, vespalib::nbostream &stream,
+                           uint16_t version);
+
+    std::unique_ptr<SpanTree> readSpanTree();
+    std::unique_ptr<SpanNode> readSpanNode();
+    // returns 0 if the annotation type is unknown.
+    std::unique_ptr<AlternateSpanList> readAlternateSpanList();
+    void readAnnotation(Annotation & annotation);
+private:
+    std::unique_ptr<SpanList> readSpanList();
+    std::unique_ptr<SimpleSpanList> readSimpleSpanList();
+    void readSpan(Span & span) {
+        span.from(getInt1_2_4Bytes(_stream));
+        span.length(getInt1_2_4Bytes(_stream));
+    }
+
+    const FixedTypeRepo    &_repo;
+    vespalib::nbostream    &_stream;
+    uint16_t                _version;
+    std::vector<SpanNode *> _nodes;
+};
+}  // namespace document
+
diff --git a/document/src/vespa/document/serialization/annotationserializer.cpp b/document/src/vespa/document/serialization/annotationserializer.cpp
new file mode 100644
index 00000000000..16fc0ffe5df
--- /dev/null
+++ b/document/src/vespa/document/serialization/annotationserializer.cpp
@@ -0,0 +1,117 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".annotationserializer");
+
+#include "annotationserializer.h"
+
+#include "util.h"
+#include "vespadocumentserializer.h"
+#include <vespa/document/annotation/alternatespanlist.h>
+#include <vespa/document/annotation/annotation.h>
+#include <vespa/document/annotation/span.h>
+#include <vespa/document/annotation/spanlist.h>
+#include <vespa/document/annotation/spannode.h>
+#include <vespa/document/annotation/spantree.h>
+#include <vespa/document/fieldvalue/stringfieldvalue.h>
+#include <vespa/vespalib/objects/nbostream.h>
+
+using vespalib::nbostream;
+
+namespace document {
+
+AnnotationSerializer::AnnotationSerializer(nbostream &stream)
+    : _stream(stream),
+      _span_node_map() {
+}
+
+void AnnotationSerializer::write(const SpanTree &tree) {
+    _span_node_map.clear();
+    StringFieldValue name(tree.getName());
+    VespaDocumentSerializer serializer(_stream);
+    serializer.write(name);
+    write(tree.getRoot());
+    putInt1_2_4Bytes(_stream, tree.numAnnotations());
+    for (const Annotation & a : tree) {
+        write(a);
+    }
+}
+
+void AnnotationSerializer::write(const SpanNode &node) {
+    size_t node_id = _span_node_map.size();
+    _span_node_map[&node] = node_id;
+    node.accept(*this);
+}
+
+void AnnotationSerializer::writeSpan(const Span &node) {
+    _stream << static_cast<uint8_t>(1);  // Span.ID
+    putInt1_2_4Bytes(_stream, node.from());
+    putInt1_2_4Bytes(_stream, node.length());
+}
+
+namespace {
+void writeSpanList(const SpanList &list, nbostream &stream,
+                   AnnotationSerializer &serializer) {
+    putInt1_2_4Bytes(stream, list.size());
+    for (const SpanNode * node : list) {
+        serializer.write(*node);
+    }
+}
+void writeSpanList(const SimpleSpanList &list, nbostream &stream,
+                   AnnotationSerializer &serializer) {
+    putInt1_2_4Bytes(stream, list.size());
+    for (const SpanNode & node : list) {
+        serializer.write(node);
+    }
+}
+}  // namespace
+
+void AnnotationSerializer::writeList(const SpanList &list) {
+    _stream << static_cast<uint8_t>(2);  // SpanList.ID
+    writeSpanList(list, _stream, *this);
+}
+
+void AnnotationSerializer::writeList(const SimpleSpanList &list) {
+    _stream << static_cast<uint8_t>(2);  // SpanList.ID
+    writeSpanList(list, _stream, *this);
+}
+
+void AnnotationSerializer::writeList(const AlternateSpanList &list) {
+    _stream << static_cast<uint8_t>(4);  // AlternateSpanList.ID
+    putInt1_2_4Bytes(_stream, list.getNumSubtrees());
+    for (size_t i = 0; i < list.getNumSubtrees(); ++i) {
+        _stream << list.getProbability(i);
+        writeSpanList(list.getSubtree(i), _stream, *this);
+    }
+}
+
+namespace {
+uint8_t getAnnotationFeatures(const Annotation &annotation) {
+    uint8_t features = annotation.getSpanNode() ? 1 : 0;
+    features |= annotation.getFieldValue() ? 2 : 0;
+    return features;
+}
+}  // namespace
+
+void AnnotationSerializer::write(const Annotation &annotation) {
+    _stream << annotation.getTypeId();
+    _stream << getAnnotationFeatures(annotation);
+
+    nbostream tmp_stream;
+    if (annotation.getSpanNode()) {
+        size_t node_index = _span_node_map[annotation.getSpanNode()];
+        putInt1_2_4Bytes(tmp_stream, node_index);
+    }
+    if (annotation.getFieldValue()) {
+        uint32_t type_id = annotation.getFieldValue()->getDataType()->getId();
+        tmp_stream << type_id;
+        VespaDocumentSerializer serializer(tmp_stream);
+        serializer.write(*annotation.getFieldValue());
+    }
+
+    putInt1_2_4BytesAs4(_stream, tmp_stream.size());
+    _stream.write(tmp_stream.peek(), tmp_stream.size());
+}
+
+}  // namespace document
diff --git a/document/src/vespa/document/serialization/annotationserializer.h b/document/src/vespa/document/serialization/annotationserializer.h
new file mode 100644
index 00000000000..cd8f8066f6c
--- /dev/null
+++ b/document/src/vespa/document/serialization/annotationserializer.h
@@ -0,0 +1,41 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/document/annotation/spantreevisitor.h>
+#include <map>
+
+namespace vespalib { class nbostream; }
+
+namespace document {
+class AlternateSpanList;
+class Annotation;
+class Span;
+class SpanList;
+class SimpleSpanList;
+class SpanNode;
+class SpanTree;
+
+class AnnotationSerializer : private SpanTreeVisitor {
+    vespalib::nbostream &_stream;
+    std::map<const SpanNode *, size_t> _span_node_map;
+
+    void visit(const Span &node) override { writeSpan(node); }
+    void visit(const SpanList &node) override { writeList(node); }
+    void visit(const SimpleSpanList &node) override { writeList(node); }
+    void visit(const AlternateSpanList &node) override { writeList(node); }
+
+public:
+    AnnotationSerializer(vespalib::nbostream &stream);
+
+    void write(const SpanTree &tree);
+    void write(const SpanNode &node);
+    void write(const Annotation &annotation);
+    void writeSpan(const Span &node);
+    void writeList(const SpanList &list);
+    void writeList(const SimpleSpanList &list);
+    void writeList(const AlternateSpanList &list);
+};
+
+}  // namespace document
+
diff --git a/document/src/vespa/document/serialization/documentreader.h b/document/src/vespa/document/serialization/documentreader.h
new file mode 100644
index 00000000000..9e6f15b13e6
--- /dev/null
+++ b/document/src/vespa/document/serialization/documentreader.h
@@ -0,0 +1,18 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/document/serialization/fieldreader.h>
+
+namespace document {
+class DocumentId;
+class DocumentType;
+
+class DocumentReader : public FieldReader {
+public:
+    virtual void read(Document &value) = 0;
+    virtual void read(DocumentId &value) = 0;
+    virtual void read(DocumentType &value) = 0;
+};
+
+}
diff --git a/document/src/vespa/document/serialization/documentwriter.h b/document/src/vespa/document/serialization/documentwriter.h
new file mode 100644
index 00000000000..765b85c61c4
--- /dev/null
+++ b/document/src/vespa/document/serialization/documentwriter.h
@@ -0,0 +1,18 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/document/serialization/fieldwriter.h>
+
+namespace document {
+class DocumentId;
+class DocumentType;
+
+class DocumentWriter : public FieldWriter {
+public:
+    virtual void write(const Document &value) = 0;
+    virtual void write(const DocumentId &value) = 0;
+    virtual void write(const DocumentType &value) = 0;
+};
+
+}
diff --git a/document/src/vespa/document/serialization/fieldreader.h b/document/src/vespa/document/serialization/fieldreader.h
new file mode 100644
index 00000000000..a04a58272d9
--- /dev/null
+++ b/document/src/vespa/document/serialization/fieldreader.h
@@ -0,0 +1,53 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+namespace vespalib { class FieldBase; }
+
+namespace document {
+class Document;
+class ArrayFieldValue;
+class MapFieldValue;
+class ByteFieldValue;
+class IntFieldValue;
+class LongFieldValue;
+class FloatFieldValue;
+class DoubleFieldValue;
+class RawFieldValue;
+class ShortFieldValue;
+class StringFieldValue;
+class StructFieldValue;
+class WeightedSetFieldValue;
+
+class FieldReader {
+public:
+    virtual ~FieldReader() {}
+    virtual void read(const vespalib::FieldBase &field,
+                      Document &value) = 0;
+    virtual void read(const vespalib::FieldBase &field,
+                      ArrayFieldValue &value) = 0;
+    virtual void read(const vespalib::FieldBase &field,
+                      MapFieldValue &value) = 0;
+    virtual void read(const vespalib::FieldBase &field,
+                      ByteFieldValue &value) = 0;
+    virtual void read(const vespalib::FieldBase &field,
+                      DoubleFieldValue &value) = 0;
+    virtual void read(const vespalib::FieldBase &field,
+                      FloatFieldValue &value) = 0;
+    virtual void read(const vespalib::FieldBase &field,
+                      IntFieldValue &value) = 0;
+    virtual void read(const vespalib::FieldBase &field,
+                      LongFieldValue &value) = 0;
+    virtual void read(const vespalib::FieldBase &field,
+                      RawFieldValue &value) = 0;
+    virtual void read(const vespalib::FieldBase &field,
+                      ShortFieldValue &value) = 0;
+    virtual void read(const vespalib::FieldBase &field,
+                      StringFieldValue &value) = 0;
+    virtual void read(const vespalib::FieldBase &field,
+                      StructFieldValue &value) = 0;
+    virtual void read(const vespalib::FieldBase &field,
+                      WeightedSetFieldValue &value) = 0;
+};
+}  // namespace document
+
diff --git a/document/src/vespa/document/serialization/fieldwriter.h b/document/src/vespa/document/serialization/fieldwriter.h
new file mode 100644
index 00000000000..770f4fa8e2e
--- /dev/null
+++ b/document/src/vespa/document/serialization/fieldwriter.h
@@ -0,0 +1,53 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+namespace vespalib { class FieldBase; }
+
+namespace document {
+class Document;
+class ArrayFieldValue;
+class MapFieldValue;
+class ByteFieldValue;
+class IntFieldValue;
+class LongFieldValue;
+class FloatFieldValue;
+class DoubleFieldValue;
+class RawFieldValue;
+class StringFieldValue;
+class ShortFieldValue;
+class StructFieldValue;
+class WeightedSetFieldValue;
+
+class FieldWriter {
+public:
+    virtual ~FieldWriter() {}
+    virtual void write(const vespalib::FieldBase &field,
+                       const Document &value) = 0;
+    virtual void write(const vespalib::FieldBase &field,
+                       const ArrayFieldValue &value) = 0;
+    virtual void write(const vespalib::FieldBase &field,
+                       const MapFieldValue &value) = 0;
+    virtual void write(const vespalib::FieldBase &field,
+                       const ByteFieldValue &value) = 0;
+    virtual void write(const vespalib::FieldBase &field,
+                       const DoubleFieldValue &value) = 0;
+    virtual void write(const vespalib::FieldBase &field,
+                       const FloatFieldValue &value) = 0;
+    virtual void write(const vespalib::FieldBase &field,
+                       const IntFieldValue &value) = 0;
+    virtual void write(const vespalib::FieldBase &field,
+                       const LongFieldValue &value) = 0;
+    virtual void write(const vespalib::FieldBase &field,
+                       const RawFieldValue &value) = 0;
+    virtual void write(const vespalib::FieldBase &field,
+                       const ShortFieldValue &value) = 0;
+    virtual void write(const vespalib::FieldBase &field,
+                       const StringFieldValue &value) = 0;
+    virtual void write(const vespalib::FieldBase &field,
+                       const StructFieldValue &value) = 0;
+    virtual void write(const vespalib::FieldBase &field,
+                       const WeightedSetFieldValue &value) = 0;
+};
+
+}
diff --git a/document/src/vespa/document/serialization/slime_output_to_vector.h b/document/src/vespa/document/serialization/slime_output_to_vector.h
new file mode 100644
index 00000000000..4dcd6524a96
--- /dev/null
+++ b/document/src/vespa/document/serialization/slime_output_to_vector.h
@@ -0,0 +1,32 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/vespalib/data/slime/output.h>
+#include <vector>
+
+namespace document {
+
+class SlimeOutputToVector : public vespalib::slime::Output {
+    std::vector<char> _buf;
+    size_t _size;
+
+public:
+    SlimeOutputToVector() : _buf(), _size(0) {}
+
+    virtual char *exchange(char *p, size_t commit, size_t reserve) {
+        assert(!commit || p == &_buf[_size]);
+        (void) p;
+        _size += commit;
+        if (_size + reserve > _buf.size()) {
+            _buf.resize(_size + reserve);
+        }
+        return &_buf[_size];
+    }
+
+    const char *data() const { return &_buf[0]; }
+    size_t size() const { return _size; }
+};
+
+}  // namespace document
+
diff --git a/document/src/vespa/document/serialization/util.h b/document/src/vespa/document/serialization/util.h
new file mode 100644
index 00000000000..06a8af418ed
--- /dev/null
+++ b/document/src/vespa/document/serialization/util.h
@@ -0,0 +1,128 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+namespace document {
+
+// Sets the value of a variable for the duration of this object's lifetime.
+// The original value is restored when this object is destroyed.
+template <typename T>
+class VarScope {
+    T &_ptr;
+    T _old;
+
+public:
+    VarScope (T &ptr, T new_val) :
+        _ptr(ptr),
+        _old(_ptr) {
+        _ptr = new_val;
+    }
+    ~VarScope() { _ptr = _old; }
+};
+
+template <typename T, typename Input>
+T readValue(Input &input) {
+    T val;
+    input >> val;
+    return val;
+}
+
+template <typename Input>
+uint32_t getInt1_4Bytes(Input &input) {
+    char first_byte = *input.peek();
+    if (!(first_byte & 0x80)) {
+        return readValue<uint8_t>(input);
+    } else {
+        return readValue<uint32_t>(input) & 0x7fffffff;
+    }
+}
+
+template <typename Input>
+uint32_t getInt1_2_4Bytes(Input &input) {
+    char first_byte = *input.peek();
+    if (!(first_byte & 0x80)) {
+        return readValue<uint8_t>(input);
+    } else if (!(first_byte & 0x40)) {
+        return readValue<uint16_t>(input) & 0x3fff;
+    } else {
+        return readValue<uint32_t>(input) & 0x3fffffff;
+    }
+}
+
+template <typename Input>
+uint64_t getInt2_4_8Bytes(Input &input) {
+    char first_byte = *input.peek();
+    if (!(first_byte & 0x80)) {
+        return readValue<uint16_t>(input);
+    } else if (!(first_byte & 0x40)) {
+        return readValue<uint32_t>(input) & 0x3fffffff;
+    } else {
+        return readValue<uint64_t>(input) & 0x3fffffffffffffff;
+    }
+}
+
+template <typename Output>
+void putInt1_4Bytes(Output &out, uint32_t val) {
+    if (val < 0x80) {
+        out << static_cast<uint8_t>(val);
+    } else {
+        out << (val | 0x80000000);
+    }
+}
+
+template <typename Output>
+void putInt1_2_4Bytes(Output &out, uint32_t val) {
+    if (val < 0x80) {
+        out << static_cast<uint8_t>(val);
+    } else if (val < 0x4000) {
+        out << static_cast<uint16_t>(val | 0x8000);
+    } else {
+        out << (val | 0xc0000000);
+    }
+}
+
+template <typename Output>
+void putInt1_2_4BytesAs4(Output &out, uint32_t val) {
+    out << (val | 0xc0000000);
+}
+
+template <typename Output>
+void putInt2_4_8Bytes(Output &out, uint64_t val) {
+    if (val < 0x8000) {
+        out << static_cast<uint16_t>(val);
+    } else if (val < 0x40000000) {
+        out << static_cast<uint32_t>(val | 0x80000000);
+    } else {
+        out << (val | 0xc000000000000000);
+    }
+}
+
+inline uint32_t sizeOfInt1_4Bytes(uint32_t val) {
+    if (val < 0x80) {
+        return 1;
+    } else {
+        return 4;
+    }
+}
+
+inline uint32_t sizeOfInt1_2_4Bytes(uint32_t val) {
+    if (val < 0x80) {
+        return 1;
+    } else if (val < 0x4000) {
+        return 2;
+    } else {
+        return 4;
+    }
+}
+
+inline uint32_t sizeOfInt2_4_8Bytes(uint64_t val) {
+    if (val < 0x8000) {
+        return 2;
+    } else if (val < 0x40000000) {
+        return 4;
+    } else {
+        return 8;
+    }
+}
+}  // namespace document
+
diff --git a/document/src/vespa/document/serialization/vespadocumentdeserializer.cpp b/document/src/vespa/document/serialization/vespadocumentdeserializer.cpp
new file mode 100644
index 00000000000..d9912c735c5
--- /dev/null
+++ b/document/src/vespa/document/serialization/vespadocumentdeserializer.cpp
@@ -0,0 +1,397 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".vespadocumentdeserializer");
+
+#include <vespa/document/serialization/vespadocumentdeserializer.h>
+
+#include "annotationdeserializer.h"
+#include "util.h"
+#include <vespa/document/annotation/spantree.h>
+#include <vespa/document/fieldvalue/annotationreferencefieldvalue.h>
+#include <vespa/document/fieldvalue/arrayfieldvalue.h>
+#include <vespa/document/fieldvalue/bytefieldvalue.h>
+#include <vespa/document/fieldvalue/document.h>
+#include <vespa/document/fieldvalue/doublefieldvalue.h>
+#include <vespa/document/fieldvalue/floatfieldvalue.h>
+#include <vespa/document/fieldvalue/longfieldvalue.h>
+#include <vespa/document/fieldvalue/mapfieldvalue.h>
+#include <vespa/document/fieldvalue/predicatefieldvalue.h>
+#include <vespa/document/fieldvalue/rawfieldvalue.h>
+#include <vespa/document/fieldvalue/shortfieldvalue.h>
+#include <vespa/document/fieldvalue/stringfieldvalue.h>
+#include <vespa/document/fieldvalue/weightedsetfieldvalue.h>
+#include <vespa/document/fieldvalue/tensorfieldvalue.h>
+#include <vespa/document/repo/documenttyperepo.h>
+#include <vespa/document/repo/fixedtyperepo.h>
+#include <vespa/document/util/compressionconfig.h>
+#include <vespa/vespalib/data/slime/binary_format.h>
+#include <vespa/vespalib/data/slime/memory.h>
+#include <vespa/vespalib/data/slime/slime.h>
+#include <vespa/vespalib/stllike/asciistream.h>
+#include <vespa/vespalib/stllike/string.h>
+#include <vespa/vespalib/util/backtrace.h>
+#include <vespa/vespalib/tensor/tensor.h>
+#include <vespa/vespalib/tensor/serialization/typed_binary_format.h>
+#include <vector>
+
+using std::vector;
+using vespalib::Slime;
+using vespalib::asciistream;
+using vespalib::nbostream;
+using vespalib::slime::Memory;
+using vespalib::stringref;
+
+
+namespace document {
+
+namespace {
+template <typename Input>
+uint32_t readSize(Input &input, uint16_t version) {
+    if (version < 7) {
+        readValue<uint32_t>(input);  // type id
+        return readValue<uint32_t>(input);
+    } else {
+        return getInt1_2_4Bytes(input);
+    }
+}
+
+template <typename T, typename Input>
+void skipIfOld(Input &input, uint16_t version) {
+    if (version < 7) {
+        readValue<T>(input);
+    }
+}
+
+uint32_t
+getChunkCount(uint8_t contentCode)
+{
+    uint8_t chunks = 0;
+    if (contentCode & 0x02) {
+        ++chunks;
+    }
+    if (contentCode & 0x04) {
+        ++chunks;
+    }
+    return chunks;
+}
+
+}  // namespace
+
+void VespaDocumentDeserializer::readDocument(Document &value) {
+    read(value.getId());
+    uint8_t content_code = readValue<uint8_t>(_stream);
+
+    LOG(spam, "content_code is %u", content_code);
+    const DocumentType *type = readDocType(value.getType());
+    if (type) {
+        Document newDoc(*type, value.getId(), true);
+        value.swap(newDoc);
+    }
+
+    FixedTypeRepo repo(_repo.getDocumentTypeRepo(), value.getType());
+    VarScope<FixedTypeRepo> repo_scope(_repo, repo);
+    uint32_t chunkCount = getChunkCount(content_code);
+    value.getFields().reset();
+    for (uint32_t i = 0; i < chunkCount; ++i) {
+        readStructNoReset(value.getFields());
+    }
+}
+
+void VespaDocumentDeserializer::read(FieldValue &value) {
+    value.accept(*this);
+}
+
+const DocumentType*
+VespaDocumentDeserializer::readDocType(const DocumentType &guess)
+{
+    stringref type_name(_stream.peek());
+
+    _stream.adjustReadPos(type_name.size() + 1);
+    readValue<uint16_t>(_stream);  // skip version
+
+    if (guess.getName() != type_name) {
+        const DocumentType *type =
+            _repo.getDocumentTypeRepo().getDocumentType(type_name);
+        if (!type) {
+            throw DocumentTypeNotFoundException(type_name, VESPA_STRLOC);
+        }
+        return type;
+    }
+    return 0;
+}
+
+void VespaDocumentDeserializer::read(DocumentId &value) {
+    stringref s(_stream.peek());
+    value.set(s);
+    _stream.adjustReadPos(s.size() + 1);
+}
+
+void VespaDocumentDeserializer::read(DocumentType &value) {
+    const DocumentType *doc_type = readDocType(value);
+    if (doc_type) {
+        value = *doc_type;
+    }
+}
+
+void VespaDocumentDeserializer::read(Document &value) {
+    uint16_t version = readValue<uint16_t>(_stream);
+    VarScope<uint16_t> version_scope(_version, version);
+
+    if (version < 6 || version > 8) {
+        asciistream msg;
+        msg << "Unrecognized serialization version " << version;
+        throw DeserializeException(msg.str(), VESPA_STRLOC);
+    }
+
+    if (version >= 7) {
+        uint32_t data_size = readValue<uint32_t>(_stream);
+        size_t data_start_size = _stream.size();
+        readDocument(value);
+        if (version == 7) {
+            readValue<uint32_t>(_stream);  // Skip crc value.
+        }
+        if (data_start_size - _stream.size() != data_size) {
+            asciistream msg;
+            msg << "Length mismatch. Was "
+                << data_start_size - _stream.size()
+                << ", expected " << data_size << ".";
+            throw DeserializeException(msg.str(), VESPA_STRLOC);
+        }
+    } else {  // version <= 6
+        getInt2_4_8Bytes(_stream);  // skip document length
+        readDocument(value);
+        readValue<uint32_t>(_stream);  // Skip crc value.
+    }
+}
+
+void VespaDocumentDeserializer::read(AnnotationReferenceFieldValue &value) {
+    value.setAnnotationIndex(getInt1_2_4Bytes(_stream));
+}
+
+void VespaDocumentDeserializer::read(ArrayFieldValue &value) {
+    uint32_t size = readSize(_stream, _version);
+    value.clear();
+    value.resize(size);
+    for (uint32_t i = 0; i < size; ++i) {
+        skipIfOld<uint32_t>(_stream, _version);  // element size
+        value[i].accept(*this);  // Double dispatch to call the correct read()
+    }
+}
+
+void VespaDocumentDeserializer::read(MapFieldValue &value) {
+    value.clear();
+    uint32_t size = readSize(_stream, _version);
+    value.resize(size);
+    for (auto & pair : value) {
+        skipIfOld<uint32_t>(_stream, _version);  // element size
+        pair.first->accept(*this);  // Double dispatch to call the correct read()
+        pair.second->accept(*this);  // Double dispatch to call the correct read()
+    }
+}
+
+namespace {
+template <typename T> struct ValueType { typedef typename T::Number Type; };
+template <> struct ValueType<ShortFieldValue> { typedef uint16_t Type; };
+template <> struct ValueType<IntFieldValue> { typedef uint32_t Type; };
+template <> struct ValueType<LongFieldValue> { typedef uint64_t Type; };
+template <>
+struct ValueType<RawFieldValue> { typedef vespalib::string Type; };
+
+template <typename T>
+void readFieldValue(nbostream &input, T &value) {
+    typename ValueType<T>::Type val;
+    input >> val;
+    value.setValue(val);
+}
+
+template <typename Input>
+stringref readAttributeString(Input &input) {
+    uint8_t size;
+    input >> size;
+    stringref s(input.peek(), size);
+    input.adjustReadPos(size + 1);
+    return s;
+}
+}  // namespace
+
+void VespaDocumentDeserializer::read(ByteFieldValue &value) {
+    readFieldValue(_stream, value);
+}
+
+void VespaDocumentDeserializer::read(DoubleFieldValue &value) {
+    readFieldValue(_stream, value);
+}
+
+void VespaDocumentDeserializer::read(FloatFieldValue &value) {
+    readFieldValue(_stream, value);
+}
+
+void VespaDocumentDeserializer::read(IntFieldValue &value) {
+    readFieldValue(_stream, value);
+}
+
+void VespaDocumentDeserializer::read(LongFieldValue &value) {
+    readFieldValue(_stream, value);
+}
+
+void VespaDocumentDeserializer::read(PredicateFieldValue &value) {
+    uint32_t stored_size = readValue<uint32_t>(_stream);
+    Memory memory(_stream.peek(), _stream.size());
+    std::unique_ptr<Slime> slime(new Slime);
+    size_t size = vespalib::slime::BinaryFormat::decode(memory, *slime);
+    if (size != stored_size) {
+        throw DeserializeException("Specified slime size don't match actual "
+                                   "slime size.", VESPA_STRLOC);
+    }
+    value = PredicateFieldValue(std::move(slime));
+    _stream.adjustReadPos(size);
+}
+
+namespace {
+template <typename FV>
+void setValue(FV &field_value, const stringref &val, bool use_ref) {
+    if (use_ref) {
+        field_value.setValueRef(val);
+    } else {
+        field_value.setValue(val);
+    }
+}
+}  // namespace
+
+void VespaDocumentDeserializer::read(RawFieldValue &value) {
+    uint32_t size = readValue<uint32_t>(_stream);
+    stringref val(_stream.peek(), size);
+    setValue(value, val, _stream.isLongLivedBuffer());
+    _stream.adjustReadPos(size);
+}
+
+void VespaDocumentDeserializer::read(ShortFieldValue &value) {
+    readFieldValue(_stream, value);
+}
+
+void VespaDocumentDeserializer::read(StringFieldValue &value) {
+    uint8_t coding = readValue<uint8_t>(_stream);
+    size_t size = getInt1_4Bytes(_stream);
+    if (size == 0) {
+        throw DeserializeException("invalid zero string length", VESPA_STRLOC);
+    }
+    stringref val(_stream.peek(), size - 1);
+    _stream.adjustReadPos(size);
+    setValue(value, val, _stream.isLongLivedBuffer());
+    if (coding & 0x40) {
+        uint32_t serializedAnnotationsSize = readValue<uint32_t>(_stream);
+        value.setSpanTrees(vespalib::ConstBufferRef(_stream.peek(), serializedAnnotationsSize),
+                           _repo, _version, _stream.isLongLivedBuffer());
+        _stream.adjustReadPos(serializedAnnotationsSize);
+    }
+}
+
+namespace {
+
+typedef SerializableArray::EntryMap FieldInfo;
+
+void readFieldInfo(nbostream& input, SerializableArray::EntryMap & field_info) __attribute__((noinline));
+
+void readFieldInfo(nbostream& input, SerializableArray::EntryMap & field_info) {
+    size_t field_count = getInt1_4Bytes(input);
+    field_info.reserve(field_count);
+    uint32_t offset = 0;
+    for (size_t i = 0; i < field_count; ++i) {
+        const uint32_t id = getInt1_4Bytes(input);
+        const uint32_t size = getInt2_4_8Bytes(input);
+        field_info.emplace_back(id, size, offset);
+        offset += size;
+    }
+}
+}  // namespace
+
+void VespaDocumentDeserializer::readStructNoReset(StructFieldValue &value) {
+    size_t start_size = _stream.size();
+    size_t data_size;
+    if (_version < 6) {
+        throw DeserializeException("Illegal struct serialization version.", VESPA_STRLOC);
+    } else if (_version < 7) {
+        data_size = getInt2_4_8Bytes(_stream);
+    } else {
+        data_size = readValue<uint32_t>(_stream);
+    }
+
+    CompressionConfig::Type compression_type =
+        CompressionConfig::Type(readValue<uint8_t>(_stream));
+
+    SerializableArray::EntryMap  field_info;
+    size_t uncompressed_size = 0;
+    if (CompressionConfig::isCompressed(compression_type)) {
+        uncompressed_size = getInt2_4_8Bytes(_stream);
+    }
+    readFieldInfo(_stream, field_info);
+    if (_version < 7) {
+        data_size -= (start_size - _stream.size());
+    }
+    if (CompressionConfig::isCompressed(compression_type)) {
+        if ((compression_type != CompressionConfig::LZ4)) {
+            throw DeserializeException("Unsupported compression type.", VESPA_STRLOC);
+        } else if (data_size > _stream.size()) {
+            throw DeserializeException("Invalid compressed struct data.", VESPA_STRLOC);
+        }
+    }
+
+    if (data_size > 0) {
+        ByteBuffer::UP buffer(_stream.isLongLivedBuffer()
+                          ? new ByteBuffer(_stream.peek(), data_size)
+                          : ByteBuffer::copyBuffer(_stream.peek(), data_size));
+        LOG(spam, "Lazy deserializing into %s with _version %u",
+            value.getDataType()->getName().c_str(), _version);
+        value.lazyDeserialize(_repo, _version, std::move(field_info),
+                              std::move(buffer), compression_type, uncompressed_size);
+        _stream.adjustReadPos(data_size);
+    }
+}
+
+void
+VespaDocumentDeserializer::read(StructFieldValue& value)
+{
+    value.reset();
+    readStructNoReset(value);
+}
+
+void VespaDocumentDeserializer::read(WeightedSetFieldValue &value) {
+    value.clear();
+    readValue<uint32_t>(_stream);  // skip type id
+    uint32_t size = readValue<uint32_t>(_stream);
+    value.reserve(size);
+    for (uint32_t i = 0; i < size; ++i) {
+        readValue<uint32_t>(_stream);  // skip element size
+        FieldValue::UP child = value.createNested();
+        child->accept(*this);  // Double dispatch to call the correct read()
+        uint32_t weight = readValue<uint32_t>(_stream);
+        value.push_back(std::move(child), weight);
+    }
+}
+
+
+void
+VespaDocumentDeserializer::read(TensorFieldValue &value)
+{
+    size_t length = _stream.getInt1_4Bytes();
+    if (length < _stream.size()) {
+        throw DeserializeException(vespalib::make_string("Stream failed size(%zu), needed(%zu)", _stream.size(), length),
+                                   VESPA_STRLOC);
+    }
+    std::unique_ptr<vespalib::tensor::Tensor> tensor;
+    if (length != 0) {
+        nbostream wrapStream(_stream.peek(), length);
+        tensor = vespalib::tensor::TypedBinaryFormat::deserialize(wrapStream);
+        if (wrapStream.size() != 0) {
+            throw DeserializeException("Leftover bytes deserializing "
+                                       "tensor field value.",
+                                       VESPA_STRLOC);
+        }
+    }
+    value.assignDeserialized(std::move(tensor));
+    _stream.adjustReadPos(length);
+}
+
+}  // document
diff --git a/document/src/vespa/document/serialization/vespadocumentdeserializer.h b/document/src/vespa/document/serialization/vespadocumentdeserializer.h
new file mode 100644
index 00000000000..13828957c91
--- /dev/null
+++ b/document/src/vespa/document/serialization/vespadocumentdeserializer.h
@@ -0,0 +1,82 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <memory>
+#include <vespa/document/fieldvalue/fieldvaluevisitor.h>
+#include <vespa/document/repo/fixedtyperepo.h>
+
+namespace vespalib { class nbostream; }
+
+namespace document {
+class DocumentId;
+class DocumentType;
+class DocumentTypeRepo;
+class FieldValue;
+
+class VespaDocumentDeserializer : private FieldValueVisitor {
+    vespalib::nbostream &_stream;
+    FixedTypeRepo _repo;
+    uint16_t _version;
+
+    virtual void visit(AnnotationReferenceFieldValue &value) { read(value); }
+    virtual void visit(ArrayFieldValue &value) { read(value); }
+    virtual void visit(ByteFieldValue &value) { read(value); }
+    virtual void visit(Document &value) { read(value); }
+    virtual void visit(DoubleFieldValue &value) { read(value); }
+    virtual void visit(FloatFieldValue &value) { read(value); }
+    virtual void visit(IntFieldValue &value) { read(value); }
+    virtual void visit(LongFieldValue &value) { read(value); }
+    virtual void visit(MapFieldValue &value) { read(value); }
+    virtual void visit(PredicateFieldValue &value) { read(value); }
+    virtual void visit(RawFieldValue &value) { read(value); }
+    virtual void visit(ShortFieldValue &value) { read(value); }
+    virtual void visit(StringFieldValue &value) { read(value); }
+    virtual void visit(StructFieldValue &value) { read(value); }
+    virtual void visit(WeightedSetFieldValue &value) { read(value); }
+    virtual void visit(TensorFieldValue &value) { read(value); }
+
+    void readDocument(Document &value);
+
+public:
+    VespaDocumentDeserializer(const DocumentTypeRepo &repo, vespalib::nbostream &stream, uint16_t version) :
+        _stream(stream),
+        _repo(repo),
+        _version(version)
+    {
+    }
+
+    VespaDocumentDeserializer(const FixedTypeRepo &repo, vespalib::nbostream &stream, uint16_t version) :
+        _stream(stream),
+        _repo(repo),
+        _version(version)
+    {
+    }
+
+    // returns NULL if the read doc type equals guess.
+    const DocumentType *readDocType(const DocumentType &guess);
+
+    void read(FieldValue &value);
+
+    void read(DocumentId &value);
+    void read(DocumentType &value);
+    void read(Document &value);
+    void read(AnnotationReferenceFieldValue &value);
+    void read(ArrayFieldValue &value);
+    void read(MapFieldValue &value);
+    void read(ByteFieldValue &value);
+    void read(DoubleFieldValue &value);
+    void read(FloatFieldValue &value);
+    void read(IntFieldValue &value);
+    void read(LongFieldValue &value);
+    void read(PredicateFieldValue &value);
+    void read(RawFieldValue &value);
+    void read(ShortFieldValue &value);
+    void read(StringFieldValue &value);
+    void read(StructFieldValue &value);
+    void readStructNoReset(StructFieldValue &value);
+    void read(WeightedSetFieldValue &value);
+    void read(TensorFieldValue &value);
+};
+}  // namespace document
+
diff --git a/document/src/vespa/document/serialization/vespadocumentserializer.cpp b/document/src/vespa/document/serialization/vespadocumentserializer.cpp
new file mode 100644
index 00000000000..536d9366e1b
--- /dev/null
+++ b/document/src/vespa/document/serialization/vespadocumentserializer.cpp
@@ -0,0 +1,551 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".vespadocumentserializer");
+
+#include "vespadocumentserializer.h"
+
+#include "annotationserializer.h"
+#include "slime_output_to_vector.h"
+#include "util.h"
+#include <vespa/document/fieldset/fieldsets.h>
+#include <vespa/document/fieldvalue/annotationreferencefieldvalue.h>
+#include <vespa/document/fieldvalue/arrayfieldvalue.h>
+#include <vespa/document/fieldvalue/bytefieldvalue.h>
+#include <vespa/document/fieldvalue/document.h>
+#include <vespa/document/fieldvalue/doublefieldvalue.h>
+#include <vespa/document/fieldvalue/floatfieldvalue.h>
+#include <vespa/document/fieldvalue/intfieldvalue.h>
+#include <vespa/document/fieldvalue/longfieldvalue.h>
+#include <vespa/document/fieldvalue/mapfieldvalue.h>
+#include <vespa/document/fieldvalue/predicatefieldvalue.h>
+#include <vespa/document/fieldvalue/rawfieldvalue.h>
+#include <vespa/document/fieldvalue/shortfieldvalue.h>
+#include <vespa/document/fieldvalue/stringfieldvalue.h>
+#include <vespa/document/fieldvalue/structfieldvalue.h>
+#include <vespa/document/fieldvalue/weightedsetfieldvalue.h>
+#include <vespa/document/fieldvalue/tensorfieldvalue.h>
+#include <vespa/document/update/updates.h>
+#include <vespa/document/update/fieldpathupdates.h>
+#include <vespa/document/util/compressionconfig.h>
+#include <vespa/vespalib/data/slime/binary_format.h>
+#include <vespa/vespalib/objects/nbostream.h>
+#include <vespa/vespalib/tensor/serialization/typed_binary_format.h>
+#include <utility>
+#include <vector>
+
+using std::make_pair;
+using std::pair;
+using std::vector;
+using vespalib::nbostream;
+using vespalib::stringref;
+using vespalib::string;
+using vespalib::slime::BinaryFormat;
+
+namespace document {
+
+VespaDocumentSerializer::VespaDocumentSerializer(nbostream &stream)
+    : _stream(stream) {
+}
+
+void VespaDocumentSerializer::writeFieldValue(const FieldValue &value) {
+    write(value);
+}
+
+void VespaDocumentSerializer::writeSerializedData(const void *buf, size_t length) {
+    _stream.write(buf, length);
+}
+
+void VespaDocumentSerializer::write(const ValueUpdate &value) {
+    value.accept(*this);
+}
+
+void VespaDocumentSerializer::write(const FieldPathUpdate &value) {
+    value.accept(*this);
+}
+
+void VespaDocumentSerializer::write(const FieldValue &value) {
+    value.accept(*this);
+}
+
+void VespaDocumentSerializer::write(const DocumentId &value) {
+    string id_string = value.getScheme().toString();
+    _stream.write(id_string.data(), id_string.size());
+    _stream << static_cast<uint8_t>(0);
+}
+
+void VespaDocumentSerializer::write(const DocumentType &value) {
+    _stream.write(value.getName().data(), value.getName().size());
+
+    _stream << static_cast<uint8_t>(0)
+            << static_cast<uint16_t>(0);  // version
+}
+
+uint8_t
+VespaDocumentSerializer::getContentCode(bool hasHeader, bool hasBody) const
+{
+    uint8_t content = 0x01;  // Document type is always present.
+    if (hasHeader) {
+        content |= 0x02;  // Header is present.
+    }
+    if (hasBody) {
+        content |= 0x04;  // Body is present.
+    }
+    return content;
+}
+
+void VespaDocumentSerializer::write(const Document &value,
+                                    DocSerializationMode mode) {
+    nbostream doc_stream;
+    VespaDocumentSerializer doc_serializer(doc_stream);
+    doc_serializer.write(value.getId());
+
+    int hasHeader = 0;
+    int hasBody = 0;
+
+    for (StructuredFieldValue::const_iterator it(value.getFields().begin()), mt(value.getFields().end());
+         it != mt;
+         ++it)
+    {
+        if (it.field().isHeaderField()) {
+            hasHeader = 1;
+        } else {
+            hasBody = 1;
+        }
+
+        if (hasHeader && hasBody) {
+            break;
+        }
+    }
+
+    if (mode != COMPLETE) {
+        hasBody = 0;
+    }
+
+    doc_stream << getContentCode(hasHeader, hasBody);
+    doc_serializer.write(value.getType());
+
+    if (!structNeedsReserialization(value.getFields())) {
+        // FIXME(vekterli):
+        // Currently assume legacy serialization; a chunk will only ever contain fields
+        // _either_ for the header _or_ for the body, never a mixture!
+        // This is to avoid horrible breakage whilst ripping out old guts.
+        const StructFieldValue::Chunks & chunks = value.getFields().getChunks();
+        LOG(spam, "Writing %zu chunks unchanged", chunks.size());
+        if (hasHeader) {
+            assert(chunks.size() >= 1);
+            doc_serializer.writeUnchanged(chunks[0]);
+            if (hasBody) {
+                assert(chunks.size() == 2);
+                doc_serializer.writeUnchanged(chunks[1]);
+            }
+        } else if (hasBody) {
+            assert(chunks.size() == 1);
+            doc_serializer.writeUnchanged(chunks[0]);
+        }
+    } else {
+        if (hasHeader) {
+            doc_serializer.write(value.getFields(), HeaderFields());
+        }
+        if (hasBody) {
+            doc_serializer.write(value.getFields(), BodyFields());
+        }
+    }
+
+    const uint16_t version = serialize_version;
+    _stream << version
+            << static_cast<uint32_t>(doc_stream.size());
+    _stream.write(doc_stream.peek(), doc_stream.size());
+}
+
+void VespaDocumentSerializer::write(const AnnotationReferenceFieldValue &value)
+{
+    putInt1_2_4Bytes(_stream, value.getAnnotationIndex());
+}
+
+void VespaDocumentSerializer::write(const ArrayFieldValue &value) {
+    putInt1_2_4Bytes(_stream, value.size());
+    for (size_t i(0), m(value.size()); i < m; ++i) {
+        value[i].accept(*this);
+    }
+}
+
+void VespaDocumentSerializer::write(const MapFieldValue &value) {
+    putInt1_2_4Bytes(_stream, value.size());
+    for (const auto & entry : value) {
+        (*entry.first).accept(*this);
+        (*entry.second).accept(*this);
+    }
+}
+
+void VespaDocumentSerializer::write(const ByteFieldValue &value) {
+    _stream << value.getValue();
+}
+
+void VespaDocumentSerializer::write(const DoubleFieldValue &value) {
+    _stream << value.getValue();
+}
+
+void VespaDocumentSerializer::write(const FloatFieldValue &value) {
+    _stream << value.getValue();
+}
+
+void VespaDocumentSerializer::write(const IntFieldValue &value) {
+    _stream << static_cast<uint32_t>(value.getValue());
+}
+
+void VespaDocumentSerializer::write(const LongFieldValue &value) {
+    _stream << static_cast<uint64_t>(value.getValue());
+}
+
+void VespaDocumentSerializer::write(const PredicateFieldValue &value) {
+    SlimeOutputToVector output;
+    vespalib::slime::BinaryFormat::encode(value.getSlime(), output);
+    _stream << static_cast<uint32_t>(output.size());
+    _stream.write(output.data(), output.size());
+}
+
+void VespaDocumentSerializer::write(const RawFieldValue &value) {
+    _stream << static_cast<uint32_t>(value.getValueRef().size());
+    _stream.write(value.getValueRef().data(), value.getValueRef().size());
+}
+
+void VespaDocumentSerializer::write(const ShortFieldValue &value) {
+    _stream << static_cast<uint16_t>(value.getValue());
+}
+
+namespace {
+template <typename Map>
+void writeAnnotations(AnnotationSerializer &serializer, const Map &m) {
+    for (const auto & annotation : m) {
+        serializer.write(*annotation.second);
+    }
+}
+}  // namespace
+
+void VespaDocumentSerializer::write(const StringFieldValue &value) {
+    uint8_t coding = (value.hasSpanTrees() << 6);
+    _stream << coding;
+    putInt1_4Bytes(_stream, value.getValueRef().size() + 1);
+    _stream.write(value.getValueRef().data(), value.getValueRef().size());
+    _stream << static_cast<uint8_t>(0);  // add null-termination.
+    if (value.hasSpanTrees()) {
+        vespalib::ConstBufferRef buffer = value.getSerializedAnnotations();
+        _stream << static_cast<uint32_t>(buffer.size());
+        _stream.write(buffer.data(), buffer.size());
+    }
+}
+
+namespace {
+void serializeFields(const StructFieldValue &value, nbostream &stream,
+                     vector<pair<uint32_t, uint32_t> > &field_info,
+                     const FieldSet& fieldSet) {
+    VespaDocumentSerializer serializer(stream);
+    for (StructuredFieldValue::const_iterator
+             it(value.begin()), e(value.end());
+         it != e; ++it)
+    {
+        if (!fieldSet.contains(it.field())) {
+            continue;
+        }
+        size_t original_size = stream.size();
+        int id = it.field().getId(value.getVersion());
+        if (!value.serializeField(id, VespaDocumentSerializer::getCurrentVersion(), serializer)) {
+            continue;
+        }
+        size_t field_size = stream.size() - original_size;
+        field_info.push_back(make_pair(it.field().getId(VespaDocumentSerializer::getCurrentVersion()), field_size));
+    }
+}
+
+bool compressionSufficient(const CompressionConfig &config,
+                           uint64_t old_size, size_t new_size)
+{
+    return (new_size + 8) < (old_size * config.threshold / 100);
+}
+
+bool bigEnough(size_t size, const CompressionConfig &config)
+{
+    return (size >= config.minSize);
+}
+
+void compressStream(const CompressionConfig &config, nbostream &stream,
+                    vespalib::DataBuffer & compressed_data)
+{
+    if (config.useCompression() && bigEnough(stream.size(), config)) {
+        CompressionConfig::Type compressedType = compress(config, vespalib::ConstBufferRef(stream.c_str(), stream.size()), compressed_data, false);
+        if (compressedType != config.type ||
+            ! compressionSufficient(config, stream.size(), compressed_data.getDataLen()))
+        {
+            compressed_data.clear();
+        }
+    }
+}
+
+void putFieldInfo(nbostream &output,
+                  const vector<pair<uint32_t, uint32_t> > &field_info) {
+    putInt1_4Bytes(output, field_info.size());
+    for (size_t i = 0; i < field_info.size(); ++i) {
+        putInt1_4Bytes(output, field_info[i].first);
+        putInt2_4_8Bytes(output, field_info[i].second);
+    }
+}
+}  // namespace
+
+/**
+ * Reserialize if value has been modified since deserialization
+ * or we are bumping version
+ * or compression type has changed AND config says compress.
+ * The last and is to make sure that we do not decompress a document
+ * unintentionally.
+ */
+bool VespaDocumentSerializer::structNeedsReserialization(
+        const StructFieldValue &value)
+{
+    if (value.hasChanged()) {
+        return true;
+    }
+
+    if (value.getVersion() != serialize_version) {
+        return true;
+    }
+
+    if (value.getCompressionConfig().type == CompressionConfig::NONE) {
+        return false;
+    }
+
+    const StructFieldValue::Chunks & chunks = value.getChunks();
+
+    for (uint32_t i = 0; i < chunks.size(); ++i) {
+        if (chunks[i].getCompression() != value.getCompressionConfig().type &&
+            chunks[i].getCompression() != CompressionConfig::UNCOMPRESSABLE)
+        {
+            return true;
+        }
+    }
+
+    return false;
+}
+
+void VespaDocumentSerializer::writeUnchanged(const SerializableArray &value) {
+    vector<pair<uint32_t, uint32_t> > field_info;
+    const std::vector<SerializableArray::Entry>& entries = value.getEntries();
+
+    field_info.reserve(entries.size());
+    for(const auto & entry : entries) {
+        field_info.emplace_back(entry.id(), entry.size());
+    }
+
+    const ByteBuffer* buffer = value.getSerializedBuffer();
+    uint32_t sz = (buffer != NULL) ? buffer->getLength() : 0;
+    _stream << sz;
+    _stream << static_cast<uint8_t>(value.getCompression());
+    if (CompressionConfig::isCompressed(value.getCompression())) {
+        putInt2_4_8Bytes(_stream, value.getCompressionInfo().getUncompressedSize());
+    }
+    putFieldInfo(_stream, field_info);
+    if (sz) {
+        _stream.write(buffer->getBuffer(), buffer->getLength());
+    }
+}
+
+void VespaDocumentSerializer::write(const StructFieldValue &value,
+                                    const FieldSet& fieldSet)
+{
+    nbostream value_stream;
+    vector<pair<uint32_t, uint32_t> > field_info;
+    serializeFields(value, value_stream, field_info, fieldSet);
+    if (field_info.empty()) {
+        LOG(debug, "Not writing struct since it has no fields");
+        return;
+    }
+
+    const CompressionConfig &comp_config = value.getCompressionConfig();
+    vespalib::DataBuffer compressed_data;
+    compressStream(comp_config, value_stream, compressed_data);
+
+    if (compressed_data.getDataLen() == 0) {
+        uint8_t comp_type = comp_config.type == CompressionConfig::NONE ?
+                            CompressionConfig::NONE :
+                            CompressionConfig::UNCOMPRESSABLE;
+        _stream << static_cast<uint32_t>(value_stream.size());
+        _stream << comp_type;
+        putFieldInfo(_stream, field_info);
+        _stream.write(value_stream.c_str(), value_stream.size());
+    } else {
+        _stream << static_cast<uint32_t>(compressed_data.getDataLen());
+        _stream << static_cast<uint8_t>(comp_config.type);
+        putInt2_4_8Bytes(_stream, value_stream.size());
+        putFieldInfo(_stream, field_info);
+        _stream.write(compressed_data.getData(), compressed_data.getDataLen());
+    }
+}
+
+void VespaDocumentSerializer::write(const WeightedSetFieldValue &value) {
+    const WeightedSetDataType *type =
+        static_cast<const WeightedSetDataType *>(value.getDataType());
+    _stream << static_cast<uint32_t>(type->getNestedType().getId());
+    _stream << static_cast<uint32_t>(value.size());
+    for (const auto & entry : value) {
+        nbostream stream;
+        VespaDocumentSerializer serializer(stream);
+        serializer.write(*entry.first);
+        serializer.write(*entry.second);
+        _stream << static_cast<uint32_t>(stream.size());  // This is unused
+        _stream.write(stream.peek(), stream.size());
+    }
+}
+
+
+void
+VespaDocumentSerializer::write(const TensorFieldValue &value) {
+    vespalib::nbostream tmpStream;
+    auto &tensor = value.getAsTensorPtr();
+    if (tensor) {
+        vespalib::tensor::TypedBinaryFormat::serialize(tmpStream, *tensor);
+        assert(tmpStream.size() != 0);
+        _stream.putInt1_4Bytes(tmpStream.size());
+        _stream.write(tmpStream.peek(), tmpStream.size());
+    } else {
+        _stream.putInt1_4Bytes(0);
+    }
+}
+
+namespace {
+    const uint8_t CONTENT_HASTYPE(0x01);
+    const uint8_t CONTENT_HASVALUE(0x01);
+}
+
+void VespaDocumentSerializer::write42(const DocumentUpdate &value)
+{
+    _stream << static_cast<uint16_t>(value.getVersion());
+    write(value.getId());
+    _stream << static_cast<uint8_t>(CONTENT_HASTYPE);
+    _stream.write(value.getType().getName().c_str(), value.getType().getName().size() + 1);
+    _stream << static_cast<uint16_t>(0);
+    const DocumentUpdate::FieldUpdateV & updates(value.getUpdates());
+    _stream << static_cast<uint32_t>(value.serializeFlags(updates.size()));
+    for (const auto & update : updates) {
+        write(update);
+    }
+}
+
+void VespaDocumentSerializer::writeHEAD(const DocumentUpdate &value)
+{
+    write(value.getId());
+    _stream.write(value.getType().getName().c_str(), value.getType().getName().size() + 1);
+    _stream << static_cast<uint16_t>(0);
+    const DocumentUpdate::FieldUpdateV & updates(value.getUpdates());
+    _stream << static_cast<uint32_t>(updates.size());
+    for (const auto & update : updates) {
+        write(update);
+    }
+    const DocumentUpdate::FieldPathUpdateV & fieldPathUpdates(value.getFieldPathUpdates());
+    _stream << static_cast<uint32_t>(value.serializeFlags(fieldPathUpdates.size()));
+    for (const auto & update : fieldPathUpdates) {
+        _stream << update->getSerializedType();
+        write(*update);
+    }
+}
+
+void VespaDocumentSerializer::write(const FieldUpdate &value)
+{
+    _stream << static_cast<int32_t>(value.getField().getId(Document::getNewestSerializationVersion()));
+    _stream << static_cast<int32_t>(value.size());
+    for (size_t i(0), m(value.size()); i < m; i++) {
+        write(value[i]);
+    }
+}
+
+void VespaDocumentSerializer::write(const RemoveValueUpdate &value)
+{
+    _stream << RemoveValueUpdate::classId;
+    write(value.getKey());
+}
+
+
+void VespaDocumentSerializer::write(const AddValueUpdate &value)
+{
+    _stream << AddValueUpdate::classId;
+    write(value.getValue());
+    _stream << static_cast<int32_t>(value.getWeight());
+}
+
+void VespaDocumentSerializer::write(const ArithmeticValueUpdate &value)
+{
+    _stream << ArithmeticValueUpdate::classId;
+    _stream << static_cast<uint32_t>(value.getOperator());
+    _stream << static_cast<double>(value.getOperand());
+}
+
+void VespaDocumentSerializer::write(const AssignValueUpdate &value)
+{
+    _stream << AssignValueUpdate::classId;
+    if (value.hasValue()) {
+        _stream << static_cast<uint8_t>(CONTENT_HASVALUE);
+        write(value.getValue());
+    } else {
+        _stream << static_cast<uint8_t>(0);
+    }
+}
+
+void VespaDocumentSerializer::write(const ClearValueUpdate &value)
+{
+    (void) value;
+    _stream << ClearValueUpdate::classId;
+}
+
+void VespaDocumentSerializer::write(const MapValueUpdate &value)
+{
+    _stream << MapValueUpdate::classId;
+    write(value.getKey());
+    write(value.getUpdate());
+}
+
+namespace {
+
+void writeStringWithZeroTermination(nbostream & os, stringref s)
+{
+    uint32_t sz(s.size() + 1);
+    os << sz;
+    os.write(s.c_str(), sz);
+}
+
+void writeFieldPath(nbostream & os, const FieldPathUpdate & value)
+{
+    writeStringWithZeroTermination(os, value.getOriginalFieldPath());
+    writeStringWithZeroTermination(os, value.getOriginalWhereClause());
+}
+
+}
+
+void VespaDocumentSerializer::write(const AddFieldPathUpdate &value)
+{
+    writeFieldPath(_stream, value);
+    write(value.getValues());
+}
+
+void VespaDocumentSerializer::write(const AssignFieldPathUpdate &value)
+{
+    writeFieldPath(_stream, value);
+    uint8_t flags = 0;
+    flags |= value.getRemoveIfZero() ? AssignFieldPathUpdate::REMOVE_IF_ZERO : 0;
+    flags |= value.getCreateMissingPath() ? AssignFieldPathUpdate::CREATE_MISSING_PATH : 0;
+    flags |= (! value.hasValue()) ?  AssignFieldPathUpdate::ARITHMETIC_EXPRESSION : 0;
+    _stream << flags;
+    if (value.hasValue()) {
+        write(value.getValue());
+    } else {
+        writeStringWithZeroTermination(_stream, value.getExpression());
+    }
+
+}
+
+void VespaDocumentSerializer::write(const RemoveFieldPathUpdate &value)
+{
+    writeFieldPath(_stream, value);
+}
+
+
+}  // namespace document
diff --git a/document/src/vespa/document/serialization/vespadocumentserializer.h b/document/src/vespa/document/serialization/vespadocumentserializer.h
new file mode 100644
index 00000000000..0c58cd44cfe
--- /dev/null
+++ b/document/src/vespa/document/serialization/vespadocumentserializer.h
@@ -0,0 +1,107 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/document/fieldvalue/fieldvaluevisitor.h>
+#include <vespa/document/fieldvalue/fieldvaluewriter.h>
+#include <vespa/document/fieldset/fieldsets.h>
+#include <vespa/document/update/updatevisitor.h>
+#include <vespa/vespalib/objects/nbostream.h>
+
+namespace document {
+
+class DocumentId;
+class DocumentType;
+class SerializableArray;
+class ValueUpdate;
+class FieldPathUpdate;
+
+enum DocSerializationMode { COMPLETE, WITHOUT_BODY };
+
+class VespaDocumentSerializer : private ConstFieldValueVisitor,
+                                private UpdateVisitor,
+                                public FieldValueWriter {
+public:
+    VespaDocumentSerializer(vespalib::nbostream &stream);
+
+    static bool structNeedsReserialization(const StructFieldValue &value);
+
+    void writeSerializedData(const void *buf, size_t length) override;
+    void writeFieldValue(const FieldValue &value) override;
+
+    void write(const FieldValue &value);
+
+    void write(const DocumentId &value);
+    void write(const DocumentType &value);
+    void write(const Document &value, DocSerializationMode mode);
+    void write(const AnnotationReferenceFieldValue &value);
+    void write(const ArrayFieldValue &value);
+    void write(const MapFieldValue &map);
+    void write(const ByteFieldValue &value);
+    void write(const DoubleFieldValue &val);
+    void write(const FloatFieldValue &value);
+    void write(const IntFieldValue &value);
+    void write(const LongFieldValue &value);
+    void write(const PredicateFieldValue &value);
+    void write(const RawFieldValue &value);
+    void write(const ShortFieldValue &value);
+    void write(const StringFieldValue &val);
+    void write(const StructFieldValue &val, const FieldSet& fieldSet);
+    void write(const WeightedSetFieldValue &value);
+    void write(const TensorFieldValue &value);
+
+    void write42(const DocumentUpdate &value);
+    void writeHEAD(const DocumentUpdate &value);
+    void write(const FieldUpdate &value);
+    void write(const ValueUpdate &value);
+    static uint16_t getCurrentVersion() { return serialize_version; }
+private:
+    static constexpr int serialize_version = 8;
+    void writeUnchanged(const SerializableArray &val);
+    uint8_t getContentCode(bool hasHeader, bool hasBody) const;
+
+    void write(const FieldPathUpdate &value);
+
+    void write(const RemoveValueUpdate &value);
+    void write(const AddValueUpdate &value);
+    void write(const ArithmeticValueUpdate &value);
+    void write(const AssignValueUpdate &value);
+    void write(const ClearValueUpdate &value);
+    void write(const MapValueUpdate &value);
+    void write(const AddFieldPathUpdate &value);
+    void write(const AssignFieldPathUpdate &value);
+    void write(const RemoveFieldPathUpdate &value);
+
+    void visit(const DocumentUpdate &value)        override { write42(value); }
+    void visit(const FieldUpdate &value)           override { write(value); }
+    void visit(const RemoveValueUpdate &value)     override { write(value); }
+    void visit(const AddValueUpdate &value)        override { write(value); }
+    void visit(const ArithmeticValueUpdate &value) override { write(value); }
+    void visit(const AssignValueUpdate &value)     override { write(value); }
+    void visit(const ClearValueUpdate &value)      override { write(value); }
+    void visit(const MapValueUpdate &value)        override { write(value); }
+    void visit(const AddFieldPathUpdate &value)    override { write(value); }
+    void visit(const AssignFieldPathUpdate &value) override { write(value); }
+    void visit(const RemoveFieldPathUpdate &value) override { write(value); }
+
+    void visit(const AnnotationReferenceFieldValue &value) override { write(value); }
+    void visit(const ArrayFieldValue &value)               override { write(value); }
+    void visit(const ByteFieldValue &value)                override { write(value); }
+    void visit(const Document &value)                      override { write(value, COMPLETE); }
+    void visit(const DoubleFieldValue &value)              override { write(value); }
+    void visit(const FloatFieldValue &value)               override { write(value); }
+    void visit(const IntFieldValue &value)                 override { write(value); }
+    void visit(const LongFieldValue &value)                override { write(value); }
+    void visit(const MapFieldValue &value)                 override { write(value); }
+    void visit(const PredicateFieldValue &value)           override { write(value); }
+    void visit(const RawFieldValue &value)                 override { write(value); }
+    void visit(const ShortFieldValue &value)               override { write(value); }
+    void visit(const StringFieldValue &value)              override { write(value); }
+    void visit(const StructFieldValue &value)              override { write(value, AllFields()); }
+    void visit(const WeightedSetFieldValue &value)         override { write(value); }
+    void visit(const TensorFieldValue &value) override { write(value); }
+
+    vespalib::nbostream &_stream;
+};
+}  // namespace document
+
diff --git a/document/src/vespa/document/update/.gitignore b/document/src/vespa/document/update/.gitignore
new file mode 100644
index 00000000000..dfa09296ddb
--- /dev/null
+++ b/document/src/vespa/document/update/.gitignore
@@ -0,0 +1,4 @@
+*.So
+.*.swp
+.depend
+Makefile
diff --git a/document/src/vespa/document/update/CMakeLists.txt b/document/src/vespa/document/update/CMakeLists.txt
new file mode 100644
index 00000000000..0e54130e342
--- /dev/null
+++ b/document/src/vespa/document/update/CMakeLists.txt
@@ -0,0 +1,20 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_library(document_updates OBJECT
+    SOURCES
+    addfieldpathupdate.cpp
+    addvalueupdate.cpp
+    arithmeticvalueupdate.cpp
+    assignfieldpathupdate.cpp
+    assignvalueupdate.cpp
+    clearvalueupdate.cpp
+    documentupdate.cpp
+    fieldpathupdate.cpp
+    fieldupdate.cpp
+    mapvalueupdate.cpp
+    removefieldpathupdate.cpp
+    removevalueupdate.cpp
+    valueupdate.cpp
+    DEPENDS
+    AFTER
+    document_documentconfig
+)
diff --git a/document/src/vespa/document/update/addfieldpathupdate.cpp b/document/src/vespa/document/update/addfieldpathupdate.cpp
new file mode 100644
index 00000000000..2d0855a44c2
--- /dev/null
+++ b/document/src/vespa/document/update/addfieldpathupdate.cpp
@@ -0,0 +1,92 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/document/fieldvalue/fieldvalues.h>
+#include <vespa/document/repo/fixedtyperepo.h>
+#include <vespa/document/select/parser.h>
+#include <vespa/document/serialization/vespadocumentdeserializer.h>
+#include <vespa/document/update/addfieldpathupdate.h>
+#include <vespa/log/log.h>
+#include <vespa/vespalib/objects/nbostream.h>
+
+using vespalib::nbostream;
+
+LOG_SETUP(".document.update.fieldpathupdate");
+
+namespace document {
+
+IMPLEMENT_IDENTIFIABLE(AddFieldPathUpdate, FieldPathUpdate);
+
+AddFieldPathUpdate::AddFieldPathUpdate(
+        const DocumentTypeRepo& repo,
+        const DataType& type,
+        stringref fieldPath,
+        stringref whereClause,
+        const ArrayFieldValue& values)
+    : FieldPathUpdate(repo, type, fieldPath, whereClause),
+      _values(vespalib::CloneablePtr<ArrayFieldValue>(values.clone()))
+{
+    checkCompatibility(*_values);
+}
+
+AddFieldPathUpdate::AddFieldPathUpdate()
+    : FieldPathUpdate(), _values()
+{
+}
+
+FieldValue::IteratorHandler::ModificationStatus
+AddFieldPathUpdate::AddIteratorHandler::doModify(FieldValue& fv)
+{
+    LOG(spam, "Adding values to %s", fv.toString().c_str());
+    if (fv.inherits(CollectionFieldValue::classId)) {
+        CollectionFieldValue& cf = static_cast<CollectionFieldValue&>(fv);
+        for (std::size_t i = 0; i < _values.size(); ++i) {
+            cf.add(_values[i]);
+        }
+    } else {
+        vespalib::string err = vespalib::make_string(
+                "Unable to add a value to a \"%s\" field value.",
+                fv.getClass().name());
+                throw vespalib::IllegalArgumentException(err, VESPA_STRLOC);
+    }
+    return MODIFIED;
+}
+
+bool
+AddFieldPathUpdate::operator==(const FieldPathUpdate& other) const
+{
+    if (other.getClass().id() != AddFieldPathUpdate::classId) return false;
+    if (!FieldPathUpdate::operator==(other)) return false;
+    const AddFieldPathUpdate& addOther
+        = static_cast<const AddFieldPathUpdate&>(other);
+    return *addOther._values == *_values;
+}
+
+void
+AddFieldPathUpdate::print(std::ostream& out, bool verbose,
+                          const std::string& indent) const
+{
+    out << "AddFieldPathUpdate(\n";
+    FieldPathUpdate::print(out, verbose, indent + "  ");
+    out << ",\n" << indent << "  " << "values=";
+    _values->print(out, verbose, indent + "  ");
+    out << "\n" << indent << ")";
+}
+
+void
+AddFieldPathUpdate::deserialize(const DocumentTypeRepo& repo,
+                                const DataType& type,
+                                ByteBuffer& buffer, uint16_t version)
+{
+    FieldPathUpdate::deserialize(repo, type, buffer, version);
+
+    const DataType& fieldType = getResultingDataType();
+    assert(fieldType.inherits(ArrayDataType::classId));
+    FieldValue::UP val = fieldType.createFieldValue();
+    _values.reset(static_cast<ArrayFieldValue*>(val.release()));
+    nbostream stream(buffer.getBufferAtPos(), buffer.getRemaining());
+    VespaDocumentDeserializer deserializer(repo, stream, version);
+    deserializer.read(*_values);
+    buffer.incPos(buffer.getRemaining() - stream.size());
+}
+
+} // ns document
diff --git a/document/src/vespa/document/update/addfieldpathupdate.h b/document/src/vespa/document/update/addfieldpathupdate.h
new file mode 100644
index 00000000000..16d2cfa24f4
--- /dev/null
+++ b/document/src/vespa/document/update/addfieldpathupdate.h
@@ -0,0 +1,65 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/document/update/fieldpathupdate.h>
+
+namespace document {
+
+class AddFieldPathUpdate : public FieldPathUpdate
+{
+public:
+    /** For deserialization */
+    AddFieldPathUpdate();
+
+    AddFieldPathUpdate(const DocumentTypeRepo& repo,
+                       const DataType& type,
+                       stringref fieldPath,
+                       stringref whereClause,
+                       const ArrayFieldValue& values);
+
+    FieldPathUpdate* clone() const { return new AddFieldPathUpdate(*this); }
+
+    bool operator==(const FieldPathUpdate& other) const;
+
+    virtual void print(std::ostream& out, bool verbose,
+                       const std::string& indent) const;
+
+    const ArrayFieldValue & getValues() const { return *_values; }
+
+    DECLARE_IDENTIFIABLE(AddFieldPathUpdate);
+    ACCEPT_UPDATE_VISITOR;
+
+private:
+    uint8_t getSerializedType() const override { return AddMagic; }
+    virtual void deserialize(const DocumentTypeRepo& repo,
+                             const DataType& type,
+                             ByteBuffer& buffer, uint16_t version);
+
+    class AddIteratorHandler : public FieldValue::IteratorHandler
+    {
+    public:
+        AddIteratorHandler(const ArrayFieldValue& values)
+            : _values(values)
+        {
+        }
+
+        ModificationStatus doModify(FieldValue& fv);
+
+        bool createMissingPath() const { return true; }
+
+        bool onComplex(const Content&) { return false; }
+    private:
+        const ArrayFieldValue& _values;
+    };
+
+    std::unique_ptr<FieldValue::IteratorHandler> getIteratorHandler(Document&) const {
+        return std::unique_ptr<FieldValue::IteratorHandler>(
+                new AddIteratorHandler(*_values));
+    }
+
+    vespalib::CloneablePtr<ArrayFieldValue> _values;
+};
+
+
+} // ns document
+
diff --git a/document/src/vespa/document/update/addvalueupdate.cpp b/document/src/vespa/document/update/addvalueupdate.cpp
new file mode 100644
index 00000000000..10a0e76872f
--- /dev/null
+++ b/document/src/vespa/document/update/addvalueupdate.cpp
@@ -0,0 +1,111 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".document.addvalueupdate");
+
+#include <vespa/document/base/field.h>
+#include <vespa/document/datatype/arraydatatype.h>
+#include <vespa/document/datatype/weightedsetdatatype.h>
+#include <vespa/document/fieldvalue/fieldvalues.h>
+#include <vespa/document/repo/fixedtyperepo.h>
+#include <vespa/document/serialization/vespadocumentdeserializer.h>
+#include <vespa/document/update/addvalueupdate.h>
+#include <vespa/document/util/serializable.h>
+#include <vespa/vespalib/objects/nbostream.h>
+
+using vespalib::IllegalArgumentException;
+using vespalib::IllegalStateException;
+using vespalib::nbostream;
+
+namespace document
+{
+
+IMPLEMENT_IDENTIFIABLE(AddValueUpdate, ValueUpdate);
+
+bool
+AddValueUpdate::operator==(const ValueUpdate& other) const
+{
+    if (other.getClass().id() != AddValueUpdate::classId) return false;
+    const AddValueUpdate& o(static_cast<const AddValueUpdate&>(other));
+    if (*_value != *o._value) return false;
+    if (_weight != o._weight) return false;
+    return true;
+}
+
+// Ensure that this update is compatible with given field.
+void
+AddValueUpdate::checkCompatibility(const Field& field) const
+{
+    if (field.getDataType().inherits(CollectionDataType::classId)) {
+        const CollectionDataType& type(
+                static_cast<const CollectionDataType&>(field.getDataType()));
+        if (!type.getNestedType().isValueType(*_value)) {
+            throw IllegalArgumentException(
+                "Cannot add value of type "
+                + _value->getDataType()->toString() + " to field "
+                + field.getName().c_str() + " of container type "
+                + field.getDataType().toString(), VESPA_STRLOC);
+        }
+    } else {
+        throw IllegalArgumentException(
+            "Can not add a value to field of type"
+            + field.getDataType().toString(), VESPA_STRLOC);
+    }
+}
+
+// Print this update in human readable form.
+void
+AddValueUpdate::print(std::ostream& out, bool, const std::string& indent) const
+{
+    out << indent << "AddValueUpdate(" << *_value << ", " << _weight << ")";
+}
+
+// Apply this update to the given document.
+bool
+AddValueUpdate::applyTo(FieldValue& value) const
+{
+    if (value.inherits(ArrayFieldValue::classId)) {
+        ArrayFieldValue& doc(static_cast<ArrayFieldValue&>(value));
+        doc.add(*_value);	
+    } else if (value.inherits(WeightedSetFieldValue::classId)) {
+        WeightedSetFieldValue& doc(static_cast<WeightedSetFieldValue&>(value));
+        doc.add(*_value, _weight);	
+    } else {
+        std::string err = vespalib::make_string(
+                "Unable to add a value to a \"%s\" field value.",
+                value.getClass().name());
+        throw IllegalStateException(err, VESPA_STRLOC);
+    }
+    return true;
+}
+
+void
+AddValueUpdate::printXml(XmlOutputStream& xos) const
+{
+    xos << XmlTag("add") << XmlAttribute("weight", _weight)
+        << *_value
+        << XmlEndTag();
+}
+
+// Deserialize this update from the given buffer.
+void
+AddValueUpdate::deserialize(const DocumentTypeRepo& repo,
+                            const DataType& type,
+                            ByteBuffer& buffer, uint16_t version)
+{
+    const CollectionDataType* ctype =
+        Identifiable::cast<const CollectionDataType*>(&type);
+    if (ctype == NULL) {
+        throw DeserializeException("Can not perform add operation on "
+                                   "non-collection type.");
+    }
+    _value.reset(ctype->getNestedType().createFieldValue().release());
+    nbostream stream(buffer.getBufferAtPos(), buffer.getRemaining());
+    VespaDocumentDeserializer deserializer(repo, stream, version);
+    deserializer.read(*_value);
+    buffer.incPos(buffer.getRemaining() - stream.size());
+    buffer.getIntNetwork(_weight);
+}
+
+}
+
diff --git a/document/src/vespa/document/update/addvalueupdate.h b/document/src/vespa/document/update/addvalueupdate.h
new file mode 100644
index 00000000000..e112d6a504a
--- /dev/null
+++ b/document/src/vespa/document/update/addvalueupdate.h
@@ -0,0 +1,83 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * @class document::AddValueUpdate
+ * @ingroup document
+ *
+ * @brief Represents an update that specifies an addition to a field value.
+ */
+#pragma once
+
+#include <vespa/document/fieldvalue/fieldvalue.h>
+#include <vespa/document/update/valueupdate.h>
+
+namespace document {
+
+class AddValueUpdate : public ValueUpdate {
+    FieldValue::CP _value; // The field value to add by this update.
+    int _weight; // The weight to assign to the contained value.
+
+    // Used by ValueUpdate's static factory function
+    // Private because it generates an invalid object.
+    friend class ValueUpdate;
+    AddValueUpdate() : ValueUpdate(), _value(0), _weight(1) {}
+    ACCEPT_UPDATE_VISITOR;
+public:
+    typedef std::unique_ptr<AddValueUpdate> UP;
+
+    /**
+     * The default constructor requires initial values for all member variables.
+     *
+     * @param value The field value to add.
+     * @param weight The weight for the field value.
+     */
+    AddValueUpdate(const FieldValue& value, int weight = 1)
+        : ValueUpdate(),
+          _value(value.clone()),
+          _weight(weight) {}
+
+    virtual bool operator==(const ValueUpdate& other) const;
+
+    /** @return the field value to add during this update. */
+    const FieldValue& getValue() const { return *_value; }
+
+    /** @return The weight to assign to the value of this. */
+    int getWeight() const { return _weight; }
+
+    /**
+     * Sets the field value to add during this update.
+     *
+     * @param value The new field value.
+     * @return A reference to this object so you can chain calls.
+     */
+    AddValueUpdate& setValue(const FieldValue& value) {
+        _value.reset(value.clone());
+        return *this;
+    }
+
+    /**
+     * Sets the weight to assign to the value of this.
+     *
+     * @return A reference to this object so you can chain calls.
+     */
+    AddValueUpdate& setWeight(int weight) {
+        _weight = weight;
+        return *this;
+    }
+
+    // ValueUpdate implementation
+    virtual void checkCompatibility(const Field& field) const;
+    virtual bool applyTo(FieldValue& value) const;
+    virtual void printXml(XmlOutputStream& xos) const;
+    virtual void print(std::ostream& out, bool verbose,
+                       const std::string& indent) const;
+    virtual void deserialize(const DocumentTypeRepo& repo,
+                             const DataType& type,
+                             ByteBuffer& buffer, uint16_t version);
+    virtual AddValueUpdate* clone() const { return new AddValueUpdate(*this); }
+
+    DECLARE_IDENTIFIABLE(AddValueUpdate);
+
+};
+
+} // document
+
diff --git a/document/src/vespa/document/update/arithmeticvalueupdate.cpp b/document/src/vespa/document/update/arithmeticvalueupdate.cpp
new file mode 100644
index 00000000000..d01a85a5e70
--- /dev/null
+++ b/document/src/vespa/document/update/arithmeticvalueupdate.cpp
@@ -0,0 +1,140 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/document/base/field.h>
+#include <vespa/document/update/arithmeticvalueupdate.h>
+#include <vespa/document/fieldvalue/fieldvalues.h>
+
+using vespalib::IllegalArgumentException;
+using vespalib::IllegalStateException;
+
+namespace document
+{
+
+IMPLEMENT_IDENTIFIABLE(ArithmeticValueUpdate, ValueUpdate);
+
+// Declare string representations for operator names.
+static const char * operatorName[]  = { "add", "div", "mul", "sub" };
+static const char * operatorNameC[] = { "Add", "Div", "Mul", "Sub" };
+
+bool
+ArithmeticValueUpdate::operator==(const ValueUpdate& other) const
+{
+    if (other.getClass().id() != ArithmeticValueUpdate::classId) return false;
+    const ArithmeticValueUpdate& o(
+            static_cast<const ArithmeticValueUpdate&>(other));
+    if (_operator != o._operator) return false;
+    if (_operand != o._operand) return false;
+    return true;
+}
+
+// Ensure that this update is compatible with given field.
+void
+ArithmeticValueUpdate::checkCompatibility(const Field& field) const
+{
+    if ( ! field.getDataType().inherits(NumericDataType::classId)) {
+        throw IllegalArgumentException(vespalib::make_string(
+                "Can not perform arithmetic update on non-numeric field '%s'.",
+                field.getName().c_str()), VESPA_STRLOC);	
+    }
+}
+
+// Apply this update.
+bool
+ArithmeticValueUpdate::applyTo(FieldValue& value) const
+{
+    if (value.inherits(ByteFieldValue::classId)) {
+        ByteFieldValue& bValue = static_cast<ByteFieldValue&>(value);
+        bValue.setValue((int)applyTo(static_cast<int64_t>(bValue.getAsInt())));
+    } else if (value.inherits(DoubleFieldValue::classId)) {
+        DoubleFieldValue& dValue = static_cast<DoubleFieldValue&>(value);
+        dValue.setValue(applyTo(dValue.getAsDouble()));
+    } else if (value.inherits(FloatFieldValue::classId)) {
+        FloatFieldValue& fValue = static_cast<FloatFieldValue&>(value);
+        fValue.setValue((float)applyTo(fValue.getAsFloat()));
+    } else if (value.inherits(IntFieldValue::classId)) {
+        IntFieldValue& iValue = static_cast<IntFieldValue&>(value);
+        iValue.setValue((int)applyTo(static_cast<int64_t>(iValue.getAsInt())));
+    } else if (value.inherits(LongFieldValue::classId)) {
+        LongFieldValue& lValue = static_cast<LongFieldValue&>(value);
+        lValue.setValue(applyTo(lValue.getAsLong()));
+    } else {
+        std::string err = vespalib::make_string(
+                "Unable to perform an arithmetic update on a \"%s\" field "
+                "value.", value.getClass().name());
+        throw IllegalStateException(err, VESPA_STRLOC);
+    }
+    return true;
+}
+
+// Perform the contained operation on the given value.
+double
+ArithmeticValueUpdate::applyTo(double value) const
+{
+    switch(_operator) {
+    case Add:
+        return value + _operand;
+    case Div:
+        return value / _operand;
+    case Mul:
+        return value * _operand;
+    case Sub:
+        return value - _operand;
+    default:
+        return 0;
+    }
+}
+
+// Perform the contained operation on the given value.
+long
+ArithmeticValueUpdate::applyTo(int64_t value) const
+{
+    switch(_operator) {
+    case Add:
+        return (long)(value + _operand);
+    case Div:
+        return (long)(value / _operand);
+    case Mul:
+        return (long)(value * _operand);
+    case Sub:
+        return (long)(value - _operand);
+    default:
+        return 0;
+    }
+}
+
+// Perform the contained operation on the given value.
+std::string
+ArithmeticValueUpdate::applyTo(const std::string & value) const
+{
+    return value;
+}
+
+// Print this update as a human readable string.
+void
+ArithmeticValueUpdate::print(std::ostream& out, bool, const std::string& indent) const
+{
+    out << indent << "ArithmeticValueUpdate(" << operatorNameC[_operator]
+        << " " << _operand << ")";
+}
+
+void
+ArithmeticValueUpdate::printXml(XmlOutputStream& xos) const
+{
+    xos << XmlTag(operatorName[_operator])
+        << XmlAttribute("by", _operand)
+        << XmlEndTag();
+}
+
+// Deserialize this update from the given buffer.
+void
+ArithmeticValueUpdate::deserialize(
+        const DocumentTypeRepo&, const DataType&,
+        ByteBuffer& buffer, uint16_t)
+{
+    int32_t opt;
+    buffer.getIntNetwork(opt);
+    _operator = static_cast<ArithmeticValueUpdate::Operator>(opt);
+    buffer.getDoubleNetwork(_operand);
+}
+
+} // document
diff --git a/document/src/vespa/document/update/arithmeticvalueupdate.h b/document/src/vespa/document/update/arithmeticvalueupdate.h
new file mode 100644
index 00000000000..a335b09c650
--- /dev/null
+++ b/document/src/vespa/document/update/arithmeticvalueupdate.h
@@ -0,0 +1,107 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * @class document::ArithmeticValueUpdate
+ * @ingroup document
+ *
+ * @brief Represent an update that specifies an arithmetic operation that is to
+ *        be applied to the weight of a field value.
+ */
+#pragma once
+
+#include <vespa/document/update/valueupdate.h>
+
+namespace document {
+
+class ArithmeticValueUpdate : public ValueUpdate {
+public:
+    /** Declare all types of arithmetic value updates. */
+    enum Operator {
+        Add = 0, // Add the operand to the field value.
+        Div,     // Divide the field value with the operand.
+        Mul,     // Multiply the field value with the operand.
+        Sub,     // Subtract the operand from the field value.
+        MAX_NUM_OPERATORS
+    };
+
+private:
+    Operator _operator; // The operator of the arithmetic operation.
+    double _operand; // The operand of the arithmetic operation.
+
+    // Used by ValueUpdate's static factory function
+    // Private because it generates an invalid object.
+    friend class ValueUpdate;
+    ArithmeticValueUpdate()
+        : ValueUpdate(),
+          _operator(MAX_NUM_OPERATORS),
+          _operand(0.0) {}
+
+    ACCEPT_UPDATE_VISITOR;
+public:
+    typedef std::unique_ptr<ArithmeticValueUpdate> UP;
+
+    /**
+     * The default constructor requires initial values for all member variables.
+     *
+     * @param opt The operator of this arithmetic update.
+     * @param opn The operand for the operation.
+     */
+    ArithmeticValueUpdate(Operator opt, double opn)
+        : ValueUpdate(),
+          _operator(opt),
+          _operand(opn) {}
+
+    ArithmeticValueUpdate(const ArithmeticValueUpdate& update)
+        : ValueUpdate(update),
+          _operator(update._operator),
+          _operand(update._operand) {}
+
+    virtual bool operator==(const ValueUpdate& other) const;
+
+    /** @return the operator of this arithmetic update. */
+    Operator getOperator() const { return _operator; }
+
+    /** @return the operand of this arithmetic update. */
+    double getOperand() const { return _operand; }
+
+    /**
+     * Apply the contained operation on the given double.
+     *
+     * @param value The value to modify.
+     * @return The modified value.
+     */
+    double applyTo(double value) const;
+
+    /**
+     * Apply the contained operation on the given string.
+     *
+     * @param value The value to modify.
+     * @return The modified value.
+     */
+    std::string applyTo(const std::string& value) const;
+
+    /**
+     * Apply the contained operation on the given long.
+     *
+     * @param value The value to modify.
+     * @return The modified value.
+     */
+    long applyTo(int64_t value) const;
+
+    // ValueUpdate implementation
+    virtual void checkCompatibility(const Field& field) const;
+    virtual bool applyTo(FieldValue& value) const;
+    virtual void printXml(XmlOutputStream& xos) const;
+    virtual void print(std::ostream& out, bool verbose,
+                       const std::string& indent) const;
+    virtual void deserialize(const DocumentTypeRepo& repo,
+                             const DataType& type,
+                             ByteBuffer& buffer, uint16_t version);
+    virtual ArithmeticValueUpdate* clone() const
+        { return new ArithmeticValueUpdate(*this); }
+
+    DECLARE_IDENTIFIABLE(ArithmeticValueUpdate);
+
+};
+
+} // document
+
diff --git a/document/src/vespa/document/update/assignfieldpathupdate.cpp b/document/src/vespa/document/update/assignfieldpathupdate.cpp
new file mode 100644
index 00000000000..ed008494df0
--- /dev/null
+++ b/document/src/vespa/document/update/assignfieldpathupdate.cpp
@@ -0,0 +1,198 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/document/fieldvalue/fieldvalues.h>
+#include <vespa/document/repo/fixedtyperepo.h>
+#include <vespa/document/select/parser.h>
+#include <vespa/document/serialization/vespadocumentdeserializer.h>
+#include <vespa/document/update/assignfieldpathupdate.h>
+#include <vespa/log/log.h>
+#include <vespa/vespalib/objects/nbostream.h>
+
+using vespalib::nbostream;
+
+LOG_SETUP(".document.update.fieldpathupdate");
+
+namespace document {
+
+IMPLEMENT_IDENTIFIABLE(AssignFieldPathUpdate, FieldPathUpdate);
+
+AssignFieldPathUpdate::AssignFieldPathUpdate()
+    : FieldPathUpdate(),
+      _repo(),
+      _newValue(),
+      _expression(),
+      _removeIfZero(false),
+      _createMissingPath(false)
+{
+}
+
+
+AssignFieldPathUpdate::AssignFieldPathUpdate(
+        const DocumentTypeRepo& repo,
+        const DataType& type,
+        stringref fieldPath,
+        stringref whereClause,
+        const FieldValue& newValue)
+    : FieldPathUpdate(repo, type, fieldPath, whereClause),
+      _repo(&repo),
+      _newValue(newValue.clone()),
+      _expression(),
+      _removeIfZero(false),
+      _createMissingPath(true)
+{
+    checkCompatibility(*_newValue);
+}
+
+AssignFieldPathUpdate::AssignFieldPathUpdate(
+        const DocumentTypeRepo& repo,
+        const DataType& type,
+        stringref fieldPath,
+        stringref whereClause,
+        stringref expression)
+    : FieldPathUpdate(repo, type, fieldPath, whereClause),
+      _repo(&repo),
+      _newValue(),
+      _expression(expression),
+      _removeIfZero(false),
+      _createMissingPath(true)
+{
+    if (_expression.empty()) {
+        throw vespalib::IllegalArgumentException("Cannot create an arithmetic "
+                "assignment update with an empty expression", VESPA_STRLOC);
+    }
+}
+
+std::unique_ptr<FieldValue::IteratorHandler>
+AssignFieldPathUpdate::getIteratorHandler(Document& doc) const
+{
+    if (!_expression.empty()) {
+        return std::unique_ptr<FieldValue::IteratorHandler>(
+                new AssignExpressionIteratorHandler(
+                        *_repo, doc, _expression, _removeIfZero, _createMissingPath));
+    } else {
+        return std::unique_ptr<FieldValue::IteratorHandler>(
+                new AssignValueIteratorHandler(
+                        *_newValue, _removeIfZero, _createMissingPath));
+    }
+}
+
+
+FieldValue::IteratorHandler::ModificationStatus
+AssignFieldPathUpdate::AssignValueIteratorHandler::doModify(FieldValue& fv) {
+    LOG(spam, "fv = %s", fv.toString().c_str());
+    if (!(*fv.getDataType() == *_newValue.getDataType())) {
+        std::string err = vespalib::make_string(
+                "Trying to assign \"%s\" of type %s to an instance of type %s",
+                _newValue.toString().c_str(), _newValue.getClass().name(),
+                fv.getClass().name());
+        throw vespalib::IllegalArgumentException(err, VESPA_STRLOC);
+    }
+    if (_removeIfZero
+        && _newValue.inherits(NumericFieldValueBase::classId)
+        && static_cast<const NumericFieldValueBase&>(_newValue).getAsLong() == 0)
+    {
+        return REMOVED;
+    }
+    fv.assign(_newValue);
+    return MODIFIED;
+}
+
+FieldValue::IteratorHandler::ModificationStatus
+AssignFieldPathUpdate::AssignExpressionIteratorHandler::doModify(FieldValue& fv) {
+    LOG(spam, "fv = %s", fv.toString().c_str());
+    if (fv.inherits(NumericFieldValueBase::classId)) {
+        DocumentCalculator::VariableMap vars;
+        for (VariableMap::const_iterator i(getVariables().begin()),
+                 e(getVariables().end()); i != e; ++i)
+        {
+            if (i->second.key.get() && i->second.key->inherits(NumericFieldValueBase::classId)) {
+                vars[i->first] = i->second.key->getAsDouble();
+            } else {
+                vars[i->first] = i->second.index;
+            }
+        }
+
+        vars["value"] = fv.getAsDouble();
+
+        try {
+            double res = _calc.evaluate(_doc, vars);
+            if (_removeIfZero && static_cast<uint64_t>(res) == 0) {
+                return REMOVED;
+            } else {
+                fv.assign(DoubleFieldValue(res));
+            }
+        } catch (const vespalib::IllegalArgumentException&) {
+            // Divide by zero does not modify the document field
+            return NOT_MODIFIED;
+        } catch (const boost::bad_numeric_cast&) {
+            // Underflow/overflow does not modify
+            return NOT_MODIFIED;
+        }
+    } else {
+        throw vespalib::IllegalArgumentException(
+                vespalib::make_string("Trying to perform arithmetic on %s of type %s",
+                                      fv.toString().c_str(), fv.getDataType()->toString().c_str()),
+                VESPA_STRLOC);
+    }
+    return MODIFIED;
+}
+
+bool
+AssignFieldPathUpdate::operator==(const FieldPathUpdate& other) const
+{
+    if (other.getClass().id() != AssignFieldPathUpdate::classId) return false;
+    if (!FieldPathUpdate::operator==(other)) return false;
+    const AssignFieldPathUpdate& assignOther
+        = static_cast<const AssignFieldPathUpdate&>(other);
+    if (assignOther._newValue.get() && _newValue.get()) {
+        if (*assignOther._newValue != *_newValue) return false;
+    }
+    // else: should always have at least 1 with non-empty expression
+    return (assignOther._expression == _expression)
+            && (assignOther._removeIfZero == _removeIfZero)
+            && (assignOther._createMissingPath == _createMissingPath);
+}
+
+void
+AssignFieldPathUpdate::print(std::ostream& out, bool verbose,
+                             const std::string& indent) const
+{
+    out << "AssignFieldPathUpdate(\n";
+    FieldPathUpdate::print(out, verbose, indent + "  ");
+    if (_newValue.get()) {
+        out << ",\n" << indent << "  " << "newValue=";
+        _newValue->print(out, verbose, indent + "  ");
+    } else {
+        out << ",\n" << indent << "  " << "expression='" << _expression << "'";
+    }
+    out << ", removeIfZero=" << (_removeIfZero ? "yes" : "no")
+        << ", createMissingPath=" << (_createMissingPath ? "yes" : "no")
+        << "\n" << indent << ")";
+}
+
+void
+AssignFieldPathUpdate::deserialize(const DocumentTypeRepo& repo,
+                                   const DataType& type,
+                                   ByteBuffer& buffer, uint16_t version)
+{
+    FieldPathUpdate::deserialize(repo, type, buffer, version);
+    _repo = &repo;
+
+    uint8_t flags = 0x00;
+    buffer.getByte(flags);
+
+    _removeIfZero = (flags & REMOVE_IF_ZERO) != 0;
+    _createMissingPath = (flags & CREATE_MISSING_PATH) != 0;
+
+    if (flags & ARITHMETIC_EXPRESSION) {
+        _expression = getString(buffer);
+    } else {
+        _newValue.reset(getResultingDataType().createFieldValue().release());
+        nbostream stream(buffer.getBufferAtPos(), buffer.getRemaining());
+        VespaDocumentDeserializer deserializer(*_repo, stream, version);
+        deserializer.read(*_newValue);
+        buffer.incPos(buffer.getRemaining() - stream.size());
+    }
+}
+
+} // ns document
diff --git a/document/src/vespa/document/update/assignfieldpathupdate.h b/document/src/vespa/document/update/assignfieldpathupdate.h
new file mode 100644
index 00000000000..ff54f76f4ed
--- /dev/null
+++ b/document/src/vespa/document/update/assignfieldpathupdate.h
@@ -0,0 +1,124 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/document/base/documentcalculator.h>
+#include <vespa/document/update/fieldpathupdate.h>
+
+namespace document {
+
+class AssignFieldPathUpdate : public FieldPathUpdate
+{
+public:
+    enum SerializationFlag
+    {
+        ARITHMETIC_EXPRESSION = 1,
+        REMOVE_IF_ZERO = 2,
+        CREATE_MISSING_PATH = 4
+    };
+
+    /** For deserialization */
+    AssignFieldPathUpdate();
+
+    AssignFieldPathUpdate(const DocumentTypeRepo& repo,
+                          const DataType& type,
+                          stringref fieldPath,
+                          stringref whereClause,
+                          const FieldValue& newValue);
+
+    AssignFieldPathUpdate(const DocumentTypeRepo& repo,
+                          const DataType& type,
+                          stringref fieldPath,
+                          stringref whereClause,
+                          stringref expression);
+
+    void setRemoveIfZero(bool removeIfZero) {
+        _removeIfZero = removeIfZero;
+    }
+    bool getRemoveIfZero() const { return _removeIfZero; }
+    void setCreateMissingPath(bool createMissingPath) {
+        _createMissingPath = createMissingPath;
+    }
+    bool getCreateMissingPath() const { return _createMissingPath; }
+    const vespalib::string& getExpression() const { return _expression; }
+    bool hasValue() const { return _newValue.get() != nullptr; }
+    const FieldValue & getValue() const { return *_newValue; }
+
+    FieldPathUpdate* clone() const { return new AssignFieldPathUpdate(*this); }
+
+    bool operator==(const FieldPathUpdate& other) const;
+
+    virtual void print(std::ostream& out, bool verbose,
+                       const std::string& indent) const;
+
+    DECLARE_IDENTIFIABLE(AssignFieldPathUpdate);
+    ACCEPT_UPDATE_VISITOR;
+
+private:
+    uint8_t getSerializedType() const override { return AssignMagic; }
+    virtual void deserialize(const DocumentTypeRepo& repo,
+                             const DataType& type,
+                             ByteBuffer& buffer, uint16_t version);
+
+    class AssignValueIteratorHandler : public FieldValue::IteratorHandler
+    {
+    public:
+        AssignValueIteratorHandler(const FieldValue& newValue,
+                              bool removeIfZero,
+                              bool createMissingPath_)
+            : _newValue(newValue), _removeIfZero(removeIfZero),
+            _createMissingPath(createMissingPath_)
+        {
+        }
+
+        ModificationStatus doModify(FieldValue& fv);
+
+        bool onComplex(const Content&) { return false; }
+
+        bool createMissingPath() const { return _createMissingPath; }
+
+    private:
+        const FieldValue& _newValue;
+        bool _removeIfZero;
+        bool _createMissingPath;
+    };
+
+    class AssignExpressionIteratorHandler : public FieldValue::IteratorHandler
+    {
+    public:
+        AssignExpressionIteratorHandler(
+                const DocumentTypeRepo& repo,
+                Document& doc,
+                const vespalib::string& expression,
+                bool removeIfZero,
+                bool createMissingPath_)
+            : _calc(repo, expression),
+            _doc(doc),
+            _removeIfZero(removeIfZero),
+            _createMissingPath(createMissingPath_)
+        {
+        }
+
+        ModificationStatus doModify(FieldValue& fv);
+
+        bool onComplex(const Content&) { return false; }
+
+        bool createMissingPath() const { return _createMissingPath; }
+
+    private:
+        DocumentCalculator _calc;
+        Document& _doc;
+        bool _removeIfZero;
+        bool _createMissingPath;
+    };
+
+    std::unique_ptr<FieldValue::IteratorHandler> getIteratorHandler(Document& doc) const;
+
+    const DocumentTypeRepo *_repo;
+    FieldValue::CP _newValue;
+    vespalib::string _expression;
+    bool _removeIfZero;
+    bool _createMissingPath;
+};
+
+} // ns document
+
diff --git a/document/src/vespa/document/update/assignvalueupdate.cpp b/document/src/vespa/document/update/assignvalueupdate.cpp
new file mode 100644
index 00000000000..b389f015eb0
--- /dev/null
+++ b/document/src/vespa/document/update/assignvalueupdate.cpp
@@ -0,0 +1,100 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/document/base/field.h>
+#include <vespa/document/fieldvalue/fieldvalues.h>
+#include <vespa/document/repo/fixedtyperepo.h>
+#include <vespa/document/serialization/vespadocumentdeserializer.h>
+#include <vespa/document/update/assignvalueupdate.h>
+#include <vespa/vespalib/objects/nbostream.h>
+
+using vespalib::IllegalArgumentException;
+using vespalib::IllegalStateException;
+using vespalib::nbostream;
+
+namespace document
+{
+
+IMPLEMENT_IDENTIFIABLE(AssignValueUpdate, ValueUpdate);
+
+// Declare content bits.
+static const unsigned char CONTENT_HASVALUE = 0x01;
+
+bool
+AssignValueUpdate::operator==(const ValueUpdate& other) const
+{
+    if (other.getClass().id() != AssignValueUpdate::classId) return false;
+    const AssignValueUpdate& o(static_cast<const AssignValueUpdate&>(other));
+    return _value == o._value;
+}
+
+// Ensure that this update is compatible with given field.
+void
+AssignValueUpdate::checkCompatibility(const Field& field) const
+{
+        // If equal datatype we know it is ok.
+    if (!_value || field.getDataType().isValueType(*_value)) {
+        return;
+    }
+        // Deny all assignments to non-equal types
+    throw IllegalArgumentException(vespalib::make_string(
+            "Failed to assign field value of type %s to value of type %s.",
+            _value->getDataType()->toString().c_str(),
+            field.getDataType().toString().c_str()), VESPA_STRLOC);
+}
+
+// Print this update as a human readable string.
+void
+AssignValueUpdate::print(std::ostream& out, bool verbose, const std::string& indent) const
+{
+    out << indent << "AssignValueUpdate(";
+    if (_value) _value->print(out, verbose, indent);
+    out << ")";
+}
+
+// Apply this update to the given document.
+bool
+AssignValueUpdate::applyTo(FieldValue& value) const
+{
+    if (_value && (_value->getDataType() != value.getDataType())) {
+        vespalib::string err = vespalib::make_string(
+                "Unable to assign a \"%s\" value to a \"%s\" field value.",
+                _value->getClass().name(), value.getClass().name());
+        throw IllegalStateException(err, VESPA_STRLOC);	
+    }
+    if (_value) {
+        value.assign(*_value);
+    }
+    return bool(_value);
+}
+
+void
+AssignValueUpdate::printXml(XmlOutputStream& xos) const
+{
+    xos << XmlTag("assign");
+    if (_value) {
+        _value->printXml(xos);
+    }
+    xos << XmlEndTag();
+}
+
+// Deserialize this update from the given buffer.
+void
+AssignValueUpdate::deserialize(
+        const DocumentTypeRepo& repo, const DataType& type,
+        ByteBuffer& buffer, uint16_t version)
+{
+    // Read content bit vector.
+    unsigned char content = 0x00;
+    buffer.getByte(content);
+
+    // Read field value, if any.
+    if (content & CONTENT_HASVALUE) {
+        _value.reset(type.createFieldValue().release());
+        nbostream stream(buffer.getBufferAtPos(), buffer.getRemaining());
+        VespaDocumentDeserializer deserializer(repo, stream, version);
+        deserializer.read(*_value);
+        buffer.incPos(buffer.getRemaining() - stream.size());
+    }
+}
+
+}  // namespace document
diff --git a/document/src/vespa/document/update/assignvalueupdate.h b/document/src/vespa/document/update/assignvalueupdate.h
new file mode 100644
index 00000000000..221642df7b4
--- /dev/null
+++ b/document/src/vespa/document/update/assignvalueupdate.h
@@ -0,0 +1,67 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * @class document::AssignValueUpdate
+ * @ingroup document
+ *
+ * @brief Represents an update that specifies an assignment of a value to a
+ *        field, completely overwriting the previous value.
+ *
+ * Note that for multi-value types, the entire collection is overwritten with a
+ * new collection.
+ */
+#pragma once
+
+#include <vespa/document/fieldvalue/fieldvalue.h>
+#include <vespa/document/update/valueupdate.h>
+
+namespace document {
+
+class AssignValueUpdate : public ValueUpdate {
+    FieldValue::CP _value;
+
+    ACCEPT_UPDATE_VISITOR;
+public:
+    typedef std::unique_ptr<AssignValueUpdate> UP;
+
+    AssignValueUpdate() : ValueUpdate(), _value() {}
+
+    AssignValueUpdate(const FieldValue& value)
+        : ValueUpdate(),
+          _value(value.clone())
+    {
+    }
+
+    virtual bool operator==(const ValueUpdate& other) const;
+
+    /** @return The field value to assign during this update. */
+    bool hasValue() const { return bool(_value); }
+    const FieldValue& getValue() const { return *_value; }
+    const FieldValue* getValuePtr() const { return _value.get(); }
+
+    /**
+     * Sets the field value to assign during this update.
+     * @return A reference to this.
+     */
+    AssignValueUpdate& setValue(const FieldValue* value) {
+        _value.reset(value ? value->clone() : 0);
+        return *this;
+    }
+
+    // ValueUpdate implementation.
+    virtual void checkCompatibility(const Field& field) const;
+    virtual bool applyTo(FieldValue& value) const;
+    virtual void printXml(XmlOutputStream& xos) const;
+    virtual void print(std::ostream& out, bool verbose,
+                       const std::string& indent) const;
+    virtual void deserialize(const DocumentTypeRepo& repo,
+                             const DataType& type,
+                             ByteBuffer& buffer, uint16_t version);
+    virtual AssignValueUpdate* clone() const
+        { return new AssignValueUpdate(*this); }
+
+    DECLARE_IDENTIFIABLE(AssignValueUpdate);
+
+};
+
+} // document
+
diff --git a/document/src/vespa/document/update/clearvalueupdate.cpp b/document/src/vespa/document/update/clearvalueupdate.cpp
new file mode 100644
index 00000000000..0b062e6bd92
--- /dev/null
+++ b/document/src/vespa/document/update/clearvalueupdate.cpp
@@ -0,0 +1,55 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/document/base/field.h>
+#include <vespa/document/update/clearvalueupdate.h>
+#include <vespa/document/fieldvalue/document.h>
+
+using vespalib::IllegalArgumentException;
+using vespalib::IllegalStateException;
+
+namespace document
+{
+
+IMPLEMENT_IDENTIFIABLE(ClearValueUpdate, ValueUpdate);
+
+bool
+ClearValueUpdate::operator==(const ValueUpdate& other) const
+{
+    return (other.getClass().id() == ClearValueUpdate::classId);
+}
+
+// Ensure that this update is compatible with given field.
+void
+ClearValueUpdate::checkCompatibility(const Field&) const
+{
+}
+
+// Apply this update to the given document.
+bool
+ClearValueUpdate::applyTo(FieldValue& value) const
+{
+    (void) value;
+    return false;
+}
+
+void
+ClearValueUpdate::printXml(XmlOutputStream& xos) const
+{
+    xos << XmlTag("clear") << XmlEndTag();
+}
+
+// Print this update in human readable form.
+void
+ClearValueUpdate::print(std::ostream& out, bool, const std::string&) const
+{
+    out << "ClearValueUpdate()";
+}
+
+// Deserialize this update from the given buffer.
+void
+ClearValueUpdate::deserialize(const DocumentTypeRepo&, const DataType&,
+                             ByteBuffer&, uint16_t)
+{
+}
+
+}
diff --git a/document/src/vespa/document/update/clearvalueupdate.h b/document/src/vespa/document/update/clearvalueupdate.h
new file mode 100644
index 00000000000..a0cde78b6ac
--- /dev/null
+++ b/document/src/vespa/document/update/clearvalueupdate.h
@@ -0,0 +1,41 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * @class document::ClearValueUpdate
+ * @ingroup document
+ *
+ * @brief Represents an update that clears the content of a field value.
+ */
+#pragma once
+
+#include <vespa/document/update/valueupdate.h>
+
+namespace document {
+
+class ClearValueUpdate : public ValueUpdate {
+    ACCEPT_UPDATE_VISITOR;
+public:
+    typedef std::unique_ptr<ClearValueUpdate> UP;
+
+    ClearValueUpdate() : ValueUpdate() {}
+
+    ClearValueUpdate(const ClearValueUpdate& update) : ValueUpdate(update) {}
+
+    virtual bool operator==(const ValueUpdate& other) const;
+
+    // ValueUpdate implementation
+    virtual void checkCompatibility(const Field& field) const;
+    virtual bool applyTo(FieldValue& value) const;
+    virtual void printXml(XmlOutputStream& xos) const;
+    virtual void print(std::ostream& out, bool verbose,
+                       const std::string& indent) const;
+    virtual void deserialize(const DocumentTypeRepo& repo,
+                             const DataType& type,
+                             ByteBuffer& buffer, uint16_t version);
+    virtual ClearValueUpdate* clone() const
+        { return new ClearValueUpdate(*this); }
+
+    DECLARE_IDENTIFIABLE(ClearValueUpdate);
+};
+
+} // document
+
diff --git a/document/src/vespa/document/update/documentupdate.cpp b/document/src/vespa/document/update/documentupdate.cpp
new file mode 100644
index 00000000000..8b7b2c8020e
--- /dev/null
+++ b/document/src/vespa/document/update/documentupdate.cpp
@@ -0,0 +1,333 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/document/update/documentupdate.h>
+#include <vespa/document/datatype/arraydatatype.h>
+#include <vespa/document/datatype/datatype.h>
+#include <vespa/document/datatype/weightedsetdatatype.h>
+#include <vespa/document/fieldvalue/fieldvalues.h>
+#include <vespa/document/repo/documenttyperepo.h>
+#include <vespa/document/serialization/vespadocumentserializer.h>
+#include <vespa/document/update/documentupdateflags.h>
+#include <vespa/document/update/fieldupdate.h>
+#include <vespa/document/update/valueupdate.h>
+#include <vespa/vespalib/objects/nbostream.h>
+
+using vespalib::IllegalArgumentException;
+using vespalib::IllegalStateException;
+using vespalib::nbostream;
+using vespalib::make_string;
+using vespalib::string;
+
+namespace document {
+
+IMPLEMENT_IDENTIFIABLE(DocumentUpdate, vespalib::Identifiable);
+
+// Declare content bits.
+static const unsigned char CONTENT_HASTYPE = 0x01;
+
+typedef std::vector<FieldUpdate> FieldUpdateList;
+typedef std::vector<FieldPathUpdate::CP> FieldPathUpdateList;
+
+DocumentUpdate::DocumentUpdate()
+    : _documentId("doc::"),
+      _type(DataType::DOCUMENT),
+      _updates(),
+      _version(Document::getNewestSerializationVersion()),
+      _createIfNonExistent(false)
+{
+}
+
+DocumentUpdate::DocumentUpdate(const DataType &type, const DocumentId& id)
+    : _documentId(id),
+      _type(&type),
+      _updates(),
+      _version(Document::getNewestSerializationVersion()),
+      _createIfNonExistent(false)
+{
+    if (!type.getClass().inherits(DocumentType::classId)) {
+        throw IllegalArgumentException("Cannot generate a document with non-document type " + type.toString() + ".",
+                                       VESPA_STRLOC);
+    }
+}
+
+DocumentUpdate::DocumentUpdate(const DocumentTypeRepo& repo,
+                               ByteBuffer& buffer,
+                               SerializeVersion serializeVersion)
+    : _documentId("doc::"),
+      _type(DataType::DOCUMENT),
+      _updates(),
+      _version(Document::getNewestSerializationVersion()),
+      _createIfNonExistent(false)
+{
+    switch (serializeVersion) {
+    case SerializeVersion::SERIALIZE_HEAD:
+        deserializeHEAD(repo, buffer);
+        break;
+    case SerializeVersion::SERIALIZE_42:
+        deserialize42(repo, buffer);
+        break;
+    default:
+        throw IllegalArgumentException("bad serializeVersion provided.", VESPA_STRLOC);
+    }
+}
+
+
+bool
+DocumentUpdate::operator==(const DocumentUpdate& other) const
+{
+    if (_documentId != other._documentId) return false;
+    if (*_type != *other._type) return false;
+    if (_updates.size() != other._updates.size()) return false;
+    for (std::size_t i = 0, n = _updates.size(); i < n; ++i) {
+        if (_updates[i] != other._updates[i]) return false;
+    }
+    if (_fieldPathUpdates.size() != other._fieldPathUpdates.size()) return false;
+    for (std::size_t i = 0, n = _fieldPathUpdates.size(); i < n; ++i) {
+        if (*_fieldPathUpdates[i] != *other._fieldPathUpdates[i]) return false;
+    }
+    if (_createIfNonExistent != other._createIfNonExistent) return false;
+    return true;
+}
+
+bool
+DocumentUpdate::affectsDocumentBody() const
+{
+    for(const auto & update : _updates) {
+        if (!update.getField().isHeaderField()) {
+            return true;
+        }
+    }
+    for (const auto & update : _fieldPathUpdates) {
+        if (update->affectsDocumentBody()) {
+            return true;
+        }
+    }
+    return false;
+}
+
+// Print the content of this document update.
+void
+DocumentUpdate::print(std::ostream& out, bool verbose,
+                      const std::string& indent) const
+{
+    out << "DocumentUpdate(";
+    if (_type) {
+        _type->print(out, verbose, indent + "    ");
+    } else {
+        out << "No document type given";
+    }
+    std::string nestedIndent = indent + "  ";
+    out << "\n" << nestedIndent << "CreateIfNonExistent(" << (_createIfNonExistent ? "true" : "false") << ")";
+
+    for(const auto & update : _updates) {
+        out << "\n" << indent << "  ";
+        update.print(out, verbose, nestedIndent);
+    }
+    if (!_updates.empty()) {
+        out << "\n" << indent;
+    }
+    for (const auto & update : _fieldPathUpdates) {
+        out << "\n" << indent << "  ";
+        update->print(out, verbose, nestedIndent);
+    }
+    if (!_fieldPathUpdates.empty()) {
+        out << "\n" << indent;
+    }
+    out << ")";
+}
+
+// Apply this update to the given document.
+void
+DocumentUpdate::applyTo(Document& doc) const
+{
+    const DocumentType& type = doc.getType();
+    if (_type->getName() != type.getName()) {
+	string err = make_string("Can not apply a \"%s\" document update to a \"%s\" document.",
+                                 _type->getName().c_str(), type.getName().c_str());
+        throw IllegalArgumentException(err, VESPA_STRLOC);
+    }
+
+    // Apply legacy updates
+    for(const auto & update : _updates) {
+        update.applyTo(doc);
+    }
+    // Apply fieldpath updates
+    for (const auto & update : _fieldPathUpdates) {
+        update->applyTo(doc);
+    }
+}
+
+void
+DocumentUpdate::serialize42(nbostream &stream) const
+{
+    VespaDocumentSerializer serializer(stream);
+    serializer.write42(*this);
+}
+
+void
+DocumentUpdate::serializeHEAD(nbostream &stream) const
+{
+    VespaDocumentSerializer serializer(stream);
+    serializer.writeHEAD(*this);
+}
+
+int
+DocumentUpdate::serializeFlags(int size_) const
+{
+    DocumentUpdateFlags flags;
+    flags.setCreateIfNonExistent(_createIfNonExistent);
+    return flags.injectInto(size_);
+}
+
+namespace {
+    std::pair<const DocumentType *, DocumentId>
+    deserializeTypeAndId(const DocumentTypeRepo& repo, ByteBuffer& buffer) {
+        nbostream stream(buffer.getBufferAtPos(), buffer.getRemaining(),
+                         false);
+        DocumentId docId(stream);
+        buffer.incPos(stream.rp());
+
+        // Read content bit vector.
+        unsigned char content = 0x00;
+        buffer.getByte(content);
+
+        // Why on earth do we have this whether we have type part?
+        // We need type for object to work, so just throwing exception if it's
+        // not there.
+        if((content & CONTENT_HASTYPE) == 0) {
+            throw IllegalStateException("Missing document type", VESPA_STRLOC);
+        }
+
+        vespalib::stringref typestr = buffer.getBufferAtPos();
+        buffer.incPos(typestr.length() + 1);
+
+        int16_t version = 0;
+        buffer.getShortNetwork(version);
+        const DocumentType *type = repo.getDocumentType(typestr);
+        if (!type) {
+            throw DocumentTypeNotFoundException(typestr, VESPA_STRLOC);
+        }
+        return std::make_pair(type, docId);
+    }
+}
+
+// Deserialize the content of the given buffer into this document update.
+DocumentUpdate::UP
+DocumentUpdate::create42(const DocumentTypeRepo& repo, ByteBuffer& buffer)
+{
+    return std::make_unique<DocumentUpdate>(repo, buffer,
+                                            SerializeVersion::SERIALIZE_42);
+}
+
+DocumentUpdate::UP
+DocumentUpdate::createHEAD(const DocumentTypeRepo& repo, ByteBuffer& buffer)
+{
+    return std::make_unique<DocumentUpdate>(repo, buffer,
+                                            SerializeVersion::SERIALIZE_HEAD);
+}
+
+void
+DocumentUpdate::deserialize42(const DocumentTypeRepo& repo, ByteBuffer& buffer)
+{
+    int pos = buffer.getPos();
+    try{
+        buffer.getShortNetwork(_version);
+
+        std::pair<const DocumentType *, DocumentId> typeAndId(
+                deserializeTypeAndId(repo, buffer));
+        _type = typeAndId.first;
+        _documentId = typeAndId.second;
+        // Read field updates, if any.
+        if(buffer.getRemaining() > 0) {
+            int sizeAndFlags = 0;
+            buffer.getIntNetwork(sizeAndFlags);
+            int numUpdates = deserializeFlags(sizeAndFlags);
+            _updates.reserve(numUpdates);
+            for (int i = 0; i < numUpdates; i++) {
+                _updates.emplace_back(repo, *typeAndId.first, buffer, _version);
+            }
+        }
+    } catch (const DeserializeException &) {
+        buffer.setPos(pos);
+        throw;
+    } catch (const BufferOutOfBoundsException &) {
+        buffer.setPos(pos);
+        throw;
+    }
+}
+
+void
+DocumentUpdate::deserializeHEAD(const DocumentTypeRepo &repo, ByteBuffer &buffer)
+{
+    int pos = buffer.getPos();
+    try {
+        nbostream stream(buffer.getBufferAtPos(), buffer.getRemaining(), false);
+        _documentId = DocumentId(stream);
+        buffer.incPos(stream.rp());
+
+        vespalib::stringref typestr = buffer.getBufferAtPos();
+        buffer.incPos(typestr.length() + 1);
+
+        int16_t version = 0;
+        buffer.getShortNetwork(version);
+        const DocumentType *docType = repo.getDocumentType(typestr);
+        if (!docType) {
+            throw DocumentTypeNotFoundException(typestr, VESPA_STRLOC);
+        }
+        _type = docType;
+
+        // Read field updates, if any.
+        if (buffer.getRemaining() > 0) {
+            int numUpdates = 0;
+            buffer.getIntNetwork(numUpdates);
+            _updates.reserve(numUpdates);
+            for (int i = 0; i < numUpdates; i++) {
+                _updates.emplace_back(repo, *docType, buffer, 8);
+            }
+        }
+        // Read fieldpath updates, if any
+        int sizeAndFlags = 0;
+        buffer.getIntNetwork(sizeAndFlags);
+        int numUpdates = deserializeFlags(sizeAndFlags);
+        _fieldPathUpdates.reserve(numUpdates);
+        for (int i = 0; i < numUpdates; ++i) {
+            _fieldPathUpdates.emplace_back(FieldPathUpdate::createInstance(repo, *_type, buffer, 8).release());
+        }
+    } catch (const DeserializeException &) {
+        buffer.setPos(pos);
+        throw;
+    } catch (const BufferOutOfBoundsException &) {
+        buffer.setPos(pos);
+        throw;
+    }
+}
+
+int
+DocumentUpdate::deserializeFlags(int sizeAndFlags)
+{
+    _createIfNonExistent = DocumentUpdateFlags::extractFlags(sizeAndFlags).getCreateIfNonExistent();
+    return DocumentUpdateFlags::extractValue(sizeAndFlags);
+}
+
+void
+DocumentUpdate::onDeserialize42(const DocumentTypeRepo &repo,
+                                ByteBuffer& buffer)
+{
+    deserialize42(repo, buffer);
+}
+
+void
+DocumentUpdate::printXml(XmlOutputStream& xos) const
+{
+    xos << XmlTag("document")
+        << XmlAttribute("type", _type->getName())
+        << XmlAttribute("id", getId().toString());
+    for(const auto & update : _updates) {
+        xos << XmlTag("alter") << XmlAttribute("field", update.getField().getName());
+        update.printXml(xos);
+        xos << XmlEndTag();
+    }
+    xos << XmlEndTag();
+}
+
+}
diff --git a/document/src/vespa/document/update/documentupdate.h b/document/src/vespa/document/update/documentupdate.h
new file mode 100644
index 00000000000..4c7ef4fb2d0
--- /dev/null
+++ b/document/src/vespa/document/update/documentupdate.h
@@ -0,0 +1,208 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * @class document::DocumentUpdate
+ * @ingroup document
+ *
+ * @brief Holds a set of operation that may be used to update a document.
+ *
+ * Stores update values for fields defined in the common
+ * VESPA field repository. The "key" for a document is the document id, a
+ * string that must conform to the vespa URI schemes.
+ *
+ * The following update operations are supported: assign, append and remove.
+ * Append and remove are only supported for multivalued fields (arrays and
+ * weightedsets).
+ *
+ * Each document update has a document type, which defines what documents this
+ * update may be applied to and what fields may be updated in that document
+ * update.  The document type for the update is given as a pointer in the
+ * document update object's constructor.<br>
+ * A DocumentUpdate has a vector of DocumentUpdate::Update objects
+ *
+ * @see DocumentId
+ * @see IdString
+ * @see documentmanager.h
+ */
+#pragma once
+
+#include <vespa/document/base/documentid.h>
+#include <vespa/document/base/field.h>
+#include <vespa/document/datatype/documenttype.h>
+#include <vespa/document/fieldvalue/fieldvalue.h>
+#include <vespa/document/util/bytebuffer.h>
+#include <vespa/fastos/fastos.h>
+#include <string>
+
+#include <vespa/document/update/fieldupdate.h>
+#include <vespa/document/update/fieldpathupdate.h>
+
+namespace document {
+
+class Document;
+
+/**
+ * Class containing a document update.  In vespa 5.0, support for field
+ * path updates was added, and a new serialization format was
+ * introduced while keeping the old one.
+ */
+class DocumentUpdate : public vespalib::Identifiable,
+                       public Printable,
+                       public XmlSerializable
+{
+public:
+    typedef std::unique_ptr<DocumentUpdate> UP;
+    typedef std::shared_ptr<DocumentUpdate> SP;
+    typedef std::vector<FieldUpdate> FieldUpdateV;
+    typedef std::vector<FieldPathUpdate::CP> FieldPathUpdateV;
+    /**
+     * Enum class containing the legal serialization version for
+     * document updates. This version is not encoded in the serialized
+     * document update.
+     */
+    enum class SerializeVersion {
+        SERIALIZE_42,  // old style format, before vespa 5.0
+        SERIALIZE_HEAD // new style format, since vespa 5.0
+    };
+
+    /**
+     * Create old style document update, no support for field path updates.
+     */
+    static DocumentUpdate::UP create42(const DocumentTypeRepo&, ByteBuffer&);
+
+    /**
+     * Create new style document update, possibly with field path updates.
+     */
+    static DocumentUpdate::UP createHEAD(const DocumentTypeRepo&, ByteBuffer&);
+	
+    /**
+     * The document type is not strictly needed, as we know this at applyTo()
+     * time, but search does not use applyTo() code to do the update, and don't
+     * know the document type of their objects, so this is supplied for
+     * convinience. It also makes it possible to check updates for sanity at
+     * creation time.
+     *
+     * @param type The document type that this update is applicable for.
+     * @param id The identifier of the document that this update is created for.
+     */
+    DocumentUpdate(const DataType &type, const DocumentId& id);
+
+    /**
+     * Create a document update from a byte buffer containing a serialized
+     * document update.
+     *
+     * @param repo Document type repo used to find proper document type
+     * @param buffer The buffer containing the serialized document update
+     * @param serializeVersion Selector between serialization formats.
+     */
+    DocumentUpdate(const DocumentTypeRepo &repo, ByteBuffer &buffer,
+                   SerializeVersion serializeVersion);
+
+    bool operator==(const DocumentUpdate&) const;
+    bool operator!=(const DocumentUpdate & rhs) const { return ! (*this == rhs); }
+	
+    const DocumentId& getId() const { return _documentId; }
+
+    /**
+     * Applies this update object to the given {@link Document} object.
+     *
+     * @param doc The document to apply this update to. Must be of the same
+     * type as this.
+     */
+    void applyTo(Document& doc) const;
+
+    void clear() { _updates.clear(); }
+    size_t size() const { return _updates.size(); }
+    const FieldUpdate& operator[](int index) const { return _updates[index]; }
+    FieldUpdate& operator[](int index) { return _updates[index]; }
+	
+    /**
+     * Add a field update to this document update.
+     * @return A reference to this.
+     */
+    DocumentUpdate& addUpdate(const FieldUpdate& update) {
+        _updates.push_back(update);
+        return *this;
+    }
+
+    /**
+     * Add a fieldpath update to this document update.
+     * @return A reference to this.
+     */
+    DocumentUpdate& addFieldPathUpdate(const FieldPathUpdate::CP& update) {
+        _fieldPathUpdates.push_back(update);
+        return *this;
+    }
+
+    /** @return The list of updates. */
+    const FieldUpdateV & getUpdates() const { return _updates; }
+
+    /** @return The list of fieldpath updates. */
+    const FieldPathUpdateV & getFieldPathUpdates() const { return _fieldPathUpdates; }
+
+    bool affectsDocumentBody() const;
+
+    /** @return The type of document this update is for. */
+    const DocumentType& getType() const { return static_cast<const DocumentType &> (*_type); }
+	
+    // Printable implementation
+    virtual void print(std::ostream& out, bool verbose,
+                       const std::string& indent) const;
+
+    void deserialize42(const DocumentTypeRepo&, ByteBuffer&);
+    void deserializeHEAD(const DocumentTypeRepo&, ByteBuffer&);
+
+    // Deserializable implementation. Kept as search relies on it currently.
+    virtual void onDeserialize42(const DocumentTypeRepo &repo, ByteBuffer&);
+
+    void serialize42(vespalib::nbostream &stream) const;
+    void serializeHEAD(vespalib::nbostream &stream) const;
+
+    // XmlSerializable implementation
+    virtual void printXml(XmlOutputStream&) const;
+
+    // Cloneable implementation
+    virtual DocumentUpdate* clone() const {
+        return new DocumentUpdate(*this);
+    }
+
+    /**
+     * Sets whether this update should create the document it updates if that document does not exist.
+     * In this case an empty document is created before the update is applied.
+     */
+    void setCreateIfNonExistent(bool value) {
+        _createIfNonExistent = value;
+    }
+
+    /**
+     * Gets whether this update should create the document it updates if that document does not exist.
+     */
+    bool getCreateIfNonExistent() const {
+        return _createIfNonExistent;
+    }
+
+    int serializeFlags(int size_) const;
+    int16_t getVersion() const { return _version; }
+
+    DECLARE_IDENTIFIABLE(DocumentUpdate);
+private:
+    DocumentId _documentId; // The ID of the document to update.
+    const DataType *_type; // The type of document this update is for.
+    FieldUpdateV     _updates; // The list of field updates.
+    FieldPathUpdateV _fieldPathUpdates;
+    int16_t          _version; // Serialization version
+    bool             _createIfNonExistent;
+
+    /**
+     * This function exist because search relies on deserialization through
+     * creating object through empty constructor and calling deserialize.
+     *
+     * It is hidden to prevent accidental other usage.
+     */
+    DocumentUpdate();
+
+    int deserializeFlags(int sizeAndFlags);
+
+};
+
+} // document
+
diff --git a/document/src/vespa/document/update/documentupdateflags.h b/document/src/vespa/document/update/documentupdateflags.h
new file mode 100644
index 00000000000..214416abfe8
--- /dev/null
+++ b/document/src/vespa/document/update/documentupdateflags.h
@@ -0,0 +1,40 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+namespace document {
+
+/**
+ * Class used to represent up to 4 flags used in a DocumentUpdate.
+ * These flags are stored as the 4 most significant bits in a 32 bit integer.
+ *
+ * Flags currently used:
+ *   0) create-if-non-existent.
+ */
+class DocumentUpdateFlags {
+private:
+    uint8_t _flags;
+    DocumentUpdateFlags(uint8_t flags) : _flags(flags) {}
+
+public:
+    DocumentUpdateFlags() : _flags(0) {}
+    bool getCreateIfNonExistent() {
+        return (_flags & 1) != 0;
+    }
+    void setCreateIfNonExistent(bool value) {
+        _flags &= ~1; // clear flag
+        _flags |= value ? 1 : 0; // set flag
+    }
+    int injectInto(int value) {
+        return extractValue(value) | (_flags << 28);
+    }
+    static DocumentUpdateFlags extractFlags(int combined) {
+        return DocumentUpdateFlags((uint8_t)(combined >> 28));
+    }
+    static int extractValue(int combined) {
+        int mask = ~(~0U << 28);
+        return combined & mask;
+    }
+};
+
+}
+
diff --git a/document/src/vespa/document/update/fieldpathupdate.cpp b/document/src/vespa/document/update/fieldpathupdate.cpp
new file mode 100644
index 00000000000..7c222a38c29
--- /dev/null
+++ b/document/src/vespa/document/update/fieldpathupdate.cpp
@@ -0,0 +1,181 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/document/fieldvalue/fieldvalues.h>
+#include <vespa/document/select/parser.h>
+#include <vespa/document/update/fieldpathupdates.h>
+#include <vespa/log/log.h>
+
+LOG_SETUP(".document.update.fieldpathupdate");
+
+namespace document {
+
+IMPLEMENT_IDENTIFIABLE_ABSTRACT(FieldPathUpdate, Identifiable);
+
+namespace {
+
+std::unique_ptr<select::Node>
+parseDocumentSelection(vespalib::stringref query, const DocumentTypeRepo& repo)
+{
+    BucketIdFactory factory;
+    select::Parser parser(repo, factory);
+    return parser.parse(query);
+}
+
+}  // namespace
+
+FieldPathUpdate::FieldPathUpdate() :
+    _originalFieldPath(),
+    _originalWhereClause(),
+    _fieldPath(),
+    _whereClause()
+{
+}
+
+FieldPathUpdate::FieldPathUpdate(const DocumentTypeRepo& repo,
+                                 const DataType& type,
+                                 stringref fieldPath,
+                                 stringref whereClause) :
+    _originalFieldPath(fieldPath),
+    _originalWhereClause(whereClause),
+    _fieldPath(type.buildFieldPath(_originalFieldPath).release()),
+    _whereClause(!_originalWhereClause.empty()
+                 ? parseDocumentSelection(_originalWhereClause, repo)
+                 : std::unique_ptr<select::Node>())
+{
+    if (!_fieldPath.get()) {
+        throw vespalib::IllegalArgumentException(
+                vespalib::make_string("Could not create field path update for: path='%s', where='%s'",
+                                      fieldPath.c_str(), whereClause.c_str()), VESPA_STRLOC);
+    }
+}
+
+bool
+FieldPathUpdate::operator==(const FieldPathUpdate& other) const
+{
+    return (other._originalFieldPath == _originalFieldPath)
+            && (other._originalWhereClause == _originalWhereClause);
+}
+
+void
+FieldPathUpdate::applyTo(Document& doc) const
+{
+    std::unique_ptr<FieldValue::IteratorHandler> handler(getIteratorHandler(doc));
+
+    if (!_whereClause) {
+        doc.iterateNested(*_fieldPath, *handler);
+    } else {
+        select::ResultList results = _whereClause->contains(doc);
+        for (select::ResultList::const_iterator i = results.begin();
+             i != results.end(); ++i)
+        {
+            LOG(spam, "vars = %s", FieldValue::IteratorHandler::toString(handler->getVariables()).c_str());
+            if (*i->second == select::Result::True) {
+                handler->setVariables(i->first);
+                doc.iterateNested(*_fieldPath, *handler);
+            }
+        }
+    }
+}
+
+bool
+FieldPathUpdate::affectsDocumentBody() const
+{
+    if (_fieldPath->empty() || !(*_fieldPath)[0].hasField()) return false;
+    const Field& field = (*_fieldPath)[0].getFieldRef();
+    return !field.isHeaderField();
+}
+
+void
+FieldPathUpdate::print(std::ostream& out, bool,
+                       const std::string& indent) const
+{
+    out << indent << "fieldPath='" << _originalFieldPath << "',\n"
+        << indent << "whereClause='" << _originalWhereClause << "'";
+}
+
+void
+FieldPathUpdate::checkCompatibility(const FieldValue& fv) const
+{
+    if ( !getResultingDataType().isValueType(fv)) {
+        throw vespalib::IllegalArgumentException(
+                vespalib::make_string("Cannot update a '%s' field with a '%s' value",
+                                      getResultingDataType().toString().c_str(),
+                                      fv.getDataType()->toString().c_str()),
+                VESPA_STRLOC);
+    }
+}
+
+const DataType&
+FieldPathUpdate::getResultingDataType() const
+{
+    if (_fieldPath->empty()) {
+        throw vespalib::IllegalStateException("Cannot get resulting data "
+                "type from an empty field path", VESPA_STRLOC);
+    }
+    return _fieldPath->rbegin()->getDataType();
+}
+
+vespalib::string
+FieldPathUpdate::getString(ByteBuffer& buffer)
+{
+    int32_t length = 0;
+    buffer.getIntNetwork(length);
+    vespalib::string s(buffer.getBufferAtPos());
+    buffer.incPos(length);
+    return s;
+}
+
+void
+FieldPathUpdate::deserialize(const DocumentTypeRepo& repo,
+                             const DataType& type,
+                             ByteBuffer& buffer, uint16_t)
+{
+    _originalFieldPath = getString(buffer);
+    _originalWhereClause = getString(buffer);
+
+    try {
+        _fieldPath = type.buildFieldPath(_originalFieldPath).release();
+        if (!_fieldPath.get()) {
+            throw DeserializeException(
+                    vespalib::make_string("Invalid field path: '%s'", _originalFieldPath.c_str()),
+                    VESPA_STRLOC);
+        }
+        _whereClause = !_originalWhereClause.empty()
+                       ? parseDocumentSelection(_originalWhereClause, repo)
+                       : std::unique_ptr<select::Node>();
+    } catch (const select::ParsingFailedException& e) {
+        throw DeserializeException(e.what(), VESPA_STRLOC);
+    }
+}
+
+std::unique_ptr<FieldPathUpdate>
+FieldPathUpdate::createInstance(const DocumentTypeRepo& repo,
+                                const DataType &type, ByteBuffer& buffer,
+                                int serializationVersion)
+{
+    unsigned char updateType = 0;
+    buffer.getByte(updateType);
+
+    std::unique_ptr<FieldPathUpdate> update;
+    switch (updateType)
+    {
+    case 0:
+        update.reset(new AssignFieldPathUpdate());
+        break;
+    case 1:
+        update.reset(new RemoveFieldPathUpdate());
+        break;
+    case 2:
+        update.reset(new AddFieldPathUpdate());
+        break;
+    default:
+        throw DeserializeException(
+                vespalib::make_string("Unknown fieldpath update type: %d", updateType),
+                VESPA_STRLOC);
+    }
+    update->deserialize(repo, type, buffer, serializationVersion);
+    return update;
+}
+
+} // ns document
+
diff --git a/document/src/vespa/document/update/fieldpathupdate.h b/document/src/vespa/document/update/fieldpathupdate.h
new file mode 100644
index 00000000000..e6958082493
--- /dev/null
+++ b/document/src/vespa/document/update/fieldpathupdate.h
@@ -0,0 +1,123 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <boost/operators.hpp>
+
+#include <boost/operators.hpp>
+#include <vespa/vespalib/objects/cloneable.h>
+
+#include <vespa/document/datatype/datatype.h>
+#include <vespa/document/util/serializable.h>
+#include <vespa/document/util/xmlserializable.h>
+#include <vespa/document/fieldvalue/fieldvalue.h>
+#include <vespa/document/select/node.h>
+#include <vespa/document/select/resultlist.h>
+#include <vespa/document/update/updatevisitor.h>
+#include <vespa/document/update/valueupdate.h>
+
+namespace document {
+
+class ByteBuffer;
+class DocumentTypeRepo;
+class Field;
+class FieldValue;
+class BucketIdFactory;
+
+class FieldPathUpdate : public vespalib::Cloneable,
+                        public Printable,
+                        public boost::equality_comparable<FieldPathUpdate>,
+                        public vespalib::Identifiable
+{
+protected:
+    typedef vespalib::stringref stringref;
+    /** To be used for deserialization */
+    FieldPathUpdate();
+
+   static vespalib::string getString(ByteBuffer& buffer);
+public:
+    typedef select::ResultList::VariableMap VariableMap;
+    typedef std::shared_ptr<FieldPathUpdate> SP;
+    typedef vespalib::CloneablePtr<FieldPathUpdate> CP;
+
+    FieldPathUpdate(const DocumentTypeRepo& repo,
+                    const DataType& type,
+                    stringref fieldPath,
+                    stringref whereClause = stringref());
+
+    virtual ~FieldPathUpdate() {}
+
+    enum FieldPathUpdateType {
+        Add    = IDENTIFIABLE_CLASSID(AddFieldPathUpdate),
+        Assign = IDENTIFIABLE_CLASSID(AssignFieldPathUpdate),
+        Remove = IDENTIFIABLE_CLASSID(RemoveFieldPathUpdate)
+    };
+
+    void applyTo(Document& doc) const;
+
+    virtual FieldPathUpdate* clone() const = 0;
+
+    virtual bool operator==(const FieldPathUpdate& other) const;
+
+    const FieldPath& getFieldPath() const { return *_fieldPath; }
+    const select::Node& getWhereClause() const { return *_whereClause; }
+
+    const vespalib::string& getOriginalFieldPath() const { return _originalFieldPath; }
+    const vespalib::string& getOriginalWhereClause() const { return _originalWhereClause; }
+
+    /**
+     * Check that a given field value is of the type inferred by
+     * the field path.
+     * @throws IllegalArgumentException upon datatype mismatch.
+     */
+    void checkCompatibility(const FieldValue& fv) const;
+
+    /** @return Whether or not the first field path element is a body field */
+    bool affectsDocumentBody() const;
+
+    void print(std::ostream& out, bool verbose, const std::string& indent) const;
+
+    DECLARE_IDENTIFIABLE_ABSTRACT(FieldPathUpdate);
+
+    virtual void accept(UpdateVisitor &visitor) const = 0;
+    virtual uint8_t getSerializedType() const = 0;
+
+    /** Deserializes and creates a new FieldPathUpdate instance.
+     * Requires type id to be not yet read.
+     */
+    static std::unique_ptr<FieldPathUpdate> createInstance(
+            const DocumentTypeRepo& repo,
+            const DataType &type, ByteBuffer& buffer,
+            int serializationVersion);
+
+protected:
+    /**
+     * Deserializes the given byte buffer into an instance of a FieldPathUpdate
+     * object.
+     *
+     * @param type A data type that describes the content of the buffer.
+     * @param buffer The byte buffer that contains the serialized object.
+     * @param version The serialization version of the object to deserialize.
+     */
+    virtual void deserialize(const DocumentTypeRepo& repo,
+                             const DataType& type,
+                             ByteBuffer& buffer, uint16_t version);
+
+    /** @return the datatype of the last path element in the field path */
+    const DataType& getResultingDataType() const;
+    enum SerializedMagic {AssignMagic=0, RemoveMagic=1, AddMagic=2};
+private:
+    // TODO: rename to createIteratorHandler?
+    virtual std::unique_ptr<FieldValue::IteratorHandler> getIteratorHandler(
+            Document& doc) const = 0;
+
+    vespalib::string _originalFieldPath;
+    vespalib::string _originalWhereClause;
+
+    // TODO: replace these with LinkedPtr? Need to verify updates cannot
+    // be shared between threads first
+    vespalib::CloneablePtr<FieldPath> _fieldPath;
+    std::shared_ptr<select::Node> _whereClause;
+};
+
+} // ns document
+
diff --git a/document/src/vespa/document/update/fieldpathupdates.h b/document/src/vespa/document/update/fieldpathupdates.h
new file mode 100644
index 00000000000..87470de7b71
--- /dev/null
+++ b/document/src/vespa/document/update/fieldpathupdates.h
@@ -0,0 +1,7 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/document/update/addfieldpathupdate.h>
+#include <vespa/document/update/assignfieldpathupdate.h>
+#include <vespa/document/update/removefieldpathupdate.h>
+
diff --git a/document/src/vespa/document/update/fieldupdate.cpp b/document/src/vespa/document/update/fieldupdate.cpp
new file mode 100644
index 00000000000..8ae2da9ac02
--- /dev/null
+++ b/document/src/vespa/document/update/fieldupdate.cpp
@@ -0,0 +1,108 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/document/base/exceptions.h>
+#include <vespa/document/fieldvalue/document.h>
+#include <vespa/document/update/fieldupdate.h>
+
+namespace document {
+
+typedef std::vector<ValueUpdate::CP> ValueUpdateList;
+
+FieldUpdate::FieldUpdate(const Field& field)
+    : Printable(),
+      _field(field),
+      _updates()
+{
+}
+
+// Construct a field update by deserialization.
+FieldUpdate::FieldUpdate(const DocumentTypeRepo& repo,
+                         const DocumentType& type,
+                         ByteBuffer& buffer,
+                         int16_t version)
+    : Printable(),
+      _field(),
+      _updates()
+{
+    deserialize(repo, type, buffer, version);
+}
+
+bool
+FieldUpdate::operator==(const FieldUpdate& other) const
+{
+    if (_field != other._field) return false;
+    if (_updates.size() != other._updates.size()) return false;
+    for (uint32_t i=0, n=_updates.size(); i<n; ++i) {
+        if (*_updates[i] != *other._updates[i]) return false;
+    }
+    return true;
+}
+
+void
+FieldUpdate::printXml(XmlOutputStream& xos) const
+{
+    for(const auto & update : _updates) {
+        update->printXml(xos);
+    }
+}
+
+// Apply this field update to the given document.
+void
+FieldUpdate::applyTo(Document& doc) const
+{
+    const DataType& datatype = _field.getDataType();
+    FieldValue::UP value = doc.getValue(_field);
+
+    for (const ValueUpdate::CP & update : _updates) {
+        if ( ! value) {
+            // Avoid passing a null pointer to a value update.
+            value = datatype.createFieldValue();
+        }
+        if (!update->applyTo(*value)) {
+            value.reset();
+        }
+    }
+
+    if (value) {
+        doc.setFieldValue(_field, std::move(value));
+    } else {
+        doc.remove(_field);
+    }
+}
+
+// Print this field update as a human readable string.
+void
+FieldUpdate::print(std::ostream& out, bool verbose,
+                   const std::string& indent) const
+{
+    out << "FieldUpdate(" << _field.toString(verbose);
+    for(const auto & update : _updates) {
+        out << "\n" << indent << "  ";
+        update->print(out, verbose, indent + "  ");
+    }
+    if (_updates.size() > 0) {
+        out << "\n" << indent;
+    }
+    out << ")";
+}
+
+// Deserialize this field update from the given buffer.
+void
+FieldUpdate::deserialize(const DocumentTypeRepo& repo,
+                         const DocumentType& docType, ByteBuffer& buffer, int16_t version)
+{
+    int fieldId;
+    buffer.getIntNetwork(fieldId);
+    _field = docType.getField(fieldId, Document::getNewestSerializationVersion());
+    const DataType& dataType = _field.getDataType();
+
+    int numUpdates = 0;
+    buffer.getIntNetwork(numUpdates);
+    _updates.clear();
+    _updates.resize(numUpdates);
+    for(int i = 0; i < numUpdates; i++) {
+        _updates[i].reset(ValueUpdate::createInstance(repo, dataType, buffer, version).release());
+    }
+}
+
+}  // namespace document
diff --git a/document/src/vespa/document/update/fieldupdate.h b/document/src/vespa/document/update/fieldupdate.h
new file mode 100644
index 00000000000..14766af57e1
--- /dev/null
+++ b/document/src/vespa/document/update/fieldupdate.h
@@ -0,0 +1,101 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * \class document::FieldUpdate
+ * \ingroup update
+ *
+ * \brief Represent a collection of updates to be performed on a single
+ * field in a document.
+ *
+ * It inherits from Printable to produce human readable output when required.
+ * Serialization is done through Serializable and XmlSerializable.
+ * Deserialization is specially handled as document type is not serialized with
+ * the object.
+ */
+#pragma once
+
+#include <vespa/document/base/field.h>
+#include <vespa/document/update/valueupdate.h>
+#include <vespa/document/util/serializable.h>
+
+namespace document {
+
+class Document;
+class DocumentType;
+class DocumentTypeRepo;
+
+class FieldUpdate : public vespalib::Identifiable,
+                    public Printable,
+                    public XmlSerializable
+{
+    Field _field;
+    std::vector<ValueUpdate::CP> _updates;
+
+public:
+    typedef vespalib::CloneablePtr<FieldUpdate> CP;
+
+    FieldUpdate(const Field& field);
+
+    /**
+     * This is a convenience function to construct a field update directly from
+     * a byte buffer by deserializing all its content from the buffer.
+     *
+     * @param type A document type that describes the buffer content.
+     * @param buffer A byte buffer that contains a serialized field update.
+     * @param serializationVersion The serialization version the update was serialized with.
+     */
+    FieldUpdate(const DocumentTypeRepo& repo, const DocumentType& type,
+                ByteBuffer& buffer, int16_t serializationVersion);
+
+    bool operator==(const FieldUpdate&) const;
+    bool operator!=(const FieldUpdate & rhs) const { return ! (*this == rhs); }
+
+    /**
+     * Add a value update to this field update.
+     *
+     * @param update A pointer to the value update to add to this.
+     * @return A pointer to this.
+     */
+    FieldUpdate& addUpdate(const ValueUpdate& update) {
+        update.checkCompatibility(_field); // May throw exception.
+        _updates.push_back(ValueUpdate::CP(update.clone()));
+        return *this;
+    }
+
+    const ValueUpdate& operator[](int index) const { return *_updates[index]; }
+    ValueUpdate& operator[](int index) { return *_updates[index]; }
+    size_t size() const { return _updates.size(); }
+
+    /** @return The non-modifieable list of value updates to perform. */
+    const std::vector<ValueUpdate::CP>& getUpdates() const { return _updates; }
+
+    const Field& getField() const { return _field; }
+
+    /**
+     * Applies this update object to the given {@link Document} object.
+     *
+     * @param doc The document to apply this update to.
+     */
+    void applyTo(Document& doc) const;
+
+    // Printable implementation
+    void print(std::ostream& out, bool verbose,
+               const std::string& indent) const;
+
+    // XmlSerializable implementation
+    void printXml(XmlOutputStream&) const;
+
+    /**
+     * Deserializes the given byte buffer into an instance of an update object.
+     * Not a Deserializable, as document type is needed as extra information.
+     *
+     * @param type A document type that describes the buffer content.
+     * @param buffer The byte buffer that contains the serialized update object.
+     * @param serializationVersion The serialization version the update was serialized with.
+     */
+    void deserialize(const DocumentTypeRepo& repo, const DocumentType& type,
+                     ByteBuffer& buffer, int16_t serializationVersion);
+
+};
+
+} // document
+
diff --git a/document/src/vespa/document/update/mapvalueupdate.cpp b/document/src/vespa/document/update/mapvalueupdate.cpp
new file mode 100644
index 00000000000..c7aef3e8f98
--- /dev/null
+++ b/document/src/vespa/document/update/mapvalueupdate.cpp
@@ -0,0 +1,158 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/document/base/field.h>
+#include <vespa/document/datatype/arraydatatype.h>
+#include <vespa/document/datatype/weightedsetdatatype.h>
+#include <vespa/document/fieldvalue/fieldvalues.h>
+#include <vespa/document/repo/fixedtyperepo.h>
+#include <vespa/document/serialization/vespadocumentdeserializer.h>
+#include <vespa/document/update/mapvalueupdate.h>
+#include <vespa/vespalib/objects/nbostream.h>
+
+using vespalib::IllegalArgumentException;
+using vespalib::IllegalStateException;
+using vespalib::nbostream;
+
+namespace document
+{
+
+IMPLEMENT_IDENTIFIABLE(MapValueUpdate, ValueUpdate);
+
+bool
+MapValueUpdate::operator==(const ValueUpdate& other) const
+{
+    if (other.getClass().id() != MapValueUpdate::classId) return false;
+    const MapValueUpdate& o(static_cast<const MapValueUpdate&>(other));
+    if (*_key != *o._key) return false;
+    if (*_update != *o._update) return false;
+    return true;
+}
+
+// Ensure that this update is compatible with given field.
+void
+MapValueUpdate::checkCompatibility(const Field& field) const
+{
+    // Check compatibility of nested types.
+    if (field.getDataType().getClass().id() == ArrayDataType::classId) {
+	    if (_key->getClass().id() != IntFieldValue::classId) {
+            throw IllegalArgumentException(vespalib::make_string(
+                    "Key for field '%s' is of wrong type (expected '%s', "
+                    "was '%s').",
+                    field.getName().c_str(), DataType::INT->toString().c_str(),
+                    _key->getDataType()->toString().c_str()), VESPA_STRLOC);
+        }
+    } else if (field.getDataType().getClass().id() == WeightedSetDataType::classId) {
+        const WeightedSetDataType& type =
+            static_cast<const WeightedSetDataType&>(field.getDataType());
+        if (!type.getNestedType().isValueType(*_key)) {
+            throw IllegalArgumentException(vespalib::make_string(
+                    "Key for field '%s' is of wrong type (expected '%s', "
+                    "was '%s').",
+                    field.getName().c_str(), DataType::INT->toString().c_str(),
+                    _key->getDataType()->toString().c_str()), VESPA_STRLOC);
+        }
+    } else {
+        throw IllegalArgumentException("MapValueUpdate does not support "
+                "datatype " + field.getDataType().toString() + ".", VESPA_STRLOC);
+    }
+}
+
+// Apply this update to the given document.
+bool
+MapValueUpdate::applyTo(FieldValue& value) const
+{
+    if (value.getDataType()->getClass().id() == ArrayDataType::classId) {
+        ArrayFieldValue& val(static_cast<ArrayFieldValue&>(value));
+        int32_t index = _key->getAsInt();
+        if (index < 0 || static_cast<uint32_t>(index) >= val.size()) {
+            throw IllegalStateException(vespalib::make_string(
+                    "Tried to update element %i in an array of %zu elements",
+		            index, val.size()), VESPA_STRLOC);
+        }
+        if (!_update->applyTo(val[_key->getAsInt()])) {
+            val.remove(_key->getAsInt());
+        }
+    } else if (value.getDataType()->getClass().id()
+                    == WeightedSetDataType::classId)
+    {
+        const WeightedSetDataType& type(
+                static_cast<const WeightedSetDataType&>(*value.getDataType()));
+        WeightedSetFieldValue& val(static_cast<WeightedSetFieldValue&>(value));
+        WeightedSetFieldValue::iterator it = val.find(*_key);
+        if (it == val.end() && type.createIfNonExistent()) {
+            // Add item and ensure it does not get initially auto-removed if
+            // remove-if-zero is set, as this would void the update.
+            val.addIgnoreZeroWeight(*_key, 0);
+            it = val.find(*_key);
+        }
+        if (it == val.end()) {
+            // Are we sure we don't want updates to fail on missing values?
+            return true;
+        }
+        // XXX why are we removing items if updates fail? Surely, a failed
+        // update should have as a postcondition that it did not mutate the
+        // item in question? This is not exception safe either way.
+        IntFieldValue& weight = dynamic_cast<IntFieldValue&>(*it->second);
+        if (!_update->applyTo(weight)) {
+            val.remove(*_key);
+        } else if (weight.getAsInt() == 0 && type.removeIfZero()) {
+            val.remove(*_key);
+        }
+    } else {
+        throw IllegalStateException(
+                "Cannot apply map value update to field of type "
+                + value.getDataType()->toString() + ".", VESPA_STRLOC);
+    }
+    return true;
+}
+
+// Print this update in human readable form.
+void
+MapValueUpdate::print(std::ostream& out, bool, const std::string& indent) const
+{
+    out << indent << "MapValueUpdate(" << *_key << ", " << *_update << ")";
+}
+
+void
+MapValueUpdate::printXml(XmlOutputStream& xos) const
+{
+    xos << XmlTag("map")
+        << XmlTag("value") << *_key << XmlEndTag()
+        << XmlTag("update") << *_update << XmlEndTag()
+        << XmlEndTag();
+}
+
+// Deserialize this update from the given buffer.
+void
+MapValueUpdate::deserialize(const DocumentTypeRepo& repo,
+                            const DataType& type,
+                            ByteBuffer& buffer, uint16_t version)
+{
+    nbostream stream(buffer.getBufferAtPos(), buffer.getRemaining());
+    VespaDocumentDeserializer deserializer(repo, stream, version);
+    switch(type.getClass().id()) {
+        case ArrayDataType::classId:
+            _key.reset(new IntFieldValue);
+            deserializer.read(*_key);
+            buffer.incPos(buffer.getRemaining() - stream.size());
+            _update.reset(ValueUpdate::createInstance(
+                            repo, type, buffer, version).release());
+            break;
+        case WeightedSetDataType::classId:
+        {
+            const WeightedSetDataType& wset(
+                    static_cast<const WeightedSetDataType&>(type));
+            _key.reset(wset.getNestedType().createFieldValue().release());
+            deserializer.read(*_key);
+            buffer.incPos(buffer.getRemaining() - stream.size());
+            _update.reset(ValueUpdate::createInstance(
+                            repo, *DataType::INT, buffer, version).release());
+            break;
+        }
+        default:
+            throw DeserializeException("Can not perform map update on type "
+                    + type.toString() + ".", VESPA_STRLOC);
+    }
+}
+
+}
diff --git a/document/src/vespa/document/update/mapvalueupdate.h b/document/src/vespa/document/update/mapvalueupdate.h
new file mode 100644
index 00000000000..cf0869d271f
--- /dev/null
+++ b/document/src/vespa/document/update/mapvalueupdate.h
@@ -0,0 +1,94 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * @class document::MapValueUpdate
+ * @ingroup document
+ *
+ * @brief This class is intended to map a value update to a part of a collection
+ *        field value.
+ *
+ * Note that the key must be an IntFieldValue in case of an array. For a
+ * weighted set it must match the nested type of the weighted set.
+ */
+#pragma once
+
+#include <vespa/document/update/valueupdate.h>
+#include <vespa/document/fieldvalue/fieldvalue.h>
+
+namespace document {
+
+class MapValueUpdate : public ValueUpdate {
+    FieldValue::CP _key; // The field value this update is mapping to.
+                         // This is shared pointer to be able to lookup key
+                         // in weighted set map.
+    ValueUpdate::CP _update; //The update to apply to the value member of this.
+
+    // Used by ValueUpdate's static factory function
+    // Private because it generates an invalid object.
+    friend class ValueUpdate;
+    MapValueUpdate() : ValueUpdate(), _key(), _update() {}
+
+    ACCEPT_UPDATE_VISITOR;
+public:
+
+    /**
+     * The default constructor requires a value for this object's value and
+     * update member.
+     *
+     * @param key The identifier of the field value to be updated.
+     * @param update The update to map to apply to the field value of this.
+     */
+    MapValueUpdate(const FieldValue& key, const ValueUpdate& update)
+        : ValueUpdate(),
+          _key(key.clone()),
+          _update(update.clone()) {}
+
+    virtual bool operator==(const ValueUpdate& other) const;
+
+    /** @return The key of the field value to update. */
+    const FieldValue& getKey() const { return *_key; }
+    FieldValue& getKey() { return *_key; }
+
+    /** @return The update to apply to the field value of this. */
+    const ValueUpdate& getUpdate() const { return *_update; }
+    ValueUpdate& getUpdate() { return *_update; }
+
+    /**
+     * Sets the identifier of the field value to update.
+     *
+     * @param key The field value identifier.
+     * @return A pointer to this.
+     */
+    MapValueUpdate& setKey(const FieldValue& key) {
+        _key.reset(key.clone());
+        return *this;
+    }
+
+    /**
+     * Sets the update to apply to the value update of this.
+     *
+     * @param update The value update.
+     * @return A pointer to this.
+     */
+    MapValueUpdate& setUpdate(const ValueUpdate& update) {
+        _update.reset(update.clone());
+        return *this;
+    }
+
+    // ValueUpdate implementation
+    virtual void checkCompatibility(const Field& field) const;
+    virtual bool applyTo(FieldValue& value) const;
+    virtual void printXml(XmlOutputStream& xos) const;
+    virtual void print(std::ostream& out, bool verbose,
+                       const std::string& indent) const;
+    virtual void deserialize(const DocumentTypeRepo& repo,
+                             const DataType& type,
+                             ByteBuffer& buffer, uint16_t version);
+    virtual MapValueUpdate* clone() const
+        { return new MapValueUpdate(*this); }
+
+    DECLARE_IDENTIFIABLE(MapValueUpdate);
+
+};
+
+} // document
+
diff --git a/document/src/vespa/document/update/removefieldpathupdate.cpp b/document/src/vespa/document/update/removefieldpathupdate.cpp
new file mode 100644
index 00000000000..1cda533e414
--- /dev/null
+++ b/document/src/vespa/document/update/removefieldpathupdate.cpp
@@ -0,0 +1,53 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/document/fieldvalue/fieldvalues.h>
+#include <vespa/document/select/parser.h>
+#include <vespa/document/update/removefieldpathupdate.h>
+#include <vespa/log/log.h>
+
+LOG_SETUP(".document.update.fieldpathupdate");
+
+
+namespace document {
+
+IMPLEMENT_IDENTIFIABLE(RemoveFieldPathUpdate, FieldPathUpdate);
+
+RemoveFieldPathUpdate::RemoveFieldPathUpdate()
+    : FieldPathUpdate()
+{
+}
+
+RemoveFieldPathUpdate::RemoveFieldPathUpdate(
+        const DocumentTypeRepo& repo,
+        const DataType& type,
+        stringref fieldPath,
+        stringref whereClause)
+    : FieldPathUpdate(repo, type, fieldPath, whereClause)
+{
+}
+
+bool
+RemoveFieldPathUpdate::operator==(const FieldPathUpdate& other) const
+{
+    if (other.getClass().id() != RemoveFieldPathUpdate::classId) return false;
+    return FieldPathUpdate::operator==(other);
+}
+
+void
+RemoveFieldPathUpdate::print(std::ostream& out, bool verbose,
+                          const std::string& indent) const
+{
+    out << "RemoveFieldPathUpdate(\n";
+    FieldPathUpdate::print(out, verbose, indent + "  ");
+    out << "\n" << indent << ")";
+}
+
+void
+RemoveFieldPathUpdate::deserialize(
+        const DocumentTypeRepo& repo, const DataType& type,
+        ByteBuffer& buffer, uint16_t version)
+{
+    FieldPathUpdate::deserialize(repo, type, buffer, version);
+}
+
+} // ns document
diff --git a/document/src/vespa/document/update/removefieldpathupdate.h b/document/src/vespa/document/update/removefieldpathupdate.h
new file mode 100644
index 00000000000..94bd8bf48ae
--- /dev/null
+++ b/document/src/vespa/document/update/removefieldpathupdate.h
@@ -0,0 +1,53 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/document/update/fieldpathupdate.h>
+
+namespace document {
+
+class RemoveFieldPathUpdate : public FieldPathUpdate
+{
+public:
+    /** For deserialization */
+    RemoveFieldPathUpdate();
+
+    RemoveFieldPathUpdate(const DocumentTypeRepo& repo,
+                          const DataType& type,
+                          stringref fieldPath,
+                          stringref whereClause = stringref());
+
+    FieldPathUpdate* clone() const { return new RemoveFieldPathUpdate(*this); }
+
+    bool operator==(const FieldPathUpdate& other) const;
+
+    virtual void print(std::ostream& out, bool verbose,
+                       const std::string& indent) const;
+
+    DECLARE_IDENTIFIABLE(RemoveFieldPathUpdate);
+    ACCEPT_UPDATE_VISITOR;
+
+private:
+    uint8_t getSerializedType() const override { return RemoveMagic; }
+    virtual void deserialize(const DocumentTypeRepo& repo,
+                             const DataType& type,
+                             ByteBuffer& buffer, uint16_t version);
+
+    class RemoveIteratorHandler : public FieldValue::IteratorHandler
+    {
+    public:
+        RemoveIteratorHandler() {}
+
+        ModificationStatus doModify(FieldValue&) {
+            return REMOVED;
+        }
+    };
+
+    std::unique_ptr<FieldValue::IteratorHandler> getIteratorHandler(Document&) const {
+        return std::unique_ptr<FieldValue::IteratorHandler>(
+                new RemoveIteratorHandler());
+    }
+};
+
+
+} // ns document
+
diff --git a/document/src/vespa/document/update/removevalueupdate.cpp b/document/src/vespa/document/update/removevalueupdate.cpp
new file mode 100644
index 00000000000..d5f357131d3
--- /dev/null
+++ b/document/src/vespa/document/update/removevalueupdate.cpp
@@ -0,0 +1,109 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/document/base/field.h>
+#include <vespa/document/datatype/arraydatatype.h>
+#include <vespa/document/datatype/weightedsetdatatype.h>
+#include <vespa/document/fieldvalue/fieldvalues.h>
+#include <vespa/document/repo/fixedtyperepo.h>
+#include <vespa/document/serialization/vespadocumentdeserializer.h>
+#include <vespa/document/update/removevalueupdate.h>
+#include <vespa/vespalib/objects/nbostream.h>
+
+using vespalib::IllegalArgumentException;
+using vespalib::IllegalStateException;
+using vespalib::nbostream;
+
+namespace document
+{
+
+IMPLEMENT_IDENTIFIABLE(RemoveValueUpdate, ValueUpdate);
+
+bool
+RemoveValueUpdate::operator==(const ValueUpdate& other) const
+{
+    if (other.getClass().id() != RemoveValueUpdate::classId) return false;
+    const RemoveValueUpdate& o(static_cast<const RemoveValueUpdate&>(other));
+    if (*_key != *o._key) return false;
+    return true;
+}
+
+// Ensure that this update is compatible with given field.
+void
+RemoveValueUpdate::checkCompatibility(const Field& field) const
+{
+    if (field.getDataType().inherits(CollectionDataType::classId)) {
+        const CollectionDataType& type =
+            static_cast<const CollectionDataType&>(field.getDataType());
+        if (!type.getNestedType().isValueType(*_key)) {
+            throw IllegalArgumentException(
+                    "Cannot remove value of type "
+                    + _key->getDataType()->toString() + " from field "
+                    + field.getName().c_str() + " of container type "
+                    + field.getDataType().toString(), VESPA_STRLOC);
+        }
+    } else {
+        throw IllegalArgumentException(
+                "Can not remove a value from field of type "
+                + field.getDataType().toString(), VESPA_STRLOC);
+    }
+}
+
+// Apply this update to the given document.
+bool
+RemoveValueUpdate::applyTo(FieldValue& value) const
+{
+    if (value.inherits(ArrayFieldValue::classId)) {
+        ArrayFieldValue& doc(static_cast<ArrayFieldValue&>(value));
+        doc.remove(*_key);	
+    } else if (value.inherits(WeightedSetFieldValue::classId)) {
+        WeightedSetFieldValue& doc(static_cast<WeightedSetFieldValue&>(value));
+        doc.remove(*_key);
+    } else {
+        std::string err = vespalib::make_string(
+                "Unable to remove a value from a \"%s\" field value.",
+                value.getClass().name());
+        throw IllegalStateException(err, VESPA_STRLOC);
+    }
+    return true;
+}
+
+void
+RemoveValueUpdate::printXml(XmlOutputStream& xos) const
+{
+    xos << XmlTag("remove") << *_key << XmlEndTag();
+}
+
+// Print this update in human readable form.
+void
+RemoveValueUpdate::print(std::ostream& out, bool, const std::string&) const
+{
+    out << "RemoveValueUpdate(" << *_key << ")";
+}
+
+// Deserialize this update from the given buffer.
+void
+RemoveValueUpdate::deserialize(
+        const DocumentTypeRepo& repo, const DataType& type,
+        ByteBuffer& buffer, uint16_t version)
+{
+    switch(type.getClass().id()) {
+        case ArrayDataType::classId:
+        case WeightedSetDataType::classId:
+        {
+            const CollectionDataType& c(
+                    static_cast<const CollectionDataType&>(type));
+            _key.reset(c.getNestedType().createFieldValue().release());
+            nbostream stream(buffer.getBufferAtPos(), buffer.getRemaining());
+            VespaDocumentDeserializer deserializer(repo, stream, version);
+            deserializer.read(*_key);
+            buffer.incPos(buffer.getRemaining() - stream.size());
+            break;
+        }
+        default:
+            throw DeserializeException(
+                    "Can not perform remove operation on type "
+                    + type.toString() + ".", VESPA_STRLOC);
+    }
+}
+
+}
diff --git a/document/src/vespa/document/update/removevalueupdate.h b/document/src/vespa/document/update/removevalueupdate.h
new file mode 100644
index 00000000000..5552796ceb4
--- /dev/null
+++ b/document/src/vespa/document/update/removevalueupdate.h
@@ -0,0 +1,67 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * This class represents an update that removes a given field value from a
+ * field.
+ */
+#pragma once
+
+#include <vespa/document/fieldvalue/fieldvalue.h>
+#include <vespa/document/update/valueupdate.h>
+
+namespace document {
+
+class RemoveValueUpdate : public ValueUpdate {
+    FieldValue::CP _key; // The field value to remove by this update.
+
+    RemoveValueUpdate() : ValueUpdate(), _key() {}
+    ACCEPT_UPDATE_VISITOR;
+
+public:
+    typedef std::unique_ptr<RemoveValueUpdate> UP;
+
+    /**
+     * The default constructor requires initial values for all member variables.
+     *
+     * @param value The identifier of the field value to update.
+     */
+    RemoveValueUpdate(const FieldValue& key)
+        : ValueUpdate(),
+          _key(key.clone()) {}
+
+    virtual bool operator==(const ValueUpdate& other) const;
+
+    /**
+     * @return The key, whose value to remove during this update. This will be
+     *         an IntFieldValue for arrays.
+     */
+    const FieldValue& getKey() const { return *_key; }
+
+    /**
+     * Sets the field value to remove during this update.
+     *
+     * @param The new field value.
+     * @return A pointer to this.
+     */
+    RemoveValueUpdate& setKey(const FieldValue& key) {
+        _key.reset(key.clone());
+        return *this;
+    }
+
+    // ValueUpdate implementation
+    virtual void checkCompatibility(const Field& field) const;
+    virtual bool applyTo(FieldValue& value) const;
+    virtual void printXml(XmlOutputStream& xos) const;
+    virtual void print(std::ostream& out, bool verbose,
+                       const std::string& indent) const;
+    virtual void deserialize(const DocumentTypeRepo& repo,
+                             const DataType& type,
+                             ByteBuffer& buffer, uint16_t version);
+    virtual RemoveValueUpdate* clone() const
+        { return new RemoveValueUpdate(*this); }
+
+    DECLARE_IDENTIFIABLE(RemoveValueUpdate);
+
+};
+
+} // document
+
diff --git a/document/src/vespa/document/update/updates.h b/document/src/vespa/document/update/updates.h
new file mode 100644
index 00000000000..3475839243a
--- /dev/null
+++ b/document/src/vespa/document/update/updates.h
@@ -0,0 +1,14 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/document/update/documentupdate.h>
+#include <vespa/document/update/fieldupdate.h>
+
+#include <vespa/document/update/addvalueupdate.h>
+#include <vespa/document/update/arithmeticvalueupdate.h>
+#include <vespa/document/update/assignvalueupdate.h>
+#include <vespa/document/update/clearvalueupdate.h>
+#include <vespa/document/update/mapvalueupdate.h>
+#include <vespa/document/update/removevalueupdate.h>
+
diff --git a/document/src/vespa/document/update/updatevisitor.h b/document/src/vespa/document/update/updatevisitor.h
new file mode 100644
index 00000000000..f6887036f24
--- /dev/null
+++ b/document/src/vespa/document/update/updatevisitor.h
@@ -0,0 +1,36 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+namespace document {
+
+class DocumentUpdate;
+class FieldUpdate;
+class RemoveValueUpdate;
+class AddValueUpdate;
+class ArithmeticValueUpdate;
+class AssignValueUpdate;
+class ClearValueUpdate;
+class MapValueUpdate;
+class AddFieldPathUpdate;
+class AssignFieldPathUpdate;
+class RemoveFieldPathUpdate;
+
+struct UpdateVisitor {
+    virtual ~UpdateVisitor() {}
+
+    virtual void visit(const DocumentUpdate &value) = 0;
+    virtual void visit(const FieldUpdate &value) = 0;
+    virtual void visit(const RemoveValueUpdate &value) = 0;
+    virtual void visit(const AddValueUpdate &value) = 0;
+    virtual void visit(const ArithmeticValueUpdate &value) = 0;
+    virtual void visit(const AssignValueUpdate &value) = 0;
+    virtual void visit(const ClearValueUpdate &value) = 0;
+    virtual void visit(const MapValueUpdate &value) = 0;
+    virtual void visit(const AddFieldPathUpdate &value) = 0;
+    virtual void visit(const AssignFieldPathUpdate &value) = 0;
+    virtual void visit(const RemoveFieldPathUpdate &value) = 0;
+};
+
+}  // namespace document
+
diff --git a/document/src/vespa/document/update/valueupdate.cpp b/document/src/vespa/document/update/valueupdate.cpp
new file mode 100644
index 00000000000..6286c67b7fa
--- /dev/null
+++ b/document/src/vespa/document/update/valueupdate.cpp
@@ -0,0 +1,42 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <stdexcept>
+#include <vespa/document/base/field.h>
+#include <vespa/document/fieldvalue/fieldvalue.h>
+#include <vespa/document/fieldvalue/document.h>
+#include <vespa/document/update/addvalueupdate.h>
+#include <vespa/document/update/arithmeticvalueupdate.h>
+#include <vespa/document/update/assignvalueupdate.h>
+#include <vespa/document/update/clearvalueupdate.h>
+#include <vespa/document/update/mapvalueupdate.h>
+#include <vespa/document/update/removevalueupdate.h>
+#include <vespa/document/util/serializable.h>
+
+namespace document
+{
+
+IMPLEMENT_IDENTIFIABLE_ABSTRACT(ValueUpdate, Identifiable);
+
+// Create a value update from a byte buffer.
+std::unique_ptr<ValueUpdate>
+ValueUpdate::createInstance(const DocumentTypeRepo& repo,
+                            const DataType& type, ByteBuffer& buffer,
+                            int serializationVersion)
+{
+    ValueUpdate* update(NULL);
+    int32_t classId = 0;
+    buffer.getIntNetwork(classId);
+
+    const Identifiable::RuntimeClass * rtc(Identifiable::classFromId(classId));
+    if (rtc != NULL) {
+        update = static_cast<ValueUpdate*>(Identifiable::classFromId(classId)->create());
+        /// \todo TODO (was warning):  Updates are not versioned in serialization format. Will not work with altering it.
+        update->deserialize(repo, type, buffer, serializationVersion);
+    } else {
+        throw std::runtime_error(vespalib::make_string("Could not find a class for classId %d(%x)", classId, classId));
+    }
+
+    return std::unique_ptr<ValueUpdate>(update);
+}
+
+}
diff --git a/document/src/vespa/document/update/valueupdate.h b/document/src/vespa/document/update/valueupdate.h
new file mode 100644
index 00000000000..a11838095ae
--- /dev/null
+++ b/document/src/vespa/document/update/valueupdate.h
@@ -0,0 +1,118 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * @class document::ValueUpdate
+ * @ingroup document
+ *
+ * @brief Superclass for all types of field value update operations.
+ *
+ * It declares the interface required for all value updates.
+ *
+ * Furthermore, this class inherits from Printable without implementing its
+ * virtual {@link Printable#print} function, so that all subclasses must also
+ * implement a human readable output format.
+ *
+ * This class is a serializable, serializing its content to a buffer. It is
+ * however not a deserializable, as it does not serialize the datatype of the
+ * content it serializes, such that it needs to get a datatype specified upon
+ * deserialization.
+ */
+#pragma once
+
+
+#include <vespa/vespalib/objects/cloneable.h>
+
+#include <vespa/document/datatype/datatype.h>
+#include <vespa/document/util/serializable.h>
+#include <vespa/document/util/xmlserializable.h>
+#include <vespa/document/update/updatevisitor.h>
+
+namespace document {
+
+class ByteBuffer;
+class DocumentTypeRepo;
+class Field;
+class FieldValue;
+
+#define ACCEPT_UPDATE_VISITOR void accept(UpdateVisitor & visitor) const override { visitor.visit(*this); }
+
+class ValueUpdate : public vespalib::Identifiable,
+                    public Printable,
+                    public vespalib::Cloneable,
+                    public XmlSerializable
+{
+public:
+    typedef vespalib::CloneablePtr<ValueUpdate> CP;
+
+    /**
+     * Create a value update object from the given byte buffer.
+     *
+     * @param type A data type that describes the content of the buffer.
+     * @param buffer The byte buffer that containes the serialized update.
+     */
+    static std::unique_ptr<ValueUpdate> createInstance(
+            const DocumentTypeRepo& repo, const DataType& type,
+            ByteBuffer& buffer, int serializationVersion);
+
+    /** Define all types of value updates. */
+    enum ValueUpdateType {
+        Add        = IDENTIFIABLE_CLASSID(AddValueUpdate),
+        Arithmetic = IDENTIFIABLE_CLASSID(ArithmeticValueUpdate),
+        Assign     = IDENTIFIABLE_CLASSID(AssignValueUpdate),
+        Clear      = IDENTIFIABLE_CLASSID(ClearValueUpdate),
+        Map        = IDENTIFIABLE_CLASSID(MapValueUpdate),
+        Remove     = IDENTIFIABLE_CLASSID(RemoveValueUpdate)
+    };
+
+    ValueUpdate()
+        : Printable(), Cloneable(), XmlSerializable() {}
+
+    virtual ~ValueUpdate() {}
+
+    virtual bool operator==(const ValueUpdate&) const = 0;
+    bool operator != (const ValueUpdate & rhs) const { return ! (*this == rhs); }
+
+    /**
+     * Recursively checks the compatibility of this value update as
+     * applied to the given document field.
+     *
+     * @throws IllegalArgumentException Thrown if this value update
+     *                                  is not compatible.
+     */
+    virtual void checkCompatibility(const Field& field) const = 0;
+
+    /**
+     * Applies this value update to the given field value.
+     *
+     * @return True if value is updated, false if value should be removed.
+     */
+    virtual bool applyTo(FieldValue& value) const = 0;
+
+    virtual ValueUpdate* clone() const = 0;
+
+    /**
+     * Deserializes the given byte buffer into an instance of an update object.
+     *
+     * @param type A data type that describes the content of the buffer.
+     * @param buffer The byte buffer that contains the serialized update object.
+     * @param version The serialization version of the object to deserialize.
+     */
+    virtual void deserialize(const DocumentTypeRepo& repo,
+                             const DataType& type,
+                             ByteBuffer& buffer, uint16_t version) = 0;
+
+    /** @return The operation type. */
+    ValueUpdateType getType() const {
+        return static_cast<ValueUpdateType>(getClass().id());
+    }
+
+    /**
+     * Visit this fieldvalue for double dispatch.
+     */
+    virtual void accept(UpdateVisitor &visitor) const = 0;
+
+    DECLARE_IDENTIFIABLE_ABSTRACT(ValueUpdate);
+
+};
+
+} // document
+
diff --git a/document/src/vespa/document/util/.gitignore b/document/src/vespa/document/util/.gitignore
new file mode 100644
index 00000000000..583460ae288
--- /dev/null
+++ b/document/src/vespa/document/util/.gitignore
@@ -0,0 +1,3 @@
+*.So
+.depend
+Makefile
diff --git a/document/src/vespa/document/util/CMakeLists.txt b/document/src/vespa/document/util/CMakeLists.txt
new file mode 100644
index 00000000000..d8fe0d833ff
--- /dev/null
+++ b/document/src/vespa/document/util/CMakeLists.txt
@@ -0,0 +1,12 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_library(document_util OBJECT
+    SOURCES
+    stringutil.cpp
+    bytebuffer.cpp
+    serializable.cpp
+    compressor.cpp
+    printable.cpp
+    DEPENDS
+    AFTER
+    document_documentconfig
+)
diff --git a/document/src/vespa/document/util/bytebuffer.cpp b/document/src/vespa/document/util/bytebuffer.cpp
new file mode 100644
index 00000000000..070d52ef8d9
--- /dev/null
+++ b/document/src/vespa/document/util/bytebuffer.cpp
@@ -0,0 +1,761 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+   @author Thomas F. Gundersen, �ystein Fledsberg
+   @version $Id$
+   @date 2004-03-15
+*/
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/document/util/bytebuffer.h>
+#include <vespa/vespalib/stllike/asciistream.h>
+#include <sstream>
+#include <stdio.h>
+#include <string.h>
+#include <iostream>
+
+#define LOG_DEBUG1(a)
+// Enable this macros instead to see what bytebuffer calls come
+//#define LOG_DEBUG1(a) std::cerr << "ByteBuffer(" << ((void*) this) << " " << a << ")\n";
+
+#define LOG_DEBUG2(a,b) LOG_DEBUG1(vespalib::make_string(a,b));
+#define LOG_DEBUG3(a,b,c) LOG_DEBUG1(vespalib::make_string(a,b,c));
+#define LOG_DEBUG4(a,b,c,d) LOG_DEBUG1(vespalib::make_string(a,b,c,d));
+
+using vespalib::DefaultAlloc;
+
+namespace document {
+
+VESPA_IMPLEMENT_EXCEPTION_SPINE(BufferOutOfBoundsException);
+VESPA_IMPLEMENT_EXCEPTION_SPINE(InputOutOfRangeException);
+
+vespalib::string BufferOutOfBoundsException::createMessage(size_t pos, size_t len) {
+    vespalib::asciistream ost;
+    ost << pos << " > " << len;
+    return ost.str();
+}
+
+BufferOutOfBoundsException::BufferOutOfBoundsException(
+        size_t pos, size_t len, const vespalib::string& location)
+    : IoException(createMessage(pos, len), IoException::NO_SPACE, location, 1)
+{
+}
+
+InputOutOfRangeException::InputOutOfRangeException(
+        const vespalib::string& msg, const vespalib::string& location)
+    : IoException(msg, IoException::INTERNAL_FAILURE, location, 1)
+{
+}
+
+ByteBuffer::ByteBuffer() :
+      _buffer(NULL),
+      _len(0),
+      _pos(0),
+      _limit(0),
+      _bufHolder(NULL),
+      _ownedBuffer()
+{
+    set(NULL, 0);
+    LOG_DEBUG1("Created empty bytebuffer");
+}
+
+ByteBuffer::ByteBuffer(size_t len) :
+    ByteBuffer(DefaultAlloc(len), len)
+{
+}
+
+ByteBuffer::ByteBuffer(const char* buffer, size_t len) :
+      _buffer(NULL),
+      _len(0),
+      _pos(0),
+      _limit(0),
+      _bufHolder(NULL),
+      _ownedBuffer()
+{
+    set(buffer, len);
+}
+
+ByteBuffer::ByteBuffer(DefaultAlloc buffer, size_t len) :
+      _buffer(static_cast<char *>(buffer.get())),
+      _len(len),
+      _pos(0),
+      _limit(len),
+      _bufHolder(NULL),
+      _ownedBuffer(std::move(buffer))
+{
+}
+
+ByteBuffer::ByteBuffer(BufferHolder* buf, size_t pos, size_t len, size_t limit) :
+      _buffer(NULL),
+      _len(0),
+      _pos(0),
+      _limit(0),
+      _bufHolder(NULL),
+      _ownedBuffer()
+{
+    set(buf, pos, len, limit);
+    LOG_DEBUG3("Created copy of byte buffer of length %" PRIu64 " with "
+               "limit %" PRIu64 ".", len, limit);
+}
+
+ByteBuffer::ByteBuffer(const ByteBuffer& bb) :
+      _buffer(0),
+      _len(0),
+      _pos(0),
+      _limit(0),
+      _bufHolder(NULL),
+      _ownedBuffer()
+{
+    LOG_DEBUG1("Created empty byte buffer to assign to.");
+    *this = bb;
+}
+
+ByteBuffer& ByteBuffer::operator=(const ByteBuffer & org)
+{
+    if (this != & org) {
+        cleanUp();
+        if (org._len > 0 && org._buffer) {
+            DefaultAlloc(org._len + 1).swap(_ownedBuffer);
+            _buffer = static_cast<char *>(_ownedBuffer.get());
+            memcpy(_buffer,org._buffer,org._len);
+            _buffer[org._len] = 0;
+        }
+        _len = org._len;
+        _pos = org._pos;
+        _limit = org._limit;
+        LOG_DEBUG4("Assignment created new buffer of size %" PRIu64 " at pos "
+                   "%" PRIu64 " with limit %" PRIu64 ".",
+                   _len, _pos, _limit);
+    }
+    return *this;
+}
+
+void
+ByteBuffer::set(BufferHolder* buf, size_t pos, size_t len, size_t limit)
+{
+    cleanUp();
+    _bufHolder = buf;
+    _bufHolder->addRef();
+    _buffer = static_cast<char *>(_bufHolder->_buffer.get());
+    _pos=pos;
+    _len=len;
+    _limit=limit;
+    LOG_DEBUG4("set() created new buffer of size %" PRIu64 " at pos "
+               "%" PRIu64 " with limit %" PRIu64 ".",
+               _len, _pos, _limit);
+}
+
+ByteBuffer::~ByteBuffer()
+{
+    if (_bufHolder) {
+        _bufHolder->subRef();
+    }
+}
+
+std::unique_ptr<ByteBuffer>
+ByteBuffer::sliceCopy() const
+{
+    ByteBuffer* buf = new ByteBuffer;
+    buf->sliceFrom(*this, _pos, _limit);
+
+    LOG_DEBUG3("Created slice at pos %" PRIu64 " with limit %" PRIu64 ".",
+               _pos, _limit);
+    return std::unique_ptr<ByteBuffer>(buf);
+}
+
+void ByteBuffer::throwOutOfBounds(size_t want, size_t has)
+{
+    LOG_DEBUG1("Throwing out of bounds exception");
+    throw BufferOutOfBoundsException(want, has, VESPA_STRLOC);
+}
+
+void
+ByteBuffer::sliceFrom(const ByteBuffer& buf, size_t from, size_t to) // throw (BufferOutOfBoundsException)
+{
+    LOG_DEBUG3("Created slice from buffer from %" PRIu64 " to %" PRIu64 ".",
+               from, to);
+    if (from > buf._len) {
+        throwOutOfBounds(from, buf._len);
+    } else if (to > buf._len) {
+        throwOutOfBounds(to, buf._len);
+    } else if (to < from) {
+        throwOutOfBounds(to, from);
+    } else {
+
+        if (!buf._buffer) {
+            clear();
+            return;
+        }
+
+        // Slicing from someone that doesn't own their buffer, must make own copy.
+        if (( buf._ownedBuffer.get() == NULL ) && (buf._bufHolder == NULL)) {
+            cleanUp();
+            DefaultAlloc(to-from + 1).swap(_ownedBuffer);
+            _buffer = static_cast<char *>(_ownedBuffer.get());
+            memcpy(_buffer, buf._buffer + from, to-from);
+            _buffer[to-from] = 0;
+            _pos = 0;
+            _len = _limit = to-from;
+            return;
+        }
+
+        // Slicing from someone that owns, but hasn't made a reference counter yet.
+        if (!buf._bufHolder) {
+            buf._bufHolder=new BufferHolder(std::move(const_cast<DefaultAlloc &>(buf._ownedBuffer)));
+        }
+
+        // Slicing from refcounter.
+        cleanUp();
+
+        _bufHolder = buf._bufHolder;
+        _bufHolder->addRef();
+        _buffer = static_cast<char *>(_bufHolder->_buffer.get());
+        _pos=from;
+        _len=to;
+        _limit=to;
+    }
+}
+
+ByteBuffer* ByteBuffer::copyBuffer(const char* buffer, size_t len)
+{
+    if (buffer && len) {
+        DefaultAlloc newBuf(len + 1);
+        memcpy(newBuf.get(), buffer, len);
+        static_cast<char *>(newBuf.get())[len] = 0;
+        return new ByteBuffer(std::move(newBuf), len);
+    } else {
+        return NULL;
+    }
+}
+
+void
+ByteBuffer::setPos(size_t pos) // throw (BufferOutOfBoundsException)
+{
+    LOG_DEBUG3("Setting pos to be %" PRIu64 ", limit is %" PRIu64 ".",
+               pos, _limit);
+    if (pos>_limit) {
+        throwOutOfBounds(pos, _limit);
+    } else {
+        _pos=pos;
+    }
+}
+
+void
+ByteBuffer::setLimit(size_t limit) // throw (BufferOutOfBoundsException)
+{
+    LOG_DEBUG3("Setting limit to %" PRIu64 ", (size is %" PRIu64 ").", limit, _len);
+    if (limit>_len) {
+        throwOutOfBounds(limit, _len);
+    } else {
+        _limit=limit;
+    }
+}
+
+void ByteBuffer::clear()
+{
+    LOG_DEBUG1("Clearing content. Setting pos to 0");
+    _pos=0;
+    _limit=_len;
+}
+
+void ByteBuffer::flip()
+{
+    LOG_DEBUG2("Flipping buffer. Setting limit to %" PRIu64 ".", _pos);
+    _limit = _pos;
+    _pos = 0;
+}
+
+
+ByteBuffer::BufferHolder::BufferHolder(DefaultAlloc buffer)
+    : _buffer(std::move(buffer))
+{
+}
+
+ByteBuffer::BufferHolder::~BufferHolder()
+{
+}
+
+void ByteBuffer::dump() const
+{
+    fprintf(stderr, "ByteBuffer: Length %lu, Pos %lu, Limit %lu\n",
+            _len, _pos, _limit);
+    for (size_t i=0; i<_len; i++) {
+        if (_buffer[i]>32 && _buffer[i]<126) {
+            fprintf(stderr, "%c", _buffer[i]);
+        } else {
+            fprintf(stderr, "[%d]",_buffer[i]);
+        }
+    }
+}
+
+void ByteBuffer::incPos(size_t pos)
+{
+    LOG_DEBUG2("incPos(%" PRIu64 ")", pos);
+    if (_pos + pos > _limit) {
+        throwOutOfBounds(_pos + pos, _limit);
+    } else {
+        _pos+=pos;
+#ifdef __FORCE_VALGRIND_ON_SERIALIZE__
+        forceValgrindStart2Pos();
+#endif
+    }
+}
+
+void ByteBuffer::getNumeric(uint8_t & v) {
+    LOG_DEBUG2("getNumeric8(%d)", (int) v);
+    if (__builtin_expect(getRemaining() < sizeof(v), 0)) {
+        throwOutOfBounds(getRemaining(), sizeof(v));
+    } else {
+        v = *(uint8_t *) getBufferAtPos();
+        incPosNoCheck(sizeof(v));
+    }
+}
+
+void ByteBuffer::putNumeric(uint8_t v) {
+    LOG_DEBUG2("putNumeric8(%d)", (int) v);
+    if (__builtin_expect(getRemaining() < sizeof(v), 0)) {
+        throwOutOfBounds(getRemaining(), sizeof(v));
+    } else {
+        *(uint8_t *) getBufferAtPos() = v;
+        incPosNoCheck(sizeof(v));
+    }
+}
+
+size_t ByteBuffer::forceValgrindStart2Pos() const
+{
+    size_t zeroCount(0);
+    if (_buffer) {
+        for(const char * c(_buffer), *e(c + _pos); c < e; c++) {
+            if (*c == 0) {
+                zeroCount++;
+            }
+        }
+    }
+    return zeroCount;
+}
+
+size_t ByteBuffer::forceValgrindPos2Lim() const
+{
+    size_t zeroCount(0);
+    if (_buffer) {
+        for(const char * c(getBufferAtPos()), *e(c + getRemaining()); c < e; c++) {
+            if (*c == 0) {
+                zeroCount++;
+            }
+        }
+    }
+    return zeroCount;
+}
+
+
+void ByteBuffer::getNumericNetwork(int16_t & v) {
+    LOG_DEBUG2("getNumericNetwork16(%d)", (int) v);
+    if (__builtin_expect(getRemaining() < sizeof(v), 0)) {
+        throwOutOfBounds(getRemaining(), sizeof(v));
+    } else {
+        uint16_t val = *(uint16_t *) getBufferAtPos();
+        v = ntohs(val);
+        incPosNoCheck(sizeof(v));
+    }
+}
+
+void ByteBuffer::getNumeric(int16_t & v) {
+    LOG_DEBUG2("getNumeric16(%d)", (int) v);
+    if (__builtin_expect(getRemaining() < sizeof(v), 0)) {
+        throwOutOfBounds(getRemaining(), sizeof(v));
+    } else {
+        v = *(int16_t *) getBufferAtPos();
+        incPosNoCheck(sizeof(v));
+    }
+}
+
+void ByteBuffer::putNumericNetwork(int16_t v) {
+    LOG_DEBUG2("putNumericNetwork16(%d)", (int) v);
+    if (__builtin_expect(getRemaining() < sizeof(v), 0)) {
+        throwOutOfBounds(getRemaining(), sizeof(v));
+    } else {
+        uint16_t val = htons(v);
+        *(uint16_t *) getBufferAtPos() = val;
+        incPosNoCheck(sizeof(v));
+    }
+}
+
+void ByteBuffer::putNumeric(int16_t v) {
+    LOG_DEBUG2("putNumeric16(%d)", (int) v);
+    if (__builtin_expect(getRemaining() < sizeof(v), 0)) {
+        throwOutOfBounds(getRemaining(), sizeof(v));
+    } else {
+        *(int16_t *) getBufferAtPos() = v;
+        incPosNoCheck(sizeof(v));
+    }
+}
+
+void ByteBuffer::getNumericNetwork(int32_t & v) {
+    LOG_DEBUG2("getNumericNetwork32(%d)", (int) v);
+    if (__builtin_expect(getRemaining() < sizeof(v), 0)) {
+        throwOutOfBounds(getRemaining(), sizeof(v));
+    } else {
+        uint32_t val = *(uint32_t *) getBufferAtPos();
+        v = ntohl(val);
+        incPosNoCheck(sizeof(v));
+    }
+}
+
+void ByteBuffer::getNumeric(int32_t & v) {
+    LOG_DEBUG2("getNumeric32(%d)", (int) v);
+    if (__builtin_expect(getRemaining() < sizeof(v), 0)) {
+        throwOutOfBounds(getRemaining(), sizeof(v));
+    } else {
+        v = *(int32_t *) getBufferAtPos();
+        incPosNoCheck(sizeof(v));
+    }
+}
+
+
+void ByteBuffer::putNumericNetwork(int32_t v) {
+    LOG_DEBUG2("putNumericNetwork32(%d)", (int) v);
+    if (__builtin_expect(getRemaining() < sizeof(v), 0)) {
+        throwOutOfBounds(getRemaining(), sizeof(v));
+    } else {
+        uint32_t val = htonl(v);
+        *(uint32_t *) getBufferAtPos() = val;
+        incPosNoCheck(sizeof(v));
+    }
+}
+
+void ByteBuffer::putNumeric(int32_t v) {
+    LOG_DEBUG2("putNumeric32(%d)", (int) v);
+    if (__builtin_expect(getRemaining() < sizeof(v), 0)) {
+        throwOutOfBounds(getRemaining(), sizeof(v));
+    } else {
+        *(int32_t *) getBufferAtPos() = v;
+        incPosNoCheck(sizeof(v));
+    }
+}
+
+void ByteBuffer::getNumericNetwork(float & v) {
+    LOG_DEBUG2("getNumericNetworkFloat(%f)", v);
+    // XXX depends on sizeof(float) == sizeof(uint32_t) == 4
+    // and endianness same for float and ints
+    int32_t val;
+    getIntNetwork(val);
+    memcpy(&v, &val, sizeof(v));
+}
+
+void ByteBuffer::getNumeric(float & v) {
+    LOG_DEBUG2("getNumericFloat(%f)", v);
+    if (__builtin_expect(getRemaining() < sizeof(v), 0)) {
+        throwOutOfBounds(getRemaining(), sizeof(v));
+    } else {
+        v = *(float *) getBufferAtPos();
+        incPosNoCheck(sizeof(v));
+    }
+}
+
+void ByteBuffer::putNumericNetwork(float v) {
+    LOG_DEBUG2("putNumericNetworkFloat(%f)", v);
+    // XXX depends on sizeof(float) == sizeof(int32_t) == 4
+    // and endianness same for float and ints
+    int32_t val;
+    memcpy(&val, &v, sizeof(val));
+    putIntNetwork(val);
+}
+
+void ByteBuffer::putNumeric(float v) {
+    LOG_DEBUG2("putNumericFloat(%f)", v);
+    if (__builtin_expect(getRemaining() < sizeof(v), 0)) {
+        throwOutOfBounds(getRemaining(), sizeof(v));
+    } else {
+        *(float *) getBufferAtPos() = v;
+        incPosNoCheck(sizeof(v));
+    }
+}
+void ByteBuffer::getNumeric(int64_t& v) {
+    LOG_DEBUG2("getNumeric64(%" PRId64 ")", v);
+    if (__builtin_expect(getRemaining() < sizeof(v), 0)) {
+        throwOutOfBounds(getRemaining(), sizeof(v));
+    } else {
+        v = *(int64_t *) getBufferAtPos();
+        incPosNoCheck(sizeof(v));
+    }
+}
+void ByteBuffer::putNumeric(int64_t v) {
+    LOG_DEBUG2("putNumeric64(%" PRId64 ")", v);
+    if (__builtin_expect(getRemaining() < sizeof(v), 0)) {
+        throwOutOfBounds(getRemaining(), sizeof(v));
+    } else {
+        *(int64_t *) getBufferAtPos() = v;
+        incPosNoCheck(sizeof(v));
+    }
+}
+void ByteBuffer::getNumeric(double& v) {
+    LOG_DEBUG2("getNumericDouble(%f)", v);
+    if (__builtin_expect(getRemaining() < sizeof(v), 0)) {
+        throwOutOfBounds(getRemaining(), sizeof(v));
+    } else {
+        v = *(double *) getBufferAtPos();
+        incPosNoCheck(sizeof(v));
+    }
+}
+void ByteBuffer::putNumeric(double v) {
+    LOG_DEBUG2("putNumericDouble(%f)", v);
+    if (__builtin_expect(getRemaining() < sizeof(v), 0)) {
+        throwOutOfBounds(getRemaining(), sizeof(v));
+    } else {
+        *(double *) getBufferAtPos() = v;
+        incPosNoCheck(sizeof(v));
+    }
+}
+
+void ByteBuffer::getNumericNetwork(double & v) {
+    LOG_DEBUG2("getNumericNetworkDouble(%f)", v);
+    getDoubleLongNetwork(v);
+}
+void ByteBuffer::putNumericNetwork(int64_t v) {
+    LOG_DEBUG2("putNumericNetwork64(%" PRId64 ")", v);
+    putDoubleLongNetwork(v);
+}
+void ByteBuffer::putNumericNetwork(double v) {
+    LOG_DEBUG2("putNumericNetworkDouble(%f)", v);
+    putDoubleLongNetwork(v);
+}
+void ByteBuffer::getNumericNetwork(int64_t & v) {
+    LOG_DEBUG2("getNumericNetwork64(%" PRId64 ")", v);
+    getDoubleLongNetwork(v);
+}
+
+void ByteBuffer::putInt2_4_8Bytes(int64_t number, size_t len) {
+    LOG_DEBUG3("putInt2_4_8(%" PRId64 ", %" PRIu64 ")", number, len);
+    if (number < 0ll) {
+        throw InputOutOfRangeException(vespalib::make_string(
+                    "Cannot encode negative number."), VESPA_STRLOC);
+    } else if (number > 0x3FFFFFFFFFFFFFFFll) {
+        throw InputOutOfRangeException(vespalib::make_string(
+                    "Cannot encode number larger than 2^62."), VESPA_STRLOC);
+    }
+
+    if (len == 0) {
+        if (number < 0x8000ll) {
+            //length 2 bytes
+            putShortNetwork((int16_t) number);
+        } else if (number < 0x40000000ll) {
+            //length 4 bytes
+            putIntNetwork(((int32_t) number) | 0x80000000);
+        } else {
+            //length 8 bytes
+            putLongNetwork(number | 0xC000000000000000ll);
+        }
+    } else if (len == 2) {
+        //length 2 bytes
+        putShortNetwork((int16_t) number);
+    } else if (len == 4) {
+        //length 4 bytes
+        putIntNetwork(((int32_t) number) | 0x80000000);
+    } else if (len == 8) {
+        //length 8 bytes
+        putLongNetwork(number | 0xC000000000000000ll);
+    } else {
+        throw InputOutOfRangeException(vespalib::make_string(
+                "Cannot encode number using %d bytes.", (int)len), VESPA_STRLOC);
+    }
+}
+
+void ByteBuffer::getInt2_4_8Bytes(int64_t & v) {
+    LOG_DEBUG2("getInt2_4_8(%" PRId64 ")", v);
+    if (getRemaining() >= 2) {
+        uint8_t flagByte = peekByte();
+
+        if (flagByte & 0x80) {
+            if (flagByte & 0x40) {
+                //length 8 bytes
+                int64_t tmp;
+                getLongNetwork(tmp);
+                v = tmp & 0x3FFFFFFFFFFFFFFFll;
+            } else {
+                //length 4 bytes
+                int32_t tmp;
+                getIntNetwork(tmp);
+                v = (int64_t) (tmp & 0x3FFFFFFF);
+            }
+        } else {
+            //length 2 bytes
+            int16_t tmp;
+            getShortNetwork(tmp);
+            v = (int64_t) tmp;
+        }
+    } else {
+        throwOutOfBounds(getRemaining(), 2);
+    }
+}
+
+size_t ByteBuffer::getSerializedSize2_4_8Bytes(int64_t number) {
+    if (number < 0ll) {
+        throw InputOutOfRangeException(vespalib::make_string(
+                    "Cannot encode negative number."), VESPA_STRLOC);
+    } else if (number > 0x3FFFFFFFFFFFFFFFll) {
+        throw InputOutOfRangeException(vespalib::make_string(
+                "Cannot encode number larger than 2^62."), VESPA_STRLOC);
+    }
+
+    if (number < 0x8000ll) {
+        return 2;
+    } else if (number < 0x40000000ll) {
+        return 4;
+    } else {
+        return 8;
+    }
+}
+
+void ByteBuffer::putInt1_2_4Bytes(int32_t number) {
+    LOG_DEBUG2("putInt1_2_4Bytes(%i)", number);
+    if (number < 0) {
+        throw InputOutOfRangeException(vespalib::make_string(
+                    "Cannot encode negative number."), VESPA_STRLOC);
+    } else if (number > 0x3FFFFFFF) {
+        throw InputOutOfRangeException(vespalib::make_string(
+                    "Cannot encode number larger than 2^30."), VESPA_STRLOC);
+    }
+
+    if (number < 0x80) {
+        putByte((unsigned char) number);
+    } else if (number < 0x4000) {
+        putShortNetwork((int16_t) (((int16_t)number) | ((int16_t) 0x8000)));
+    } else {
+        putIntNetwork(number | 0xC0000000);
+    }
+}
+
+void ByteBuffer::getInt1_2_4Bytes(int32_t & v) {
+    LOG_DEBUG2("getInt1_2_4Bytes(%i)", v);
+    if (getRemaining() >= 1) {
+        unsigned char flagByte = peekByte();
+
+        if (flagByte & 0x80) {
+            if (flagByte & 0x40) {
+                //length 4 bytes
+                int32_t tmp;
+                getIntNetwork(tmp);
+                v = tmp & 0x3FFFFFFF;
+            } else {
+                //length 2 bytes
+                int16_t tmp;
+                getShortNetwork(tmp);
+                v = (int32_t) (tmp & ((int16_t) 0x3FFF));
+            }
+        } else {
+            v = (int32_t) flagByte;
+            incPosNoCheck(1);
+        }
+    } else {
+        throwOutOfBounds(getRemaining(), 1);
+    }
+}
+
+size_t ByteBuffer::getSerializedSize1_2_4Bytes(int32_t number) {
+    if (number < 0) {
+        throw InputOutOfRangeException(vespalib::make_string(
+                    "Cannot encode negative number."), VESPA_STRLOC);
+    } else if (number > 0x3FFFFFFF) {
+        throw InputOutOfRangeException(vespalib::make_string(
+                    "Cannot encode number larger than 2^30."), VESPA_STRLOC);
+    }
+
+    if (number < 0x80) {
+        return 1;
+    } else if (number < 0x4000) {
+        return 2;
+    } else {
+        return 4;
+    }
+}
+void ByteBuffer::putInt1_4Bytes(int32_t number) {
+    LOG_DEBUG2("putInt1_4Bytes(%i)", number);
+    if (number < 0) {
+        throw InputOutOfRangeException(vespalib::make_string(
+                    "Cannot encode negative number."), VESPA_STRLOC);
+    } else if (number > 0x7FFFFFFF) {
+        throw InputOutOfRangeException(vespalib::make_string(
+                    "Cannot encode number larger than 2^31."), VESPA_STRLOC);
+    }
+
+    if (number < 0x80) {
+        putByte((unsigned char) number);
+    } else {
+        putIntNetwork(number | 0x80000000);
+    }
+}
+void ByteBuffer::getInt1_4Bytes(int32_t & v) {
+    LOG_DEBUG2("getInt1_4Bytes(%i)", v);
+    if (getRemaining() >= 1) {
+        unsigned char flagByte = peekByte();
+
+        if (flagByte & 0x80) {
+            //length 4 bytes
+            int32_t tmp;
+            getIntNetwork(tmp);
+            v = tmp & 0x7FFFFFFF;
+        } else {
+            v = (int32_t) flagByte;
+            incPosNoCheck(1);
+        }
+    } else {
+        throwOutOfBounds(getRemaining(), 1);
+    }
+}
+size_t ByteBuffer::getSerializedSize1_4Bytes(int32_t number) {
+    if (number < 0) {
+        throw InputOutOfRangeException(vespalib::make_string(
+                "Cannot encode negative number."), VESPA_STRLOC);
+    } else if (number > 0x7FFFFFFF) {
+        throw InputOutOfRangeException(vespalib::make_string(
+            "Cannot encode number larger than 2^31."), VESPA_STRLOC);
+    }
+
+    if (number < 0x80) {
+        return 1;
+    } else {
+        return 4;
+    }
+}
+void ByteBuffer::getBytes(void *buffer, size_t count)
+{
+    LOG_DEBUG3("getBytes(%p, %" PRIu64 ")", buffer, count);
+    const char *v = getBufferAtPos();
+    incPos(count);
+    memcpy(buffer, v, count);
+}
+void ByteBuffer::putBytes(const void *buf, size_t count) {
+    LOG_DEBUG3("putBytes(%p, %" PRIu64 ")", buf, count);
+    if (__builtin_expect(getRemaining() < count, 0)) {
+        throwOutOfBounds(getRemaining(), sizeof(count));
+    } else {
+        memcpy(getBufferAtPos(), buf, count);
+        incPosNoCheck(count);
+    }
+}
+std::string ByteBuffer::toString() {
+    std::ostringstream ost;
+    StringUtil::printAsHex(ost, getBuffer(), getLength());
+    return ost.str();
+}
+
+void ByteBuffer::swap(ByteBuffer& other) {
+    LOG_DEBUG2("swap(%p)", &other);
+    std::swap(_bufHolder, other._bufHolder);
+    std::swap(_buffer, other._buffer);
+    std::swap(_len, other._len);
+    std::swap(_pos, other._pos);
+    std::swap(_limit, other._limit);
+}
+
+void ByteBuffer::cleanUp() {
+    LOG_DEBUG1("cleanUp()");
+    if (_bufHolder) {
+        _bufHolder->subRef();
+        _bufHolder = NULL;
+    } else {
+        DefaultAlloc().swap(_ownedBuffer);
+    }
+    _buffer = NULL;
+}
+
+} // document
diff --git a/document/src/vespa/document/util/bytebuffer.h b/document/src/vespa/document/util/bytebuffer.h
new file mode 100644
index 00000000000..cfb72e092a5
--- /dev/null
+++ b/document/src/vespa/document/util/bytebuffer.h
@@ -0,0 +1,423 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * \class document::ByteBuffer
+ * \ingroup util
+ *
+ * \brief Java like bytebuffer class
+ *
+ * This class wraps a char* buffer with a length and position.
+ * It can be used to hide from the user whether the buffer was
+ * allocated or not, and can hold a position in the buffer which
+ * can be used for streaming-like behaviour.
+ *
+ * @author Thomas F. Gundersen, �ystein Fledsberg, Einar Rosenvinge
+ */
+#pragma once
+
+#include <vespa/fastos/types.h>
+#include <vespa/vespalib/util/referencecounter.h>
+#include <vespa/document/util/stringutil.h>
+#include <vespa/vespalib/util/exception.h>
+#include <vespa/vespalib/util/linkedptr.h>
+#include <memory>
+
+namespace document {
+
+class BufferOutOfBoundsException : public vespalib::IoException {
+    static vespalib::string createMessage(size_t pos, size_t len);
+public:
+    BufferOutOfBoundsException(size_t pos, size_t len,
+                               const vespalib::string& location = "");
+
+    VESPA_DEFINE_EXCEPTION_SPINE(BufferOutOfBoundsException)
+};
+
+class InputOutOfRangeException : public vespalib::IoException {
+public:
+    InputOutOfRangeException(const vespalib::string& msg,
+                             const vespalib::string& location = "");
+
+    VESPA_DEFINE_EXCEPTION_SPINE(InputOutOfRangeException)
+};
+
+class ByteBuffer
+{
+public:
+    typedef vespalib::LinkedPtr<ByteBuffer> LP;
+    typedef std::unique_ptr<ByteBuffer> UP;
+    /**
+     * Creates a byte buffer with no underlying buffer.
+     * Use set() to set the buffer.
+     */
+    ByteBuffer();
+
+    ByteBuffer(const ByteBuffer &);
+    ByteBuffer& operator=(const ByteBuffer &);
+
+    ~ByteBuffer();
+
+    /** Allocates buffer with len bytes. */
+    ByteBuffer(size_t len);
+
+    /**
+     * Create a buffer with the given content.
+     *
+     * @param buffer The buffer to represent.
+     * @param len The length of the buffer
+     */
+    ByteBuffer(const char* buffer, size_t len);
+
+    /**
+     * Create a buffer with the given content.
+     *
+     * @param buffer The buffer to represent.
+     * @param len The length of the buffer
+     */
+    ByteBuffer(vespalib::DefaultAlloc buffer, size_t len);
+
+    /**
+     * Sets the buffer pointed to by this buffer. Allows for multiple
+     * usages of the same ByteBuffer.
+     */
+    void set(const char* buffer, size_t len) {
+        cleanUp();
+        _buffer = const_cast<char*>(buffer);
+        _len=len;
+        _limit=len;
+        _pos=0;
+    }
+
+    /** Clear this buffer, and set free the underlying BufferHolder. */
+    void reset() { set(NULL, 0); }
+
+    /**
+     * Creates a ByteBuffer object from another buffer. allocates
+     * a new buffer of same size and copies the content.
+     *
+     * @param buffer The buffer to copy.
+     * @param len The length of the buffer.
+     *
+     *  @return Returns a newly created bytebuffer object, or NULL
+     *  if buffer was NULL, or len was <=0.
+     */
+    static ByteBuffer* copyBuffer(const char* buffer, size_t len);
+
+    std::unique_ptr<ByteBuffer> sliceCopy() const;
+
+    /**
+     * @throws BufferOutOfBoundsException If faulty range is given.
+     */
+    void sliceFrom(const ByteBuffer& buf, size_t from, size_t to);
+
+    /** @return Returns the buffer pointed to by this object (at position 0) */
+    char* getBuffer() const { return _buffer; }
+
+    /** @return Returns the length of the buffer pointed to by this object. */
+    size_t getLength() const { return _len; }
+
+    /**
+     * Adjust the length of the buffer. Only sane to shorten it, as you do not
+     * know what is ahead.
+     */
+    void setLength(size_t len) { _len = len; }
+
+    /** @return Returns a pointer to the current position in the buffer. */
+    char* getBufferAtPos() const { return _buffer + _pos; }
+
+    /** @return Returns the index of the current position in the buffer. */
+    size_t getPos() const { return _pos; }
+
+    /** @return Returns the limit. */
+    size_t getLimit() const { return _limit; }
+
+    /**
+     * @return Returns the number of bytes remaining in the buffer - that is,
+     *         getLimit()-getPos().
+    */
+    size_t getRemaining() const { return _limit-_pos; }
+
+    /**
+     * Changes the position in the buffer.
+     *
+     * @throws BufferOutOfBoundsException;
+     */
+    void setPos(size_t pos);
+
+    /**
+     * Sets the buffer limit.
+     *
+     * @param limit The new limit.
+     * @return True if the limit is legal (less than the length)
+     * @throws BufferOutOfBoundsException;
+     */
+    void setLimit(size_t limit);
+    size_t forceValgrindStart2Pos() const __attribute__ ((noinline));
+    size_t forceValgrindPos2Lim() const __attribute__ ((noinline));
+
+    /**
+     * Moves the position in the buffer.
+     *
+     * @param pos The number of bytes to move the position. The new position
+     *            will be oldPos + pos. This is the same as doing
+     *            setPos(getPos()+pos)
+     * @return    True if the position could be moved (it was inside the limit
+     *            of the buffer).
+     * @throws BufferOutOfBoundsException;
+     */
+    void incPos(size_t pos);
+
+    void incPosNoCheck(size_t pos) {
+        _pos += pos;
+#ifdef __FORCE_VALGRIND_ON_SERIALIZE__
+        forceValgrindStart2Pos();
+#endif
+    }
+
+    /**
+     * Resets pos to 0, and sets limit to old pos. Use this before reading
+     * from a buffer you have written to
+     */
+    void flip();
+
+    /**
+     * Sets pos to 0 and limit to length. Use this to start writing from the
+     * start of the buffer.
+     */
+    void clear();
+
+    void getNumericNetwork(uint8_t & v) { getNumeric(v); }
+    void getNumeric(uint8_t & v);
+    void putNumericNetwork(uint8_t v) { putNumeric(v); }
+    void putNumeric(uint8_t v);
+    void getNumericNetwork(int16_t & v);
+    void getNumeric(int16_t & v);
+    void putNumericNetwork(int16_t v);
+    void putNumeric(int16_t v);
+    void getNumericNetwork(int32_t & v);
+    void getNumeric(int32_t & v);
+    void putNumericNetwork(int32_t v);
+    void putNumeric(int32_t v);
+    void getNumericNetwork(float & v);
+    void getNumeric(float & v);
+    void putNumericNetwork(float v);
+    void putNumeric(float v);
+    void getNumericNetwork(int64_t & v);
+    void getNumeric(int64_t& v);
+    void putNumericNetwork(int64_t v);
+    void putNumeric(int64_t v);
+    void getNumericNetwork(double & v);
+    void getNumeric(double& v);
+    void putNumericNetwork(double v);
+    void putNumeric(double v);
+
+    void getByte(uint8_t & v)         { getNumeric(v); }
+    void putByte(uint8_t v)           { putNumeric(v); }
+    void getShortNetwork(int16_t & v) { getNumericNetwork(v); }
+    void getShort(int16_t & v)        { getNumeric(v); }
+    void putShortNetwork(int16_t v)   { putNumericNetwork(v); }
+    void putShort(int16_t v)          { putNumeric(v); }
+    void getIntNetwork(int32_t & v)   { getNumericNetwork(v); }
+    void getInt(int32_t & v)          { getNumeric(v); }
+    void putIntNetwork(int32_t v)     { putNumericNetwork(v); }
+    void putInt(int32_t v)            { putNumeric(v); }
+    void getFloatNetwork(float & v)   { getNumericNetwork(v); }
+    void getFloat(float & v)          { getNumeric(v); }
+    void putFloatNetwork(float v)     { putNumericNetwork(v); }
+    void putFloat(float v)            { putNumeric(v); }
+    void getLongNetwork(int64_t & v)  { getNumericNetwork(v); }
+    void getLong(int64_t& v)          { getNumeric(v); }
+    void putLongNetwork(int64_t v)    { putNumericNetwork(v); }
+    void putLong(int64_t v)           { putNumeric(v); }
+    void getDoubleNetwork(double & v) { getNumericNetwork(v); }
+    void getDouble(double& v)         { getNumeric(v); }
+    void putDoubleNetwork(double v)   { putNumericNetwork(v); }
+    void putDouble(double v)          { putNumeric(v); }
+
+ private:
+    void throwOutOfBounds(size_t want, size_t has) __attribute__((noinline,noreturn));
+    uint8_t peekByte() const { return *getBufferAtPos(); }
+
+#if defined(__i386__) || defined(__x86_64__)
+
+    template<typename T>
+    void putDoubleLongNetwork(T val) {
+        //TODO: Change this if we move to big-endian hardware
+        if (__builtin_expect(getRemaining() < (int)sizeof(T), 0)) {
+            throw BufferOutOfBoundsException(getRemaining(), sizeof(T),
+                    VESPA_STRLOC);
+        }
+        unsigned char* data = reinterpret_cast<unsigned char*>(&val);
+        for (int i=sizeof(T)-1; i>=0; --i) {
+            putByte(data[i]);
+        }
+    }
+
+    template<typename T>
+    void getDoubleLongNetwork(T &val) {
+        //TODO: Change this if we move to big-endian hardware
+        if (__builtin_expect(getRemaining() < (int)sizeof(T), 0)) {
+            throw BufferOutOfBoundsException(getRemaining(), sizeof(T),
+                    VESPA_STRLOC);
+        }
+
+        unsigned char* data = reinterpret_cast<unsigned char*>(&val);
+        for (int i=sizeof(T)-1; i>=0; --i) {
+            getByte(data[i]);
+        }
+    }
+#else
+    #error "getDoubleLongNetwork is undefined for this arcitecture"
+#endif
+
+ public:
+    /**
+     * Writes a 62-bit positive integer to the buffer, using 2, 4, or 8 bytes.
+     *
+     * @param number the integer to write
+     */
+    void putInt2_4_8Bytes(int64_t number) {
+        putInt2_4_8Bytes(number, 0);
+    }
+
+    /**
+     * Writes a 62-bit positive integer to the buffer, using 2, 4, or 8 bytes.
+     *
+     * @param number the integer to write
+     * @param len if non-zero, force writing number using len bytes, possibly
+     *            with truncation
+     */
+    void putInt2_4_8Bytes(int64_t number, size_t len);
+
+    /**
+     * Reads a 62-bit positive integer from the buffer, which was written using
+     * 2, 4, or 8 bytes.
+     *
+     * @param v the integer read
+     */
+    void getInt2_4_8Bytes(int64_t & v);
+
+    /**
+     * Computes the size used for storing the given integer using 2, 4 or 8
+     * bytes.
+     *
+     * @param number the integer to check length of
+     * @return the number of bytes used to store it; 2, 4 or 8
+     */
+    static size_t getSerializedSize2_4_8Bytes(int64_t number);
+
+    /**
+     * Writes a 30-bit positive integer to the buffer, using 1, 2, or 4 bytes.
+     *
+     * @param number the integer to write
+     */
+    void putInt1_2_4Bytes(int32_t number);
+
+    /**
+     * Reads a 30-bit positive integer from the buffer, which was written using
+     * 1, 2, or 4 bytes.
+     *
+     * @param v the integer read
+     */
+    void getInt1_2_4Bytes(int32_t & v);
+
+    /**
+     * Computes the size used for storing the given integer using 1, 2 or 4
+     * bytes.
+     *
+     * @param number the integer to check length of
+     * @return the number of bytes used to store it; 1, 2 or 4
+     */
+    static size_t getSerializedSize1_2_4Bytes(int32_t number);
+
+    /**
+     * Writes a 31-bit positive integer to the buffer, using 1 or 4 bytes.
+     *
+     * @param number the integer to write
+     */
+    void putInt1_4Bytes(int32_t number);
+
+    /**
+     * Reads a 31-bit positive integer from the buffer, which was written using
+     * 1 or 4 bytes.
+     *
+     * @param v the integer read
+     */
+    void getInt1_4Bytes(int32_t & v);
+
+    /**
+     * Computes the size used for storing the given integer using 1 or 4 bytes.
+     *
+     * @param number the integer to check length of
+     * @return the number of bytes used to store it; 1 or 4
+     */
+    static size_t getSerializedSize1_4Bytes(int32_t number);
+
+    /**
+     * Writes a 8 bit integer to the buffer at the current position, and
+     * increases the positition accordingly.
+     *
+     * @param  val the int to store
+     * @return True if the value could be stored, false if end of buffer is
+     *         reached
+    */
+    void getChar(char & val) { unsigned char t;getByte(t); val=t; }
+    void putChar(char val)   { putByte(static_cast<unsigned char>(val)); }
+
+    /**
+     *  Reads the given number of bytes into the given pointer, and updates the
+     *  positition accordingly
+     *
+     *   @param	buffer	where to store the bytes
+     *   @param	count	number of bytes to read
+     *   @return	    True if all the bytes could be read, false if end of
+     *                  buffer is reached
+     */
+    void getBytes(void *buffer, size_t count);
+
+    /**
+     * Writes the given number of bytes into the ByteBuffer at the current
+     * position, and updates the positition accordingly
+     *
+     * @param buf the bytes to store
+     * @param count number of bytes to store
+     */
+    void putBytes(const void *buf, size_t count);
+
+    /** Debug */
+    void dump() const;
+
+    class BufferHolder : public vespalib::ReferenceCounter
+    {
+    private:
+        BufferHolder(const BufferHolder &);
+        BufferHolder& operator=(const BufferHolder &);
+
+    public:
+        BufferHolder(vespalib::DefaultAlloc buffer);
+        virtual ~BufferHolder();
+
+        vespalib::DefaultAlloc _buffer;
+    };
+
+    ByteBuffer(BufferHolder* buf, size_t pos, size_t len, size_t limit);
+
+    void set(BufferHolder* buf, size_t pos, size_t len, size_t limit);
+
+private:
+    char * _buffer;
+    size_t _len;
+    size_t _pos;
+    size_t _limit;
+    mutable BufferHolder   * _bufHolder;
+    vespalib::DefaultAlloc _ownedBuffer;
+public:
+
+    std::string toString();
+
+    void swap(ByteBuffer& other);
+
+    void cleanUp();
+};
+
+} // document
+
diff --git a/document/src/vespa/document/util/compressionconfig.h b/document/src/vespa/document/util/compressionconfig.h
new file mode 100644
index 00000000000..f9ba8932be8
--- /dev/null
+++ b/document/src/vespa/document/util/compressionconfig.h
@@ -0,0 +1,85 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <cmath>
+
+namespace document {
+
+
+struct CompressionConfig {
+    enum Type {
+               NONE = 0,
+               HISTORIC_1 = 1,
+               HISTORIC_2 = 2,
+               HISTORIC_3 = 3,
+               HISTORIC_4 = 4,
+               UNCOMPRESSABLE = 5,
+               LZ4 = 6
+    };
+
+    CompressionConfig()
+        : type(NONE), compressionLevel(0), threshold(90), minSize(0) {}
+    CompressionConfig(Type t)
+        : type(t), compressionLevel(9), threshold(90), minSize(0) {}
+
+    CompressionConfig(Type t, uint8_t level, uint8_t minRes)
+        : type(t), compressionLevel(level), threshold(minRes), minSize(0) {}
+
+    CompressionConfig(Type t, uint8_t lvl, uint8_t minRes, size_t minSz)
+        : type(t), compressionLevel(lvl), threshold(minRes), minSize(minSz) {}
+
+    bool operator==(const CompressionConfig& o) const {
+        return (type == o.type
+                && compressionLevel == o.compressionLevel
+                && threshold == o.threshold);
+    }
+    bool operator!=(const CompressionConfig& o) const {
+        return !operator==(o);
+    }
+
+    static Type toType(uint32_t val) {
+        switch (val) {
+        case 1: return HISTORIC_1;
+        case 2: return HISTORIC_2;
+        case 3: return HISTORIC_3;
+        case 4: return HISTORIC_4;
+        case 5: return UNCOMPRESSABLE;
+        case 6: return LZ4;
+        default: return NONE;
+        }
+    }
+    static bool isCompressed(Type type) {
+        return (type != CompressionConfig::NONE &&
+                type != CompressionConfig::UNCOMPRESSABLE);
+    }
+    bool useCompression() const { return isCompressed(type); }
+
+    Type type;
+    uint8_t compressionLevel;
+    uint8_t threshold;
+    size_t minSize;
+};
+
+class CompressionInfo
+{
+public:
+    CompressionInfo(size_t uncompressedSize)
+        : _uncompressedSize(uncompressedSize), _compressedSize(uncompressedSize) { }
+    CompressionInfo(size_t uncompressedSize, size_t compressedSize)
+        : _uncompressedSize(uncompressedSize), _compressedSize(compressedSize) { }
+    size_t getUncompressedSize() const { return _uncompressedSize; }
+    size_t getCompressedSize()   const { return _compressedSize; }
+    double getCompressionRatio() const { return _uncompressedSize/_compressedSize; }
+private:
+    size_t _uncompressedSize;
+    size_t _compressedSize;
+};
+
+inline CompressionInfo operator + (const CompressionInfo & a, const CompressionInfo & b)
+{
+    return CompressionInfo(a.getUncompressedSize() + b.getUncompressedSize(), a.getCompressedSize() + b.getCompressedSize());
+}
+
+}
+
+
diff --git a/document/src/vespa/document/util/compressor.cpp b/document/src/vespa/document/util/compressor.cpp
new file mode 100644
index 00000000000..f11283c7c40
--- /dev/null
+++ b/document/src/vespa/document/util/compressor.cpp
@@ -0,0 +1,161 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".document.compressor");
+#include <vespa/document/util/compressor.h>
+#include <vespa/vespalib/util/memory.h>
+#include <vespa/vespalib/util/linkedptr.h>
+#include <vespa/vespalib/util/sync.h>
+#include <vespa/vespalib/util/stringfmt.h>
+#include <stdexcept>
+#include <lz4.h>
+#include <lz4hc.h>
+
+namespace document
+{
+
+size_t LZ4Compressor::adjustProcessLen(uint16_t, size_t len)   const { return LZ4_compressBound(len); }
+size_t LZ4Compressor::adjustUnProcessLen(uint16_t, size_t len) const { return len; }
+
+bool
+LZ4Compressor::process(const CompressionConfig& config, const void * inputV, size_t inputLen, void * outputV, size_t & outputLenV)
+{
+    const char * input(static_cast<const char *>(inputV));
+    char * output(static_cast<char *>(outputV));
+    int sz(-1);
+    if (config.compressionLevel > 6) {
+        vespalib::DefaultAlloc state(LZ4_sizeofStateHC());
+        sz = LZ4_compressHC2_withStateHC(state.get(), input, output, inputLen, config.compressionLevel);
+    } else {
+        vespalib::DefaultAlloc state(LZ4_sizeofState());
+        sz = LZ4_compress_withState(state.get(), input, output, inputLen);
+    }
+    if (sz != 0) {
+        outputLenV = sz;
+    }
+    assert(sz != 0);
+    return (sz != 0);
+    
+}
+
+bool
+LZ4Compressor::unprocess(const void * inputV, size_t inputLen, void * outputV, size_t & outputLenV)
+{
+    const char * input(static_cast<const char *>(inputV));
+    char * output(static_cast<char *>(outputV));
+    int sz = LZ4_decompress_safe(input, output, inputLen, outputLenV);
+    if (sz > 0) {
+        outputLenV = sz;
+    }
+    assert(sz > 0);
+    return (sz > 0);
+}
+
+CompressionConfig::Type
+compress(ICompressor & compressor, const CompressionConfig & compression, const vespalib::ConstBufferRef & org, vespalib::DataBuffer & dest)
+{
+    CompressionConfig::Type type(CompressionConfig::NONE);
+    dest.ensureFree(compressor.adjustProcessLen(0, org.size()));
+    size_t compressedSize(dest.getFreeLen());
+    if (compressor.process(compression, org.c_str(), org.size(), dest.getFree(), compressedSize)) {
+        if (compressedSize < ((org.size() * compression.threshold)/100)) {
+            dest.moveFreeToData(compressedSize);
+            type = compression.type;
+        }
+    }
+    return type;
+}
+
+CompressionConfig::Type
+docompress(const CompressionConfig & compression, const vespalib::ConstBufferRef & org, vespalib::DataBuffer & dest)
+{
+    CompressionConfig::Type type(CompressionConfig::NONE);
+    switch (compression.type) {
+    case CompressionConfig::LZ4:
+        {
+            LZ4Compressor lz4;
+            type = compress(lz4, compression, org, dest);
+        }
+        break;
+    case CompressionConfig::NONE:
+    default:
+        break;
+    }
+    return type;
+}
+
+CompressionConfig::Type
+compress(const CompressionConfig & compression, const vespalib::ConstBufferRef & org, vespalib::DataBuffer & dest, bool allowSwap)
+{
+    CompressionConfig::Type type(CompressionConfig::NONE);
+    if (org.size() >= compression.minSize) {
+        type = docompress(compression, org, dest);
+    }
+    if (type == CompressionConfig::NONE) {
+        if (allowSwap) {
+            vespalib::DataBuffer tmp(const_cast<char *>(org.c_str()), org.size());
+            tmp.moveFreeToData(org.size());
+            dest.swap(tmp);
+        } else {
+            dest.writeBytes(org.c_str(), org.size());
+        }
+    }
+    return type;
+}
+
+
+void
+decompress(ICompressor & decompressor, size_t uncompressedLen, const vespalib::ConstBufferRef & org, vespalib::DataBuffer & dest, bool allowSwap)
+{
+    dest.ensureFree(uncompressedLen);
+    size_t realUncompressedLen(dest.getFreeLen());
+    if ( ! decompressor.unprocess(org.c_str(), org.size(), dest.getFree(), realUncompressedLen) ) {
+        if ( uncompressedLen < realUncompressedLen) {
+            if (allowSwap) {
+                vespalib::DataBuffer tmp(const_cast<char *>(org.c_str()), org.size());
+                tmp.moveFreeToData(org.size());
+                dest.swap(tmp);
+            } else {
+                dest.writeBytes(org.c_str(), org.size());
+            }
+        } else {
+            throw std::runtime_error(
+                vespalib::make_string("unprocess failed had %" PRIu64
+                                      ", wanted %" PRId64
+                                      ", got %" PRIu64,
+                                      org.size(),
+                                      uncompressedLen,
+                                      realUncompressedLen));
+        }
+    } else {
+        dest.moveFreeToData(realUncompressedLen);
+    }
+}
+
+void
+decompress(const CompressionConfig::Type & type, size_t uncompressedLen, const vespalib::ConstBufferRef & org, vespalib::DataBuffer & dest, bool allowSwap)
+{
+    switch (type) {
+    case CompressionConfig::LZ4:
+        {
+            LZ4Compressor lz4;
+            decompress(lz4, uncompressedLen, org, dest, allowSwap);
+        }
+        break;
+    case CompressionConfig::NONE:
+    case CompressionConfig::UNCOMPRESSABLE:
+        if (allowSwap) {
+            vespalib::DataBuffer tmp(const_cast<char *>(org.c_str()), org.size());
+            tmp.moveFreeToData(org.size());
+            dest.swap(tmp);
+        } else {
+            dest.writeBytes(org.c_str(), org.size());
+        }
+        break;
+    default:
+        throw std::runtime_error(vespalib::make_string("Unable to handle decompression of type '%d'", type));
+        break;
+    }
+}
+
+}
diff --git a/document/src/vespa/document/util/compressor.h b/document/src/vespa/document/util/compressor.h
new file mode 100644
index 00000000000..e4650709b96
--- /dev/null
+++ b/document/src/vespa/document/util/compressor.h
@@ -0,0 +1,56 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/document/util/compressionconfig.h>
+#include <vespa/vespalib/data/databuffer.h>
+#include <vespa/vespalib/util/buffer.h>
+
+namespace document
+{
+
+class ICompressor
+{
+public:
+    virtual ~ICompressor() { }
+    virtual bool process(const CompressionConfig& config, const void * input, size_t inputLen, void * output, size_t & outputLen) = 0;
+    virtual bool unprocess(const void * input, size_t inputLen, void * output, size_t & outputLen) = 0;
+    virtual size_t adjustProcessLen(uint16_t options, size_t len)   const = 0;
+    virtual size_t adjustUnProcessLen(uint16_t options, size_t len) const = 0;
+};
+
+class LZ4Compressor : public ICompressor
+{
+public:
+    virtual bool process(const CompressionConfig& config, const void * input, size_t inputLen, void * output, size_t & outputLen);
+    virtual bool unprocess(const void * input, size_t inputLen, void * output, size_t & outputLen);
+    virtual size_t adjustProcessLen(uint16_t options, size_t len)   const;
+    virtual size_t adjustUnProcessLen(uint16_t options, size_t len) const;
+};
+
+/**
+ * Will try to compress a buffer according to the config. If the criteria can not
+ * be met it will return NONE and dest will get the input buffer.
+ * @param compression is config for how to compress and what criteria to meet.
+ * @param org is the original input buffer.
+ * @param dest is the destination buffer. The compressed data will be appended unless allowSwap is true
+ *             and it is not compressable. Then it will be swapped in.
+ * @param allowSwap will tell it the data must be appended or if it can be swapped in if it is uncompressable or config is NONE.
+ */
+CompressionConfig::Type compress(const CompressionConfig & compression, const vespalib::ConstBufferRef & org, vespalib::DataBuffer & dest, bool allowSwap);
+
+/**
+ * Will try to decompress a buffer according to the config.
+ * be met it will return NONE and dest will get the input buffer.
+ * @param compression is the compression type used for the buffer.
+ * @param uncompressedLen is the length of the uncompressed data.
+ * @param org is the original input buffer.
+ * @param dest is the destination buffer. The decompressed data will be
+ *             appended unless allowSwap is true and compression is NONE.
+ *             Then it will be swapped in.
+ * @param allowSwap will tell it the data must be appended or if it can be swapped in if compression type is NONE.
+ */
+void decompress(const CompressionConfig::Type & compression, size_t uncompressedLen, const vespalib::ConstBufferRef & org, vespalib::DataBuffer & dest, bool allowSwap);
+
+}
+
diff --git a/document/src/vespa/document/util/identifiableid.h b/document/src/vespa/document/util/identifiableid.h
new file mode 100644
index 00000000000..75651281e83
--- /dev/null
+++ b/document/src/vespa/document/util/identifiableid.h
@@ -0,0 +1,70 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/vespalib/objects/identifiable.h>
+
+#define CID_Serializable           DOCUMENT_CID(1)
+#define CID_Deserializable         DOCUMENT_CID(2)
+#define CID_Document               DOCUMENT_CID(3)
+#define CID_DocumentId             DOCUMENT_CID(4)
+#define CID_DocumentUpdate         DOCUMENT_CID(6)
+#define CID_Update                 DOCUMENT_CID(7)
+#define CID_DocumentBase           DOCUMENT_CID(8)
+#define CID_FieldValue             DOCUMENT_CID(9)
+#define CID_ByteFieldValue         DOCUMENT_CID(10)
+#define CID_IntFieldValue          DOCUMENT_CID(11)
+#define CID_LongFieldValue         DOCUMENT_CID(12)
+#define CID_FloatFieldValue        DOCUMENT_CID(13)
+#define CID_DoubleFieldValue       DOCUMENT_CID(14)
+#define CID_StringFieldValue       DOCUMENT_CID(15)
+#define CID_RawFieldValue          DOCUMENT_CID(16)
+//Gone with vespa 6 #define CID_ContentFieldValue      DOCUMENT_CID(17)
+//Long gone #define CID_ContentMetaFieldValue  DOCUMENT_CID(18)
+#define CID_ArrayFieldValue        DOCUMENT_CID(19)
+#define CID_WeightedSetFieldValue  DOCUMENT_CID(20)
+#define CID_FieldMapValue          DOCUMENT_CID(21)
+#define CID_ShortFieldValue        DOCUMENT_CID(22)
+//#define CID_NullObject             DOCUMENT_CID(23)
+#define CID_ValueUpdate            DOCUMENT_CID(24)
+#define CID_AddValueUpdate         DOCUMENT_CID(25)
+#define CID_ArithmeticValueUpdate  DOCUMENT_CID(26)
+#define CID_AssignValueUpdate      DOCUMENT_CID(27)
+#define CID_ClearValueUpdate       DOCUMENT_CID(28)
+#define CID_MapValueUpdate         DOCUMENT_CID(29)
+#define CID_RemoveValueUpdate      DOCUMENT_CID(30)
+#define CID_CollectionFieldValue   DOCUMENT_CID(31)
+#define CID_StructuredFieldValue   DOCUMENT_CID(32)
+#define CID_StructFieldValue       DOCUMENT_CID(33)
+#define CID_LiteralFieldValueB     DOCUMENT_CID(34)
+#define CID_NumericFieldValueBase  DOCUMENT_CID(35)
+#define CID_MapFieldValue          DOCUMENT_CID(36)
+#define CID_PredicateFieldValue    DOCUMENT_CID(37)
+#define CID_TensorFieldValue       DOCUMENT_CID(38)
+
+#define CID_DataType               DOCUMENT_CID(50)
+#define CID_PrimitiveDataType      DOCUMENT_CID(51)
+#define CID_NumericDataType        DOCUMENT_CID(52)
+#define CID_CollectionDataType     DOCUMENT_CID(53)
+#define CID_ArrayDataType          DOCUMENT_CID(54)
+#define CID_WeightedSetDataType    DOCUMENT_CID(55)
+#define CID_StructuredDataType     DOCUMENT_CID(56)
+#define CID_StructDataType         DOCUMENT_CID(57)
+#define CID_DocumentType           DOCUMENT_CID(58)
+
+//#define CID_ExactStringFieldValue  DOCUMENT_CID(60)
+//#define CID_TermBoostFieldValue    DOCUMENT_CID(62)
+//#define CID_TimestampFieldValue    DOCUMENT_CID(63)
+//#define CID_UriFieldValue          DOCUMENT_CID(64)
+#define CID_MapDataType            DOCUMENT_CID(65)
+#define CID_AnnotationReferenceDataType DOCUMENT_CID(66)
+#define CID_TensorDataType         DOCUMENT_CID(67)
+
+#define CID_document_FieldPathEntry DOCUMENT_CID(80)
+
+#define CID_FieldPathUpdate        DOCUMENT_CID(85)
+#define CID_AddFieldPathUpdate     DOCUMENT_CID(86)
+#define CID_AssignFieldPathUpdate  DOCUMENT_CID(87)
+#define CID_RemoveFieldPathUpdate  DOCUMENT_CID(88)
+
+#define CID_document_DocumentUpdate    DOCUMENT_CID(999)
+
diff --git a/document/src/vespa/document/util/printable.cpp b/document/src/vespa/document/util/printable.cpp
new file mode 100644
index 00000000000..aa9334b658e
--- /dev/null
+++ b/document/src/vespa/document/util/printable.cpp
@@ -0,0 +1,14 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/document/util/printable.h>
+
+namespace document {
+
+std::string Printable::toString(bool verbose, const std::string& indent) const
+{
+    std::ostringstream o;
+    print(o, verbose, indent);
+    return o.str();
+}
+
+}
diff --git a/document/src/vespa/document/util/printable.h b/document/src/vespa/document/util/printable.h
new file mode 100644
index 00000000000..9f817e45cc7
--- /dev/null
+++ b/document/src/vespa/document/util/printable.h
@@ -0,0 +1,78 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * \class document::Printable
+ * \ingroup util
+ *
+ * \brief Interfaces for classes with nice debug output operator defined.
+ *
+ * Especially during testing, it is convenient to be able to print out the
+ * contents of a class. Using this interface one need only to implement the
+ * print function to get neat output, and hopefully we can get a more unified
+ * looking output.
+ */
+
+#pragma once
+
+#include <iostream>
+#include <sstream>
+#include <string>
+
+namespace document
+{
+
+class Printable {
+public:
+    virtual ~Printable() {}
+
+    /**
+     * Print instance textual to the given stream.
+     *
+     * This function is expected to NOT add a newline after the last line
+     * printed.
+     *
+     * You should be properly indented before calling this function. The indent
+     * variable tells you what you need to add after each newline to get
+     * indented as far as your first line was. Thus, single line output don't
+     * need to worry about indentation.
+     *
+     * A typical multiline print would thus be something like this:
+     * <pre>
+     *   out << "MyClass() {\n"
+     *       << "\n" << indent << "  some info"
+     *       << "\n" << indent << "}";
+     * </pre>
+     *
+     * @param out The stream to print itself to.
+     * @param verbose Whether to print detailed information or not. For instance
+     *                a list might print it's size and properties if not verbose
+     *                and print each singel element too if verbose.
+     * @param indent This indentation should be printed AFTER each newline
+     *               printed. (Not before output in first line)
+     */
+    virtual void print(std::ostream& out,
+                       bool verbose,
+                       const std::string& indent) const = 0;
+
+    /**
+     * Utility function, since default arguments used in virtual functions is
+     * kinda sketchy.
+     */
+    void print(std::ostream& out) const
+    { print(out, false, ""); }
+    void print(std::ostream& out, bool verbose) const
+    { print(out, verbose, ""); }
+    void print(std::ostream& out, const std::string& indent) const
+    { print(out, false, indent); }
+
+    /** Utility function to get this output as a string.  */
+    std::string toString(bool verbose=false,
+                         const std::string& indent="") const;
+};
+
+inline std::ostream& operator<<(std::ostream& out, const Printable& p) {
+    p.print(out);
+    return out;
+}
+
+} // document
+
diff --git a/document/src/vespa/document/util/queue.h b/document/src/vespa/document/util/queue.h
new file mode 100644
index 00000000000..eee0b0726c6
--- /dev/null
+++ b/document/src/vespa/document/util/queue.h
@@ -0,0 +1,280 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <queue>
+#include <vespa/vespalib/util/sync.h>
+
+#define UNUSED_PARAM(p)
+namespace document
+{
+
+// XXX move to vespalib (or remove)
+/**
+ * semaphore implementation with copy/assignment functionality.
+ **/
+class Semaphore
+{
+private:
+    int _count;
+    int _numWaiters;
+    vespalib::Monitor _sync;
+
+    // assignment would be unsafe
+    Semaphore& operator= (const Semaphore& other);
+public:
+    // XXX is it really safe to just copy other._count here?
+    Semaphore(const Semaphore& other) : _count(other._count), _numWaiters(0), _sync() {}
+
+    Semaphore(int count=0) : _count(count), _numWaiters(0), _sync() { }
+
+    virtual ~Semaphore() {
+        // XXX alternative: assert(_numWaiters == 0)
+        while (true) {
+            vespalib::MonitorGuard guard(_sync);
+            if (_numWaiters == 0) break;
+            _count++;
+            guard.signal();
+        }
+    }
+
+    bool wait(int ms) {
+        bool gotSemaphore = false;
+        vespalib::MonitorGuard guard(_sync);
+        if (_count == 0) {
+            _numWaiters++;
+            // we could retry if we get a signal but not the semaphore,
+            // but then we risk waiting longer than expected, so
+            // just ignore the return value here.
+            guard.wait(ms);
+            _numWaiters--;
+        }
+        if (_count > 0) {
+            _count--;
+            gotSemaphore = true;
+        }
+        assert(_count >= 0);
+        return gotSemaphore;
+    }
+
+    bool wait() {
+        vespalib::MonitorGuard guard(_sync);
+        while (_count == 0) {
+            _numWaiters++;
+            guard.wait();
+            _numWaiters--;
+        }
+        _count--;
+        assert(_count >= 0);
+        return true;
+    }
+
+    void post() {
+        vespalib::MonitorGuard guard(_sync);
+        assert(_count >= 0);
+        _count++;
+        if (_numWaiters > 0) {
+            guard.signal();
+        }
+    }
+};
+
+
+template <typename T, typename Q=std::queue<T> >
+class QueueBase
+{
+public:
+    QueueBase() : _cond(), _count(0), _q()   { }
+    virtual ~QueueBase() { }
+    size_t size()  const { return internal_size(); }
+    bool   empty() const { return size() == 0; }
+protected:
+    vespalib::Monitor   _cond;
+    document::Semaphore _count;
+    Q                   _q;
+
+    bool internal_push(const T& msg) { _q.push(msg); return true; }
+    bool internal_pop(T& msg) {
+        if (_q.empty()) {
+            return false;
+        } else {
+            msg = _q.front();
+            _q.pop();
+            return true;
+	}
+    }
+    size_t internal_size() const { return _q.size(); }
+};
+
+/**
+ * This is a simple queue template that implements a thread safe Q by using
+ * the stl::queue template. Not in any way optimized. Supports simple push and
+ * pop operations together with read of size and empty check.
+ **/
+template <typename T, typename Q=std::queue<T> >
+class Queue : public QueueBase<T, Q>
+{
+public:
+    Queue() : QueueBase<T,Q>() { }
+    bool push(const T& msg, int timeout=-1)
+    {
+        (void)timeout;
+        bool retval;
+        {
+            vespalib::MonitorGuard guard(this->_cond);
+            retval = this->internal_push(msg);
+        }
+        this->_count.post();
+        return retval;
+    }
+    bool pop(T& msg, int timeout=-1)
+    {
+        bool retval((timeout == -1) ?
+                    this->_count.wait() :
+                    this->_count.wait(timeout));
+        if ( retval ) {
+            vespalib::MonitorGuard guard(this->_cond);
+            retval = this->internal_pop(msg);
+        }
+        return retval;
+    }
+};
+
+template <typename T, typename Q=std::queue<T> >
+class QueueWithMax : public QueueBase<T, Q>
+{
+protected:
+    size_t _size;
+    size_t storesize() const { return _size; }
+    virtual void add(const T& UNUSED_PARAM(msg)) { _size++; }
+    virtual void sub(const T& UNUSED_PARAM(msg)) { _size--; }
+private:
+    size_t    _max;
+    size_t    _lowWaterMark;
+    int       _writersWaiting;
+public:
+    QueueWithMax(size_t max_=1000, size_t lowWaterMark_=500)
+      : QueueBase<T, Q>(),
+        _size(0),
+        _max(max_),
+        _lowWaterMark(lowWaterMark_),
+        _writersWaiting(0)
+    { }
+    bool push(const T& msg, int timeout=-1)
+    {
+        bool retval=true;
+        {
+            vespalib::MonitorGuard guard(this->_cond);
+            if (storesize() >= _max) {
+                ++_writersWaiting;
+                if (timeout >= 0) {
+                    retval = guard.wait(timeout);
+                } else {
+                    guard.wait();
+                }
+                --_writersWaiting;
+            }
+            if (retval) {
+                retval = internal_push(msg);
+            }
+	    if (retval) {
+                add(msg);
+	    }
+        }
+        if (retval) {
+            this->_count.post();
+        }
+        return retval;
+    }
+    bool pop(T& msg, int timeout=-1)
+    {
+        bool retval((timeout == -1) ?
+                    this->_count.wait() :
+                    this->_count.wait(timeout));
+        if ( retval ) {
+            vespalib::MonitorGuard guard(this->_cond);
+            retval = internal_pop(msg);
+            if (retval) {
+		sub(msg);
+		if (_writersWaiting > 0 && storesize() < _lowWaterMark) {
+		    guard.signal();
+		}
+            }
+	}
+        return retval;
+    }
+#if 0
+// XXX unused?
+    size_t max() const          { return _max; }
+    size_t lowWaterMark() const { return _lowWaterMark; }
+    void max(size_t v)
+    {
+        vespalib::MonitorGuard guard(this->_cond);
+        _max = v; _lowWaterMark = _max/2;
+    }
+    void lowWaterMark(size_t v)
+    {
+        vespalib::MonitorGuard guard(this->_cond);
+        _lowWaterMark = v;
+    }
+#endif
+};
+
+template <typename T, typename Q=std::queue<T> >
+class QueueWithMaxSerialized : public QueueWithMax<T, Q>
+{
+public:
+    QueueWithMaxSerialized(size_t max_=1000000, size_t lowWaterMark_=500000) : QueueWithMax<T, Q>(max_, lowWaterMark_) { }
+    virtual void add(const T& msg)
+    {
+        if (msg != NULL) {
+            this->_size += msg->getSerializedSize();
+        }
+    }
+    virtual void sub(const T& msg)
+    {
+        if (msg != NULL) {
+            this->_size -= msg->getSerializedSize();
+        }
+    }
+};
+
+#if 0
+
+/**
+  This is an fast Q that reduces lock/unlock to a minimum and ditto with
+  context swithes on notify/wait. enque/deque have no atomic operations
+  unless it is empty/full. This limits the use to situations where there are
+  both single consumers and single producers.
+*/
+template <typename T>
+class QueueSingleProducerConsumer {
+private:
+    typedef std::vector<T> Q;
+public:
+    typedef typename Q::iterator iterator;
+    enum { end=-1 };
+    QueueSingleProducerConsumer(size_t max=1000,
+                                size_t highWaterMark=999,
+                                size_t lowWaterMark=1);
+    T * wait(int timeOut=-1)
+    {
+        T * retval(NULL);
+        if (empty()) {
+            _cond.Wait(timeOut);
+        }
+        return retval;
+    }
+    const T & head() const { return _q[_readPos]; }
+    void removeHead()      { ~_q[_readPos](); _readPos++; }
+    bool deque();
+private:
+    std::vector<T>   _q;
+    size_t           _readPos;
+    size_t           _writePos;
+    FastOS_Condition _cond;
+};
+
+#endif
+
+} // namespace document
+
diff --git a/document/src/vespa/document/util/serializable.cpp b/document/src/vespa/document/util/serializable.cpp
new file mode 100644
index 00000000000..2e9ea321b8f
--- /dev/null
+++ b/document/src/vespa/document/util/serializable.cpp
@@ -0,0 +1,51 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/document/util/serializable.h>
+
+#include <stdio.h>
+#include <vespa/document/util/bytebuffer.h>
+
+using vespalib::DefaultAlloc;
+
+namespace document {
+
+IMPLEMENT_IDENTIFIABLE_ABSTRACT(Serializable, vespalib::Identifiable);
+IMPLEMENT_IDENTIFIABLE_ABSTRACT(Deserializable, Serializable);
+VESPA_IMPLEMENT_EXCEPTION_SPINE(DeserializeException);
+VESPA_IMPLEMENT_EXCEPTION_SPINE(SerializeException);
+
+std::unique_ptr<ByteBuffer> Serializable::serialize() const
+{
+    size_t len = getSerializedSize();
+    std::unique_ptr<ByteBuffer> retVal(new ByteBuffer(len));
+    serialize(*retVal.get());
+    return retVal;
+}
+
+DeserializeException::DeserializeException(const vespalib::string& msg, const vespalib::string& location)
+    : IoException(msg, IoException::CORRUPT_DATA, location, 1)
+{
+}
+
+DeserializeException::DeserializeException(
+        const vespalib::string& msg, const vespalib::Exception& cause,
+        const vespalib::string& location)
+    : IoException(msg, IoException::CORRUPT_DATA, cause, location, 1)
+{
+}
+
+SerializeException::SerializeException(const vespalib::string& msg, const vespalib::string& location)
+    : IoException(msg, IoException::CORRUPT_DATA, location, 1)
+{
+}
+
+SerializeException::SerializeException(
+        const vespalib::string& msg, const vespalib::Exception& cause,
+        const vespalib::string& location)
+    : IoException(msg, IoException::CORRUPT_DATA, cause, location, 1)
+{
+}
+
+
+}
diff --git a/document/src/vespa/document/util/serializable.h b/document/src/vespa/document/util/serializable.h
new file mode 100644
index 00000000000..d8f4d50c196
--- /dev/null
+++ b/document/src/vespa/document/util/serializable.h
@@ -0,0 +1,237 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * @file serializable.h
+ * @ingroup document
+ *
+ * @brief Interfaces to be used for serializing of objects.
+ *
+ * @author Thomas F. Gundersen, H�kon Humberset
+ * @date 2004-03-15
+ * @version $Id$
+ */
+
+#pragma once
+
+#include <iostream>
+#include <map>
+#include <vector>
+
+#include <vespa/vespalib/util/exception.h>
+#include <vespa/vespalib/objects/cloneable.h>
+#include <vespa/vespalib/objects/identifiable.h>
+
+#include <vespa/document/util/bytebuffer.h>
+#include <vespa/document/util/identifiableid.h>
+
+namespace document {
+class DocumentTypeRepo;
+
+class DeserializeException : public vespalib::IoException {
+public:
+    DeserializeException(const vespalib::string& msg, const vespalib::string& location = "");
+    DeserializeException(const vespalib::string& msg, const vespalib::Exception& cause,
+                         const vespalib::string& location = "");
+    VESPA_DEFINE_EXCEPTION_SPINE(DeserializeException)
+};
+
+class SerializeException : public vespalib::IoException {
+public:
+    SerializeException(const vespalib::string& msg, const vespalib::string& location = "");
+    SerializeException(const vespalib::string& msg, const vespalib::Exception& cause,
+                       const vespalib::string& location = "");
+    VESPA_DEFINE_EXCEPTION_SPINE(SerializeException)
+};
+
+class XmlElement;
+
+/**
+ * Base class for classes that can be converted into a bytestream,
+ * normally used later to create a similar instance.
+ */
+
+class Serializable : public vespalib::Identifiable
+{
+protected:
+    virtual void onSerialize(ByteBuffer& buffer) const = 0;
+public:
+    DECLARE_IDENTIFIABLE_ABSTRACT(Serializable);
+
+    virtual ~Serializable() {}
+
+    /**
+     * @return An upper limit to how many bytes serialization of this instance
+     *         need, providing instance is not altered before serialization.
+     */
+    virtual size_t getSerializedSize() const = 0;
+
+    /**
+     * Serializes the instance into the buffer given. Use getSerializedSize()
+     * before calling this method to be sure buffer is big enough.
+     * On success, the given buffers position will be just past the serialized
+     * version of this instance, on failure, position will be reset to whatever
+     * it was prior to calling this function.
+     *
+     * @throw SerializeException If for some reason instance cannot be
+     *                           serialized.
+     * @throw BufferOutOfBoundsException If buffer does not have enough space.
+     */
+    void serialize(ByteBuffer& buffer) const {
+        int pos = buffer.getPos();
+        try{
+            onSerialize(buffer);
+        } catch (...) {
+            buffer.setPos(pos);
+            throw;
+        }
+    }
+
+    /**
+     * Creates a bytebuffer with enough space to serialize this instance
+     * and serialize this instance into it.
+     *
+     * @return The created bytebuffer, positioned after the serialization.
+     *
+     * @throw SerializeException If for some reason instance cannot be
+     *                           serialized.
+     * @throw BufferOutOfBoundsException If buffer does not have enough space.
+     */
+    std::unique_ptr<ByteBuffer> serialize() const;
+};
+
+/**
+ * Base class for instances that can be overwritten from a bytestream,
+ * given that the bytestream is created from a similar instance.
+ */
+class Deserializable : public vespalib::Cloneable, public Serializable
+{
+protected:
+    virtual void onDeserialize(const DocumentTypeRepo &repo,
+                               ByteBuffer& buffer) = 0;
+
+public:
+    DECLARE_IDENTIFIABLE_ABSTRACT(Deserializable);
+    virtual ~Deserializable() {}
+
+    /**
+     * Overwrite this object with the object represented by the given
+     * bytestream. On success, buffer will be positioned after the bytestream
+     * representing the instance we've just deserialized, on failure, bytebuffer
+     * will be pointing to where it was pointing before calling this function.
+     *
+     * @throw DeserializeException If read data doesn't represent a legal object
+     *                             of this type.
+     * @throw BufferOutOfBoundsException If instance wants to read more data
+     *                                   than is available in the buffer.
+     */
+    void deserialize(const DocumentTypeRepo &repo, ByteBuffer& buffer) {
+        int pos = buffer.getPos();
+        try {
+            onDeserialize(repo, buffer);
+        } catch (const DeserializeException &) {
+            buffer.setPos(pos);
+            throw;
+        } catch (const BufferOutOfBoundsException &) {
+            buffer.setPos(pos);
+            throw;
+        }
+    }
+};
+
+/**
+ * If a deserializable needs a version number of the bytestream to deserialize,
+ * and they doesn't include this version number in their own bytestream, they
+ * can be VersionedDeserializable, getting version number externally.
+ *
+ * This is a special case used since document now uses this approach to
+ * deserialize its contents. It is preferable to let each component store its
+ * own version number, unless this has a big impact on the size of what is
+ * serialized.
+ */
+class VersionedDeserializable
+{
+protected:
+    virtual void onDeserialize(ByteBuffer& buffer, uint16_t version) = 0;
+
+public:
+    virtual ~VersionedDeserializable() {}
+
+    /**
+     * Overwrite this object with the object represented by the given
+     * bytestream. On success, buffer will be positioned after the bytestream
+     * representing the instance we've just deserialized, on failure, bytebuffer
+     * will be pointing to where it was pointing before calling this function.
+     *
+     * @throw DeserializeException If read data doesn't represent a legal object
+     *                             of this type.
+     * @throw BufferOutOfBoundsException If instance wants to read more data
+     *                                   than is available in the buffer.
+     */
+    void deserialize(ByteBuffer& buffer, uint16_t version) {
+        int pos = buffer.getPos();
+        try{
+            onDeserialize(buffer, version);
+        } catch (const DeserializeException &) {
+            buffer.setPos(pos);
+            throw;
+        } catch (const BufferOutOfBoundsException &) {
+            buffer.setPos(pos);
+            throw;
+        }
+    }
+};
+
+/*
+
+class XmlElement
+{
+public:
+    typedef std::map<std::string, std::string> AttributeMap;
+    typedef std::vector<XmlElement> ElementList;
+    XmlElement()
+        : _valid(false),
+          _name(),
+          _value(),
+          _attributes(),
+          _elements()
+    {
+    }
+    XmlElement(const std::string & n, const std::string & v)
+        : _valid(true),
+          _name(n),
+          _value(v),
+          _attributes(),
+          _elements()
+    {
+    }
+    const std::string & name()  const { return _name; }
+    const std::string & value() const { return _value; }
+    const AttributeMap & attributes() const { return _attributes; }
+    const ElementList & elements()    const { return _elements; }
+    bool valid() const { return _valid; }
+    std::string attribute(const std::string & name) const;
+    const XmlElement * element(const std::string & name) const;
+private:
+    bool         _valid;
+    std::string  _name;
+    std::string  _value;
+    AttributeMap _attributes;
+    ElementList  _elements;
+    friend std::ostream & operator << (std::ostream & os, const XmlElement & tag);
+    friend std::istream & operator >> (std::istream & is,  XmlElement & tag);
+};
+
+class XmlValue : public std::string
+{
+public:
+    XmlValue() : std::string() { }
+    XmlValue(const char * s) : std::string(s) { }
+    XmlValue(const char * s, size_t sz) : std::string(s, sz) { }
+    XmlValue(const std::string & s) : std::string(s) { }
+private:
+    friend std::ostream & operator << (std::ostream & os, const XmlValue & tag);
+    friend std::istream & operator >> (std::istream & is,  XmlValue & tag);
+};
+*/
+
+}
+
diff --git a/document/src/vespa/document/util/stringutil.cpp b/document/src/vespa/document/util/stringutil.cpp
new file mode 100644
index 00000000000..f32c32ebd50
--- /dev/null
+++ b/document/src/vespa/document/util/stringutil.cpp
@@ -0,0 +1,218 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ *
+ * $Id$
+
+ *
+ * String utilities
+ *
+ */
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/document/util/stringutil.h>
+#include <vespa/vespalib/util/exception.h>
+#include <vespa/vespalib/stllike/asciistream.h>
+
+#include <iomanip>
+#include <sstream>
+#include <vector>
+#include <cassert>
+#include <algorithm>
+
+using vespalib::IllegalArgumentException;
+
+namespace document {
+
+namespace {
+    char toHex(uint32_t val) {
+        return (val < 10 ? '0' + val : 'a' + (val - 10));
+    }
+}
+
+class ReplacementCharacters {
+public:
+    ReplacementCharacters();
+    static int  needEscape(unsigned char c) { return _needEscape[c]; }
+    static char getChar1(unsigned char c)   { return _replacement1[c]; }
+    static char getChar2(unsigned char c)   { return _replacement2[c]; }
+private:
+    static char _needEscape[256];
+    static char _replacement1[256];
+    static char _replacement2[256];
+};
+
+char ReplacementCharacters::_needEscape[256];
+char ReplacementCharacters::_replacement1[256];
+char ReplacementCharacters::_replacement2[256];
+
+ReplacementCharacters::ReplacementCharacters()
+{
+    for(size_t i(0); i < sizeof(_needEscape); i++) {
+        const char c = i;
+        if (c == '"') {
+            _needEscape[i] = 1;
+            _replacement1[i] = '\\';
+            _replacement2[i] = '"';
+        } else if (c == '\\') {
+            _needEscape[i] = 1;
+            _replacement1[i] = '\\';
+            _replacement2[i] = '\\';
+        } else if (c == '\t') {
+            _needEscape[i] = 1;
+            _replacement1[i] = '\\';
+            _replacement2[i] = 't';
+        } else if (c == '\n') {
+            _needEscape[i] = 1;
+            _replacement1[i] = '\\';
+            _replacement2[i] = 'n';
+        } else if (c == '\r') {
+            _needEscape[i] = 1;
+            _replacement1[i] = '\\';
+            _replacement2[i] = 'r';
+        } else if (c == '\f') {
+            _needEscape[i] = 1;
+            _replacement1[i] = '\\';
+            _replacement2[i] = 'f';
+        } else if ((c < 32) || (c > 126)) {
+            _needEscape[i] = 3;
+            _replacement1[i] = toHex((c >> 4) & 0xF);
+            _replacement2[i] = toHex(c & 0xF);
+        } else {
+            _needEscape[i] = 0;
+            _replacement1[i] = c;
+            _replacement2[i] = c;
+        }
+    }
+}
+
+static ReplacementCharacters _G_ForceInitialisation;
+
+const vespalib::string & StringUtil::escape(const vespalib::string & source, vespalib::string & destination,
+                                       char delimiter)
+{
+    size_t escapeCount(0);
+    for(size_t i(0), m(source.size()); i < m; i++) {
+        if (source[i] == delimiter) {
+            escapeCount += 3;
+        } else {
+            escapeCount += ReplacementCharacters::needEscape(source[i]);
+        }
+    }
+    if (escapeCount > 0) {
+        std::vector<char> dst;
+        dst.reserve(source.size() + escapeCount);
+        for(size_t i(0), m(source.size()); i < m; i++) {
+            const char c = source[i];
+            if (c == delimiter) {
+                dst.push_back('\\');
+                dst.push_back('x');
+                dst.push_back(toHex((c >> 4) & 0xF));
+                dst.push_back(toHex(c & 0xF));
+            } else {
+                int needEscape = ReplacementCharacters::needEscape(c);
+                if (needEscape == 0) {
+                    dst.push_back(c);
+                } else {
+                    if (needEscape == 3) {
+                        dst.push_back('\\');
+                        dst.push_back('x');
+                    }
+                    dst.push_back(ReplacementCharacters::getChar1(c));
+                    dst.push_back(ReplacementCharacters::getChar2(c));
+                }
+            }
+        }
+        destination.assign(&dst[0], dst.size());
+        return destination;
+    }
+    return source;
+}
+
+vespalib::string StringUtil::unescape(const vespalib::stringref & source)
+{
+    vespalib::asciistream ost;
+    for (unsigned int i=0; i<source.size(); ++i) {
+        if (source[i] != '\\') { ost << source[i]; continue; }
+        // Here we know we have an escape
+        if (i+1 == source.size()) {
+            throw IllegalArgumentException("Found backslash at end of input",
+                                           VESPA_STRLOC);
+        }
+        if (source[i+1] != 'x') {
+            switch (source[i+1]) {
+            case '\\': ost << '\\'; break;
+            case '"': ost << '"'; break;
+            case 't': ost << '\t'; break;
+            case 'n': ost << '\n'; break;
+            case 'r': ost << '\r'; break;
+            case 'f': ost << '\f'; break;
+            default:
+                throw IllegalArgumentException(
+                        vespalib::make_string("Illegal escape sequence \\%c found", source[i+1]), VESPA_STRLOC);
+            }
+            ++i;
+            continue;
+        }
+        // Only \x## sequences left..
+        if (i+3 >= source.size()) {
+            throw IllegalArgumentException("Found \\x at end of input",
+                                           VESPA_STRLOC);
+        }
+        vespalib::string hexdigits = source.substr(i+2, 2);
+        char* endp(0);
+        ost << static_cast<char>(strtol(hexdigits.c_str(), &endp, 16));
+        if (*endp) {
+            throw IllegalArgumentException("Value "+hexdigits
+                                           + " is not a two digit hexadecimal number", VESPA_STRLOC);
+        }
+        i+=3;
+    }
+    return ost.str();
+}
+
+void StringUtil::
+printAsHex(std::ostream& output, const void* source, unsigned int size,
+           unsigned int columnwidth, bool inlinePrintables,
+           const std::string& indent)
+{
+    assert(columnwidth > 0);
+    unsigned char wildChar = '.';
+    const unsigned char* start = reinterpret_cast<const unsigned char*>(source);
+    uint32_t posWidth = 1;
+    for (uint32_t i=size; i>9; i /= 10) { ++posWidth; }
+    std::vector<unsigned char> printables(columnwidth + 1);
+    printables[columnwidth] = '\0';
+    for (unsigned int i=0; i<size; i += columnwidth) {
+        std::ostringstream ost;
+        if (i != 0) ost << "\n" << indent;
+        ost << std::dec << std::setw(posWidth) << i << ":";
+        bool nonNull = false;
+        for (unsigned int j=0; j<columnwidth; ++j)
+        {
+            if (i+j >= size) {
+                ost << "   ";
+                printables[j] = '\0'; // Avoid adding extra chars.
+            } else {
+                ost << " " << std::setw(2);
+                bool printable = (start[i+j] >= 33 && start[i+j] <= 126);
+                if (inlinePrintables && printable) {
+                    ost << std::setfill(' ') << start[i+j];
+                } else {
+                    ost << std::hex << std::setfill('0')
+                        << ((unsigned int) start[i+j]);
+                    printables[j] = (printable ? start[i+j] : wildChar);
+                }
+                nonNull |= (start[i+j] != 0);
+            }
+        }
+        if (nonNull) {
+            output << ost.str();
+            if (!inlinePrintables) {
+                output << " " << &printables[0];
+            }
+        }
+    }
+}
+
+
+} // namespace document
diff --git a/document/src/vespa/document/util/stringutil.h b/document/src/vespa/document/util/stringutil.h
new file mode 100644
index 00000000000..d5dd400fa9a
--- /dev/null
+++ b/document/src/vespa/document/util/stringutil.h
@@ -0,0 +1,61 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * \class document::StringUtil
+ * \ingroup util
+ *
+ * \brief Utility class for string related functionality.
+ */
+
+#pragma once
+
+#include <vespa/vespalib/stllike/string.h>
+
+namespace document {
+
+class StringUtil {
+public:
+    /**
+     * Escapes a string, turning backslash or unprintable characters into
+     * \\\\ \\n \\t \\f \\r or \\x##.
+     *
+     * The delimiter can be set to also escape an otherwise printable character that you don't
+     * want the string to contain. (Useful to escape content to use in a context where you want
+     * to use a given delimiter)
+     */
+    static vespalib::string escape(const vespalib::string & source, char delimiter = '\0') {
+        vespalib::string escaped;
+        return escape(source, escaped, delimiter);
+    }
+    /**
+    */
+    static const vespalib::string & escape(const vespalib::string & source, vespalib::string & dst,
+                                      char delimiter = '\0');
+
+    /**
+     * Unescape a string, replacing \\\\ \\n \\t \\f \\r or \\x## with their
+     * ascii value counterparts.
+     */
+    static vespalib::string unescape(const vespalib::stringref & source);
+
+    /**
+     * Print whatever source points to in a readable format.
+     *
+     * @param output Stream to print to
+     * @param source Pointer to what should be printed
+     * @param size The number of bytes to print.
+     * @param columnwidth Max number of bytes to print per line.
+     * @param inlinePrintables If true, print printable characters in the list
+     *                         instead of ASCII values. If false, print ASCII
+     *                         values for all bytes, and add printables output
+     *                         only to the right column.
+     * @param indent Whitespace to put after each newline in output
+     */
+    static void printAsHex(std::ostream& output,
+                           const void* source, unsigned int size,
+                           unsigned int columnwidth = 16,
+                           bool inlinePrintables = false,
+                           const std::string& indent = "");
+};
+
+} // document
+
diff --git a/document/src/vespa/document/util/xmlserializable.h b/document/src/vespa/document/util/xmlserializable.h
new file mode 100644
index 00000000000..21e0b858dec
--- /dev/null
+++ b/document/src/vespa/document/util/xmlserializable.h
@@ -0,0 +1,18 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/vespalib/util/xmlserializable.h>
+
+namespace document {
+    typedef vespalib::xml::XmlOutputStream XmlOutputStream;
+    typedef vespalib::xml::XmlSerializable XmlSerializable;
+    typedef vespalib::xml::XmlTag XmlTag;
+    typedef vespalib::xml::XmlEndTag XmlEndTag;
+    typedef vespalib::xml::XmlAttribute XmlAttribute;
+    typedef vespalib::xml::XmlContent XmlContent;
+    typedef vespalib::xml::XmlEscapedContent XmlEscapedContent;
+    typedef vespalib::xml::XmlBase64Content XmlBase64Content;
+    typedef vespalib::xml::XmlContentWrapper XmlContentWrapper;
+}
+
diff --git a/document/testrun/.gitignore b/document/testrun/.gitignore
new file mode 100644
index 00000000000..c6773b6c086
--- /dev/null
+++ b/document/testrun/.gitignore
@@ -0,0 +1,6 @@
+/test-report.html
+/test-report.html.*
+/test.*.*.*
+/tmp.*
+/test.*.*.result
+Makefile