fix whitespace only

10 files changed, 4723 insertions, 4723 deletions

diff --git a/documentapi/src/test/java/com/yahoo/documentapi/VisitorIteratorTestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/VisitorIteratorTestCase.java
index 99094b09a43..bd7e8804133 100755
--- a/documentapi/src/test/java/com/yahoo/documentapi/VisitorIteratorTestCase.java
+++ b/documentapi/src/test/java/com/yahoo/documentapi/VisitorIteratorTestCase.java
@@ -1,1540 +1,1540 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.documentapi;

-

-import com.yahoo.document.select.parser.ParseException;

-import com.yahoo.documentapi.ProgressToken;

-import com.yahoo.documentapi.VisitorIterator;

-import junit.framework.TestCase;

-import com.yahoo.document.BucketId;

-import com.yahoo.document.BucketIdFactory;

-

-import java.util.Arrays;

-import java.util.Map;

-import java.util.Set;

-import java.util.TreeSet;

-import java.util.Vector;

-

-/**

- * Tests for VisitorIterator and ProgressToken (kept in one test case because their

- * interactions are so tightly coupled)

- * @author <a href="mailto:vekterli@yahoo-inc.com">Tor Brede Vekterli</a>

- */

-public class VisitorIteratorTestCase extends TestCase {

-

-    public void testIterationSingleBucketUpdate() throws ParseException {

-        BucketIdFactory idFactory = new BucketIdFactory();

-        ProgressToken progress = new ProgressToken();

-

-        VisitorIterator iter = VisitorIterator.createFromDocumentSelection(

-                "id.user = 1234", idFactory, 1, progress);

-

-        assertFalse(progress.hasActive());

-        assertEquals(progress.getPendingBucketCount(), 1);

-        assertEquals(progress.getFinishedBucketCount(), 0);

-        assertEquals(progress.getTotalBucketCount(), 1);

-        assertFalse(iter.isDone());

-        assertTrue(iter.hasNext());

-        assertEquals(iter.getRemainingBucketCount(), 1);

-        VisitorIterator.BucketProgress b1 = iter.getNext();

-        // Upon first getNext of a superbucket, progress == 0

-        assertEquals(b1.getSuperbucket(), new BucketId(32, 1234));

-        assertEquals(b1.getProgress(), new BucketId());

-        assertFalse(iter.hasNext());

-        assertFalse(iter.isDone());

-        assertEquals(iter.getRemainingBucketCount(), 1);

-        // Should only be one active bucket; the one we just got

-        assertEquals(progress.getActiveBucketCount(), 1);

-        // No pending yet

-        assertFalse(progress.hasPending());

-        // Update the bucket with a sub-bucket, moving it from active to pending

-        BucketId sub = new BucketId(b1.getSuperbucket().getUsedBits() + 1, b1.getSuperbucket().getId());

-        iter.update(b1.getSuperbucket(), sub);

-        assertFalse(progress.hasActive());

-        assertEquals(progress.getPendingBucketCount(), 1);

-        assertTrue(iter.hasNext());

-        assertFalse(iter.isDone());

-        assertEquals(iter.getRemainingBucketCount(), 1);

-        // Get the pending bucket

-        VisitorIterator.BucketProgress b2 = iter.getNext();

-        assertEquals(b2.getSuperbucket(), new BucketId(32, 1234));

-        assertEquals(b2.getProgress(), new BucketId(33, 1234));

-        assertFalse(iter.hasNext());

-        assertEquals(progress.getActiveBucketCount(), 1);

-        assertFalse(progress.hasPending());

-        // Now update with progress==super, signalling that the bucket is done

-        iter.update(b1.getSuperbucket(), ProgressToken.FINISHED_BUCKET);

-        assertFalse(progress.hasActive());

-        assertFalse(progress.hasPending());

-        assertFalse(iter.hasNext());

-        assertTrue(iter.isDone());

-        assertTrue(progress.isFinished());

-        assertEquals(progress.getFinishedBucketCount(), 1);

-        assertEquals(iter.getRemainingBucketCount(), 0);

-    }

-

-    public void testProgressSerializationRange() throws ParseException {

-        int distBits = 4;

-

-        BucketIdFactory idFactory = new BucketIdFactory();

-        ProgressToken progress = new ProgressToken();

-

-        // docsel will be unknown --> entire bucket range will be covered

-        VisitorIterator iter = VisitorIterator.createFromDocumentSelection(

-                "id.group != \"yahoo.com\"", idFactory, distBits, progress);

-

-        assertEquals(progress.getDistributionBitCount(), distBits);

-        assertTrue(iter.getBucketSource() instanceof VisitorIterator.DistributionRangeBucketSource);

-

-        assertEquals(progress.getFinishedBucketCount(), 0);

-        assertEquals(progress.getTotalBucketCount(), 1 << distBits);

-

-        // First, get+update half of the buckets, marking them as done

-        long bucketCount = 0;

-        long bucketStop = 1 << (distBits - 1);

-

-        while (iter.hasNext() && bucketCount != bucketStop) {

-            VisitorIterator.BucketProgress ids = iter.getNext();

-            iter.update(ids.getSuperbucket(), ProgressToken.FINISHED_BUCKET);

-            ++bucketCount;

-        }

-        assertEquals(bucketCount, bucketStop);

-        // Should be no buckets in limbo at this point

-        assertFalse(progress.hasActive());

-        assertFalse(progress.hasPending());

-        assertFalse(iter.isDone());

-        assertTrue(iter.hasNext());

-        assertEquals(progress.getFinishedBucketCount(), bucketCount);

-        assertFalse(progress.isFinished());

-

-        StringBuilder desired = new StringBuilder();

-        desired.append("VDS bucket progress file (50.0% completed)\n");

-        desired.append(distBits);

-        desired.append('\n');

-        desired.append(bucketCount); // Finished == cursor for this

-        desired.append('\n');

-        desired.append(bucketCount);

-        desired.append('\n');

-        desired.append(1 << distBits);

-        desired.append('\n');

-

-        assertEquals(desired.toString(), progress.toString());

-

-        // Test import, in which case distribution bits are 1

-        BucketIdFactory idFactory2 = new BucketIdFactory();

-

-        // De-serialization with no pending buckets

-        {

-            ProgressToken progDs = new ProgressToken(progress.toString());

-

-            assertEquals(progDs.getDistributionBitCount(), distBits);

-            assertEquals(progDs.getTotalBucketCount(), 1 << distBits);

-            assertEquals(progDs.getFinishedBucketCount(), bucketCount);

-

-            VisitorIterator iterDs = VisitorIterator.createFromDocumentSelection(

-                "id.group != \"yahoo.com\"", idFactory2, 1, progDs);

-

-            assertFalse(progDs.hasPending());

-            assertFalse(progDs.hasActive());

-            assertTrue(iterDs.hasNext());

-            assertFalse(iterDs.isDone());

-            assertEquals(distBits, iterDs.getDistributionBitCount());

-            assertEquals(distBits, progDs.getDistributionBitCount());

-

-            // Iterator must start up on next bucket in range

-            VisitorIterator.BucketProgress idDs = iterDs.getNext();

-            long resumeKey = ProgressToken.makeNthBucketKey(bucketCount, distBits);

-            assertEquals(idDs.getSuperbucket(), new BucketId(ProgressToken.keyToBucketId(resumeKey)));

-            assertEquals(idDs.getProgress(), new BucketId());

-        }

-

-        // Now fetch a subset of the remaining buckets without finishing them,

-        // keeping some in the active set and some in pending

-        int pendingTotal = 1 << (distBits - 3);

-        int activeTotal = 1 << (distBits - 3);

-        Vector<VisitorIterator.BucketProgress> buckets = new Vector<VisitorIterator.BucketProgress>();

-

-        // Pre-fetch, since otherwise we'd reuse pending buckets

-        for (int i = 0; i < pendingTotal + activeTotal; ++i) {

-            buckets.add(iter.getNext());

-        }

-

-        for (int i = 0; i < pendingTotal + activeTotal; ++i) {

-            VisitorIterator.BucketProgress idTemp = buckets.get(i);

-            if (i < activeTotal) {

-                // Make them 50% done

-                iter.update(idTemp.getSuperbucket(),

-                        new BucketId(distBits + 2, idTemp.getSuperbucket().getId() | (2 << distBits)));

-            }

-            // else: leave hanging as active

-        }

-

-        assertEquals(progress.getActiveBucketCount(), activeTotal);

-        assertEquals(progress.getPendingBucketCount(), pendingTotal);

-

-        // we can't reuse the existing string builder, since the bucket cursor

-        // has changed

-        desired = new StringBuilder();

-        desired.append("VDS bucket progress file (").append(progress.percentFinished()).append("% completed)\n");

-        desired.append(distBits);

-        desired.append('\n');

-        desired.append(bucketCount + pendingTotal + activeTotal);

-        desired.append('\n');

-        desired.append(bucketCount);

-        desired.append('\n');

-        desired.append(1 << distBits);

-        desired.append('\n');

-

-        assertEquals(progress.getBuckets().entrySet().size(), pendingTotal + activeTotal);

-

-        for (Map.Entry<ProgressToken.BucketKeyWrapper, ProgressToken.BucketEntry> entry

-                : progress.getBuckets().entrySet()) {

-            desired.append(Long.toHexString(ProgressToken.keyToBucketId(entry.getKey().getKey())));

-            desired.append(':');

-            desired.append(Long.toHexString(entry.getValue().getProgress().getRawId()));

-            desired.append('\n');

-        }

-

-        assertEquals(progress.toString(), desired.toString());

-

-        {

-            // Deserialization with pending buckets

-            ProgressToken progDs = new ProgressToken(progress.toString());

-

-            assertEquals(progDs.getDistributionBitCount(), distBits);

-            assertEquals(progDs.getTotalBucketCount(), 1 << distBits);

-            assertEquals(progDs.getFinishedBucketCount(), bucketCount);

-

-            VisitorIterator iterDs = VisitorIterator.createFromDocumentSelection(

-                "id.group != \"yahoo.com\"", idFactory2, 1, progDs);

-

-            // All started but nonfinished buckets get placed in pending upon

-            // deserialization

-            assertEquals(progDs.getPendingBucketCount(), pendingTotal + activeTotal);

-            assertEquals(distBits, progDs.getDistributionBitCount());

-            assertEquals(distBits, iterDs.getDistributionBitCount());

-            assertFalse(progDs.hasActive());

-            assertTrue(iterDs.hasNext());

-            assertFalse(iterDs.isDone());

-            assertEquals(progDs.getBucketCursor(), bucketCount + pendingTotal + activeTotal);

-        }

-

-        // Finish all the active buckets

-        for (int i = activeTotal; i < activeTotal + pendingTotal; ++i) {

-            iter.update(buckets.get(i).getSuperbucket(), ProgressToken.FINISHED_BUCKET);

-            ++bucketCount;

-        }

-

-        assertEquals(progress.getActiveBucketCount(), 0);

-        boolean consistentNext = true;

-        // Get all pending/remaining sourced and finish them all

-        while (!iter.isDone()) {

-            if (!iter.hasNext()) {

-                consistentNext = false;

-                break;

-            }

-            VisitorIterator.BucketProgress bp = iter.getNext();

-            iter.update(bp.getSuperbucket(), ProgressToken.FINISHED_BUCKET);

-            ++bucketCount;

-        }

-

-        assertTrue(consistentNext);

-        assertFalse(iter.hasNext());

-        assertTrue(progress.isFinished());

-        // Cumulative number of finished buckets must match 2^distbits

-        assertEquals(bucketCount, 1 << distBits);

-        StringBuilder finished = new StringBuilder();

-        finished.append("VDS bucket progress file (100.0% completed)\n");

-        finished.append(distBits);

-        finished.append('\n');

-        finished.append(1 << distBits); // Cursor

-        finished.append('\n');

-        finished.append(1 << distBits); // Finished

-        finished.append('\n');

-        finished.append(1 << distBits); // Total

-        finished.append('\n');

-

-        assertEquals(progress.toString(), finished.toString());

-    }

-

-    public void testProgressSerializationExplicit() throws ParseException {

-        int distBits = 16;

-

-        BucketIdFactory idFactory = new BucketIdFactory();

-        ProgressToken progress = new ProgressToken();

-

-        VisitorIterator iter = VisitorIterator.createFromDocumentSelection(

-                "id.user == 1234 or id.user == 6789 or id.user == 8009", idFactory, distBits, progress);

-

-        assertEquals(progress.getDistributionBitCount(), distBits);

-        assertTrue(iter.getBucketSource() instanceof VisitorIterator.ExplicitBucketSource);

-

-        assertEquals(progress.getFinishedBucketCount(), 0);

-        assertEquals(progress.getTotalBucketCount(), 3);

-        assertEquals(progress.getPendingBucketCount(), 3);

-

-        VisitorIterator.BucketProgress bp1 = iter.getNext();

-        VisitorIterator.BucketProgress bp2 = iter.getNext();

-        assertEquals(progress.getPendingBucketCount(), 1);

-        assertEquals(progress.getActiveBucketCount(), 2);

-        // Buckets are ordered by their reverse bucket id key

-        assertEquals(bp1.getSuperbucket(), new BucketId(32, 1234));

-        assertEquals(bp1.getProgress(), new BucketId());

-        // Put bucket 1234 back into pending

-        iter.update(bp1.getSuperbucket(), new BucketId(36, 1234));

-        assertEquals(progress.getPendingBucketCount(), 2);

-

-        assertEquals(bp2.getSuperbucket(), new BucketId(32, 8009));

-        assertEquals(bp2.getProgress(), new BucketId());

-

-        {

-            StringBuilder desired = new StringBuilder();

-            desired.append("VDS bucket progress file (").append(progress.percentFinished()).append("% completed)\n");

-            desired.append(distBits);

-            desired.append('\n');

-            desired.append(0);

-            desired.append('\n');

-            desired.append(0);

-            desired.append('\n');

-            desired.append(3);

-            desired.append('\n');

-            // Pending/active buckets are written in an increasing (key, not

-            // bucket-id!) order

-            desired.append(Long.toHexString(new BucketId(32, 1234).getRawId()));

-            desired.append(':');

-            desired.append(Long.toHexString(new BucketId(36, 1234).getRawId()));

-            desired.append('\n');

-            desired.append(Long.toHexString(new BucketId(32, 8009).getRawId()));

-            desired.append(":0\n");

-            desired.append(Long.toHexString(new BucketId(32, 6789).getRawId()));

-            desired.append(":0\n");

-

-            assertEquals(desired.toString(), progress.toString());

-

-            ProgressToken prog2 = new ProgressToken(progress.toString());

-            assertEquals(prog2.getDistributionBitCount(), distBits);

-            assertEquals(prog2.getTotalBucketCount(), 3);

-            assertEquals(prog2.getFinishedBucketCount(), 0);

-

-            VisitorIterator iter2 = VisitorIterator.createFromDocumentSelection(

-                "id.user == 1234 or id.user == 6789 or id.user == 8009", idFactory, distBits, prog2);

-

-            assertEquals(prog2.getPendingBucketCount(), 3);

-            assertFalse(prog2.hasActive());

-            assertTrue(iter2.hasNext());

-            assertFalse(iter2.isDone());

-

-            assertTrue(iter2.getBucketSource() instanceof VisitorIterator.ExplicitBucketSource);

-            assertFalse(iter2.getBucketSource().hasNext());

-

-            VisitorIterator.BucketProgress bp = iter2.getNext();

-            assertEquals(bp.getSuperbucket(), new BucketId(32, 1234));

-            assertEquals(bp.getProgress(), new BucketId(36, 1234));

-            assertEquals(prog2.getPendingBucketCount(), 2);

-

-            assertTrue(iter2.hasNext());

-            assertFalse(iter2.isDone());

-            bp = iter2.getNext();

-            assertEquals(bp.getSuperbucket(), new BucketId(32, 8009));

-            assertEquals(bp.getProgress(), new BucketId());

-            assertEquals(prog2.getPendingBucketCount(), 1);

-

-            assertTrue(iter2.hasNext());

-            assertFalse(iter2.isDone());

-            bp = iter2.getNext();

-            assertEquals(prog2.getPendingBucketCount(), 0);

-            assertEquals(bp.getSuperbucket(), new BucketId(32, 6789));

-            assertEquals(bp.getProgress(), new BucketId());

-            assertFalse(iter2.hasNext());

-            assertFalse(iter2.isDone()); // Active buckets

-            assertEquals(prog2.getActiveBucketCount(), 3);

-        }

-

-        // Finish off all active buckets

-        assertTrue(iter.hasNext());

-        assertFalse(iter.isDone());

-        bp1 = iter.getNext();

-        assertEquals(bp1.getSuperbucket(), new BucketId(32, 1234));

-        assertEquals(bp1.getProgress(), new BucketId(36, 1234));

-

-        iter.update(bp1.getSuperbucket(), ProgressToken.FINISHED_BUCKET);

-

-        assertTrue(iter.hasNext());

-        assertFalse(iter.isDone());

-        bp1 = iter.getNext();

-        assertEquals(bp1.getSuperbucket(), new BucketId(32, 6789));

-        assertEquals(bp1.getProgress(), new BucketId());

-

-        // Just to make sure Java serializes the long properly

-        assertEquals(

-                progress.toString(),

-                "VDS bucket progress file (" + progress.percentFinished() + "% completed)\n" +

-                "16\n" +

-                "0\n" +

-                "1\n" +

-                "3\n" +

-                "8000000000001f49:0\n" +

-                "8000000000001a85:0\n");

-

-        iter.update(bp1.getSuperbucket(), ProgressToken.FINISHED_BUCKET);

-

-        // At this point, we've got one active but no pending, so hasNext == false,

-        // but isDone is also == false

-        assertFalse(iter.hasNext());

-        assertFalse(iter.isDone());

-        assertEquals(progress.getPendingBucketCount(), 0);

-        assertEquals(progress.getActiveBucketCount(), 1);

-

-        iter.update(bp2.getSuperbucket(), ProgressToken.FINISHED_BUCKET);

-        assertFalse(iter.hasNext());

-        assertTrue(iter.isDone());

-        assertTrue(progress.isFinished());

-        assertEquals(progress.getActiveBucketCount(), 0);

-

-        {

-            StringBuilder finished = new StringBuilder();

-            finished.append("VDS bucket progress file (100.0% completed)\n");

-            finished.append(distBits);

-            finished.append('\n');

-            finished.append(0); // Cursor (not used by explicit)

-            finished.append('\n');

-            finished.append(3); // Finished

-            finished.append('\n');

-            finished.append(3); // Total

-            finished.append('\n');

-

-            assertEquals(finished.toString(), progress.toString());

-        }

-    }

-

-    /**

-     * Test that doing update() on a bucket several times in a row (without re-fetching

-     * from getNext first) works

-     * @throws ParseException

-     */

-    public void testActiveUpdate() throws ParseException {

-        BucketIdFactory idFactory = new BucketIdFactory();

-        ProgressToken progress = new ProgressToken();

-

-        VisitorIterator iter = VisitorIterator.createFromDocumentSelection(

-                "id.group = \"yahoo.com\"", idFactory, 16, progress);

-

-        VisitorIterator.BucketProgress bp = iter.getNext();

-

-        assertEquals(progress.getPendingBucketCount(), 0);

-        assertEquals(progress.getActiveBucketCount(), 1);

-

-        BucketId superbucket = bp.getSuperbucket();

-        int usedBits = superbucket.getUsedBits();

-

-        iter.update(superbucket, new BucketId(usedBits + 2, superbucket.getId() | (2L << usedBits)));

-        assertEquals(progress.getPendingBucketCount(), 1);

-        assertEquals(progress.getActiveBucketCount(), 0);

-        iter.update(superbucket, new BucketId(usedBits + 2, superbucket.getId() | (1L << usedBits)));

-        assertEquals(progress.getPendingBucketCount(), 1);

-        assertEquals(progress.getActiveBucketCount(), 0);

-

-        bp = iter.getNext();

-        assertEquals(bp.getSuperbucket(), superbucket);

-        assertEquals(bp.getProgress(), new BucketId(usedBits + 2, superbucket.getId() | (1L << usedBits)));

-        assertEquals(progress.getPendingBucketCount(), 0);

-        assertEquals(progress.getActiveBucketCount(), 1);

-    }

-

-    /**

-     * Test that ensures doing update(superbucket, 0) simply puts the bucket back in

-     * pending

-     * @throws ParseException

-     */

-    public void testNullAndSuperUpdate() throws ParseException {

-        BucketIdFactory idFactory = new BucketIdFactory();

-        ProgressToken progress = new ProgressToken();

-

-        VisitorIterator iter = VisitorIterator.createFromDocumentSelection(

-                "id.group = \"yahoo.com\"", idFactory, 16, progress);

-

-        assertEquals(progress.getPendingBucketCount(), 1);

-

-        VisitorIterator.BucketProgress bp = iter.getNext();

-        assertEquals(bp.getProgress(), new BucketId());

-        BucketId superbucket = bp.getSuperbucket();

-        BucketId sub = bp.getProgress();

-

-        assertFalse(iter.hasNext());

-        assertFalse(iter.isDone());

-        assertEquals(progress.getPendingBucketCount(), 0);

-        assertEquals(progress.getActiveBucketCount(), 1);

-

-        // 0-bucket

-        iter.update(superbucket, ProgressToken.NULL_BUCKET);

-        assertTrue(iter.hasNext());

-        assertFalse(iter.isDone());

-        assertEquals(progress.getPendingBucketCount(), 1);

-        assertEquals(progress.getActiveBucketCount(), 0);

-

-        VisitorIterator.BucketProgress bp2 = iter.getNext();

-        assertEquals(bp2.getSuperbucket(), superbucket);

-        assertEquals(bp2.getProgress(), ProgressToken.NULL_BUCKET);

-        assertEquals(progress.getPendingBucketCount(), 0);

-        assertEquals(progress.getActiveBucketCount(), 1);

-

-        // progress == super

-        iter.update(superbucket, superbucket);

-        assertTrue(iter.hasNext());

-        assertFalse(iter.isDone());

-        assertEquals(progress.getPendingBucketCount(), 1);

-        assertEquals(progress.getActiveBucketCount(), 0);

-

-        bp2 = iter.getNext();

-        assertEquals(bp2.getSuperbucket(), superbucket);

-        assertEquals(bp2.getProgress(), superbucket);

-        assertEquals(progress.getPendingBucketCount(), 0);

-        assertEquals(progress.getActiveBucketCount(), 1);

-    }

-

-    public void testDeserializedFinishedProgress() {

-        StringBuilder finished = new StringBuilder();

-        finished.append("VDS bucket progress file\n"); // legacy; no completion percentage

-        finished.append(17);

-        finished.append('\n');

-        finished.append(1L << 17); // Cursor

-        finished.append('\n');

-        finished.append(1L << 17); // Finished

-        finished.append('\n');

-        finished.append(1L << 17); // Total

-        finished.append('\n');

-

-        ProgressToken token = new ProgressToken(finished.toString());

-        assertEquals(token.getDistributionBitCount(), 17);

-        assertEquals(token.getTotalBucketCount(), 1L << 17);

-        assertEquals(token.getFinishedBucketCount(), 1L << 17);

-        assertEquals(token.getBucketCursor(), 1L << 17);

-        assertTrue(token.isFinished());

-        

-        ProgressToken token2 = new ProgressToken(token.serialize());

-        assertEquals(17, token2.getDistributionBitCount());

-        assertEquals(1L << 17, token2.getTotalBucketCount());

-        assertEquals(1L << 17, token2.getFinishedBucketCount());

-        assertEquals(1L << 17, token2.getBucketCursor());

-        assertTrue(token2.isFinished());

-    }

-

-    public void testBucketProgressFraction() {

-        double epsilon = 0.00001;

-        // No progress

-        BucketId b_0 = new BucketId();

-        // No split; only superbucket (100%)

-        BucketId b_100_0 = new BucketId(16, 1234);

-        // 1 split (1/2)

-        BucketId b_50_1 = new BucketId(17,  1234);

-        BucketId b_100_1 = new BucketId(17, 1234 | (1 << 16));

-        // 2 splits (1/4)

-        BucketId b_25_2 = new BucketId(18, 1234);

-        BucketId b_50_2 = new BucketId(18, 1234 | (2 << 16));

-        BucketId b_75_2 = new BucketId(18, 1234 | (1 << 16));

-        BucketId b_100_2 = new BucketId(18, 1234 | (3 << 16));

-

-        ProgressToken p = new ProgressToken(16);

-

-        BucketId sb = new BucketId(16, 1234);

-

-        assertEquals(p.progressFraction(new BucketId(32, 1234), b_0), 0.0, epsilon);

-

-        assertEquals(p.progressFraction(sb, b_100_0), 1.0, epsilon);

-

-        assertEquals(p.progressFraction(sb, b_50_1),  0.5, epsilon);

-        assertEquals(p.progressFraction(sb, b_100_1), 1.0, epsilon);

-

-        assertEquals(p.progressFraction(sb, b_25_2),  0.25, epsilon);

-        assertEquals(p.progressFraction(sb, b_50_2),  0.5,  epsilon);

-        assertEquals(p.progressFraction(sb, b_75_2),  0.75, epsilon);

-        assertEquals(p.progressFraction(sb, b_100_2), 1.0,  epsilon);

-

-        assertEquals(p.progressFraction(new BucketId(0x8000000000000000L),

-                new BucketId(0xb0000fff00000000L)), 1.0, epsilon);

-    }

-

-    public void testProgressEstimation() throws ParseException {

-        int distBits = 4;

-

-        BucketIdFactory idFactory = new BucketIdFactory();

-        ProgressToken progress = new ProgressToken();

-

-        // Create a range of [0, 16) superbuckets

-        VisitorIterator iter = VisitorIterator.createFromDocumentSelection(

-                "id.group != \"yahoo.com\"", idFactory, distBits, progress);

-

-        assertEquals(progress.getDistributionBitCount(), 4);

-

-        double epsilon = 0.00001;

-        assertEquals(progress.percentFinished(), 0, epsilon);

-        VisitorIterator.BucketProgress bp = iter.getNext();

-        // Finish first superbucket (6.25% total)

-        iter.update(bp.getSuperbucket(), ProgressToken.FINISHED_BUCKET);

-        assertEquals(progress.percentFinished(), 6.25, epsilon);

-        assertEquals(progress.getFinishedBucketCount(), 1);

-

-        bp = iter.getNext();

-        VisitorIterator.BucketProgress bp3 = iter.getNext();

-        VisitorIterator.BucketProgress bp4 = iter.getNext();

-

-        // Finish second (12.5% total)

-        iter.update(bp.getSuperbucket(), ProgressToken.FINISHED_BUCKET);

-        assertEquals(progress.percentFinished(), 12.5, epsilon);

-        assertEquals(progress.getFinishedBucketCount(), 2);

-

-        // Finish third bucket 75% through (17.1875% total)

-        iter.update(bp3.getSuperbucket(), new BucketId(distBits + 2, bp3.getSuperbucket().getId() | (1 << distBits)));

-        assertEquals(progress.percentFinished(), 17.1875, epsilon);

-        assertEquals(progress.getFinishedBucketCount(), 2);

-

-        // Finish fourth bucket 25% through (18.75% total)

-        iter.update(bp4.getSuperbucket(), new BucketId(distBits + 2, bp4.getSuperbucket().getId()));

-        assertEquals(progress.percentFinished(), 18.75, epsilon);

-        assertEquals(progress.getFinishedBucketCount(), 2);

-        // Finish all buckets

-        iter.update(bp4.getSuperbucket(), ProgressToken.FINISHED_BUCKET);

-        iter.update(bp3.getSuperbucket(), ProgressToken.FINISHED_BUCKET);

-        assertEquals(progress.percentFinished(), 25, epsilon);

-        assertEquals(progress.getFinishedBucketCount(), 4);

-

-        while (iter.hasNext()) {

-            bp = iter.getNext();

-            iter.update(bp.getSuperbucket(), ProgressToken.FINISHED_BUCKET);

-        }

-

-        assertEquals(progress.getFinishedBucketCount(), 16);

-        assertEquals(progress.percentFinished(), 100, epsilon);

-    }

-

-    public void testBucketKeyWrapperOrdering() {

-        ProgressToken.BucketKeyWrapper bk1 = new ProgressToken.BucketKeyWrapper(0x0000000000000001L);

-        ProgressToken.BucketKeyWrapper bk2 = new ProgressToken.BucketKeyWrapper(0x7FFFFFFFFFFFFFFFL);

-        ProgressToken.BucketKeyWrapper bk3 = new ProgressToken.BucketKeyWrapper(0x8000000000000000L);

-        ProgressToken.BucketKeyWrapper bk4 = new ProgressToken.BucketKeyWrapper(0xFFFFFFFFFFFFFFFFL);

-        assertTrue(bk1.compareTo(bk2) < 0);

-        assertTrue(bk2.compareTo(bk3) < 0);

-        assertTrue(bk3.compareTo(bk4) < 0);

-        assertTrue(bk2.compareTo(bk1) > 0);

-        assertTrue(bk3.compareTo(bk2) > 0);

-        assertTrue(bk4.compareTo(bk3) > 0);

-        ProgressToken.BucketKeyWrapper bk5 = new ProgressToken.BucketKeyWrapper(0x7FFFFFFFFFFFFFFFL);

-        ProgressToken.BucketKeyWrapper bk6 = new ProgressToken.BucketKeyWrapper(0x8000000000000000L);

-        assertTrue(bk5.compareTo(bk2) == 0);

-        assertTrue(bk6.compareTo(bk3) == 0);

-    }

-

-    private void doTestBucketKeyGeneration(int db) {

-        // Can't use longs since they won't sort properly when MSB is set

-        ProgressToken.BucketKeyWrapper[] keys = new ProgressToken.BucketKeyWrapper[1 << db];

-

-        // Generate entire bucket space for db

-        for (int i = 0; i < (1 << db); ++i) {

-            keys[i] = new ProgressToken.BucketKeyWrapper(

-                    ProgressToken.bucketToKey(new BucketId(db, i).getId()));

-        }

-        Arrays.sort(keys);

-

-        boolean consistentKeys = true;

-        // Verify that makeNthBucketKey yields the same result as the equivalent

-        // ordered value in the array of keys

-        for (int i = 0; i < (1 << db); ++i) {

-            long genKey = ProgressToken.makeNthBucketKey(i, db);

-            long knownKey = keys[i].getKey();

-            if (genKey != knownKey) {

-                consistentKeys = false;

-                break;

-            }

-        }

-        assertTrue(consistentKeys);

-    }

-

-    public void testBucketKeyGeneration() {

-        // Due to the number of objects needed to be allocated, only test for a

-        // small set of distribution bits

-        for (int i = 1; i < 14; ++i) {

-            doTestBucketKeyGeneration(i);

-        }

-    }

-

-    public void testSingleBucketSplits() throws ParseException {

-        int db = 2;

-        BucketIdFactory idFactory = new BucketIdFactory();

-        ProgressToken p = new ProgressToken();

-

-        // Create a range of [0, 4) superbuckets

-        VisitorIterator iter = VisitorIterator.createFromDocumentSelection(

-                "id.group != \"yahoo.com\"", idFactory, db, p);

-        VisitorIterator.BucketProgress bp = iter.getNext();

-        assertEquals(bp.getSuperbucket(), new BucketId(db, 0));

-        // Put back as pending

-        iter.update(bp.getSuperbucket(), new BucketId());

-        assertEquals(p.getPendingBucketCount(), 1);

-        p.splitPendingBucket(new BucketId(db, 0));

-        assertEquals(p.getPendingBucketCount(), 2);

-        bp = iter.getNext();

-        assertEquals(bp.getSuperbucket(), new BucketId(db + 1, 0)); // left split

-        assertEquals(bp.getProgress(), new BucketId(0));

-        bp = iter.getNext();

-        assertEquals(bp.getSuperbucket(), new BucketId(db + 1, 4)); // right split

-        assertEquals(bp.getProgress(), new BucketId(0));

-

-        bp = iter.getNext();

-        assertEquals(bp.getSuperbucket(), new BucketId(db, 2));

-        // Put back as pending, with a progress of 10010. This implies splitting

-        // the bucket should set both splits with a progress to 10010

-        iter.update(bp.getSuperbucket(), new BucketId(db + 3, 0x12));

-        assertEquals(p.getPendingBucketCount(), 1);

-        p.splitPendingBucket(new BucketId(db, 2));

-        assertEquals(p.getPendingBucketCount(), 2);

-        bp = iter.getNext();

-        assertEquals(bp.getSuperbucket(), new BucketId(db + 1, 2)); // left split

-        assertEquals(bp.getProgress(), new BucketId(db + 3, 0x12));

-        bp = iter.getNext();

-        assertEquals(bp.getSuperbucket(), new BucketId(db + 1, 6)); // right split

-        assertEquals(bp.getProgress(), new BucketId(db + 3, 0x12));

-

-        bp = iter.getNext();

-        // Put back as pending with a progress of 10101. This implies splitting the

-        // bucket should _discard_ left and set right's progress to 10101.

-        // Update: no it shouldn't, we now split with equal progress without

-        // discarding

-        assertEquals(bp.getSuperbucket(), new BucketId(db, 1));

-        iter.update(bp.getSuperbucket(), new BucketId(db + 3, 0x15));

-        assertEquals(p.getPendingBucketCount(), 1);

-        p.splitPendingBucket(new BucketId(db, 1));

-        assertEquals(p.getPendingBucketCount(), 2);

-        bp = iter.getNext();

-        assertEquals(bp.getSuperbucket(), new BucketId(db + 1, 1));

-        assertEquals(bp.getProgress(), new BucketId(db + 3, 0x15));

-        bp = iter.getNext();

-        assertEquals(bp.getSuperbucket(), new BucketId(db + 1, 5)); // right split

-        assertEquals(bp.getProgress(), new BucketId(db + 3, 0x15));

-    }

-

-    /**

-     * Test increasing the distribution bits for a full bucket space range

-     * source with no finished, active or pending buckets

-     * @throws ParseException upon docsel parse failure (shouldn't happen)

-     */

-    public void testRangeDistributionBitIncrease1NoPending() throws ParseException {

-        int db = 2;

-        BucketIdFactory idFactory = new BucketIdFactory();

-        ProgressToken p = new ProgressToken();

-

-        // Test for empty progress token. no splitting involved

-        VisitorIterator iter = VisitorIterator.createFromDocumentSelection(

-                "id.group != \"yahoo.com\"", idFactory, db, p);

-

-        assertEquals(p.getTotalBucketCount(), 4);

-        iter.setDistributionBitCount(db + 1);

-        assertEquals(p.getTotalBucketCount(), 8);

-        assertEquals(p.getDistributionBitCount(), db + 1);

-        assertEquals(iter.getDistributionBitCount(), db + 1);

-        assertEquals(iter.getBucketSource().getDistributionBitCount(), db + 1);

-

-        int[] desired = new int[] { 0, 4, 2, 6, 1, 5, 3, 7 };

-        for (int i = 0; i < 8; ++i) {

-            VisitorIterator.BucketProgress bp = iter.getNext();

-            assertEquals(bp.getSuperbucket(), new BucketId(db + 1, desired[i]));

-        }

-    }

-

-    public void testRangeDistributionBitIncrease1AllBucketStates() throws ParseException {

-        int db = 3;

-        BucketIdFactory idFactory = new BucketIdFactory();

-        ProgressToken p = new ProgressToken();

-

-        VisitorIterator iter = VisitorIterator.createFromDocumentSelection(

-                "id.group != \"yahoo.com\"", idFactory, db, p);

-

-        // For this test, have 1 finished bucket, 3 pending and 0 active (we

-        // want to have the splitting to be triggered immediately)

-        VisitorIterator.BucketProgress bp = iter.getNext();

-        iter.update(bp.getSuperbucket(), ProgressToken.FINISHED_BUCKET);

-        VisitorIterator.BucketProgress[] bpp = new VisitorIterator.BucketProgress[3];

-        bpp[0] = iter.getNext();

-        bpp[1] = iter.getNext();

-        bpp[2] = iter.getNext();

-        iter.update(bpp[0].getSuperbucket(), new BucketId());

-        iter.update(bpp[1].getSuperbucket(), new BucketId());

-        iter.update(bpp[2].getSuperbucket(), new BucketId());

-

-        assertEquals(p.getFinishedBucketCount(), 1);

-        assertEquals(p.getPendingBucketCount(), 3);

-        assertEquals(p.getActiveBucketCount(), 0);

-

-        iter.setDistributionBitCount(db + 1);

-

-        assertEquals(p.getTotalBucketCount(), 16);

-        assertEquals(p.getFinishedBucketCount(), 2);

-        assertEquals(p.getPendingBucketCount(), 6);

-        assertEquals(p.getActiveBucketCount(), 0);

-        assertEquals(p.getDistributionBitCount(), db + 1);

-        assertEquals(iter.getDistributionBitCount(), db + 1);

-        assertEquals(iter.getBucketSource().getDistributionBitCount(), db + 1);

-

-        // Bucket 3:0x4 -> 4:0x4 & 4:0xC

-        assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x04));

-        assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x0C));

-        // Bucket 3:0x2 -> 4:0x2 & 4:0xA

-        assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x02));

-        assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x0A));

-        // Bucket 3:0x6 -> 4:0x6 & 4:0xE

-        assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x06));

-        assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x0E));

-

-        assertEquals(p.getPendingBucketCount(), 0);

-        // Bucket source should now begin returning from bucket 4:0x1

-        assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x01));

-        assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x09));

-        assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x05));

-        assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x0D));

-        assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x03));

-        // Assume correct from here on

-    }

-

-    public void testRangeDistributionIncreaseMultipleBits() throws ParseException {

-        int db = 16;

-        BucketIdFactory idFactory = new BucketIdFactory();

-        ProgressToken p = new ProgressToken();

-

-        VisitorIterator iter = VisitorIterator.createFromDocumentSelection(

-                "id.group != \"yahoo.com\"", idFactory, db, p);

-

-        // For this test, have 3 finished bucket, 2 pending and 1 active

-        for (int i = 0; i < 3; ++i) {

-            iter.update(iter.getNext().getSuperbucket(), ProgressToken.FINISHED_BUCKET);

-        }

-

-        VisitorIterator.BucketProgress[] bpp = new VisitorIterator.BucketProgress[2];

-        bpp[0] = iter.getNext();

-        bpp[1] = iter.getNext();

-        VisitorIterator.BucketProgress bpa = iter.getNext(); // Leave this hanging as active

-        iter.update(bpp[0].getSuperbucket(), new BucketId());

-        iter.update(bpp[1].getSuperbucket(), new BucketId());

-

-        iter.setDistributionBitCount(20);

-        // ProgressToken doesn't change yet, since it had active buckets 

-        assertEquals(p.getDistributionBitCount(), 16);

-        assertEquals(iter.getDistributionBitCount(), 20);

-        assertEquals(iter.getBucketSource().getDistributionBitCount(), 20);

-

-        assertFalse(iter.hasNext());

-        assertFalse(iter.isDone());

-        assertTrue(iter.getBucketSource().shouldYield());

-        assertEquals(p.getPendingBucketCount(), 2);

-        assertEquals(p.getActiveBucketCount(), 1);

-

-        // Finish active, triggering the consistency fixes

-        iter.update(bpa.getSuperbucket(), ProgressToken.FINISHED_BUCKET);

-

-        assertEquals(p.getDistributionBitCount(), 20);

-        assertEquals(p.getPendingBucketCount(), 32);

-        assertEquals(p.getActiveBucketCount(), 0);

-        // Each bucket with db:16 becomes equal to 16 buckets with db:20, so

-        // the bucket space position must be 16 * 6 = 96

-        assertEquals(p.getBucketCursor(), 96);

-        // Each finished bucket also covers less ground, so count is upped

-        // accordingly

-        assertEquals(p.getFinishedBucketCount(), 16 * 4);

-

-        // Remove pending that came from the split

-        // Bucket space that should be covered by the 32 buckets is [48, 80)

-        // when using 20 distribution bits

-        for (int i = 0; i < 32; ++i) {

-            long testKey = ProgressToken.makeNthBucketKey(i + 48, 20);

-            VisitorIterator.BucketProgress bp = iter.getNext();

-            assertEquals(bp.getSuperbucket(), new BucketId(ProgressToken.keyToBucketId(testKey)));

-            iter.update(bp.getSuperbucket(), ProgressToken.FINISHED_BUCKET);

-        }

-        assertEquals(p.getPendingBucketCount(), 0);

-        assertEquals(p.getFinishedBucketCount(), 16 * 6);

-

-        // Bucket source should now begin returning from bucket 20:0x6000

-        assertEquals(iter.getNext().getSuperbucket(), new BucketId(20, 0x6000));

-    }

-

-    public void testSingleBucketMerge() throws ParseException {

-        int db = 2;

-        BucketIdFactory idFactory = new BucketIdFactory();

-        ProgressToken p = new ProgressToken();

-

-        // Create a range of [0, 4) superbuckets

-        VisitorIterator iter = VisitorIterator.createFromDocumentSelection(

-                "id.group != \"yahoo.com\"", idFactory, db, p);

-

-        VisitorIterator.BucketProgress bp = iter.getNext();

-        // Put back as pending and split it

-        iter.update(bp.getSuperbucket(), new BucketId());

-        p.splitPendingBucket(new BucketId(db, 0));

-        assertEquals(p.getPendingBucketCount(), 2);

-        // Merge both back into one node. Merge from left sibling with right present

-        p.mergePendingBucket(new BucketId(db + 1, 0));

-        assertEquals(p.getPendingBucketCount(), 1);

-        bp = iter.getNext();

-        assertEquals(bp.getSuperbucket(), new BucketId(db, 0));

-    }

-

-    public void testRangeDistributionBitDecrease1() throws ParseException {

-        int db = 16;

-        BucketIdFactory idFactory = new BucketIdFactory();

-        ProgressToken p = new ProgressToken();

-

-        VisitorIterator iter = VisitorIterator.createFromDocumentSelection(

-                "id.group != \"yahoo.com\"", idFactory, db, p);

-

-        VisitorIterator.DistributionRangeBucketSource src

-                = (VisitorIterator.DistributionRangeBucketSource)iter.getBucketSource();

-

-        assertTrue(src.isLosslessResetPossible());

-

-        // For this test, have 3 finished buckets, 6 pending and 1 active

-        // This gives a sibling "distribution" of FF FP PP PP PA. When all

-        // active buckets have been updated, 3 merges should be triggered

-        for (int i = 0; i < 3; ++i) {

-            iter.update(iter.getNext().getSuperbucket(), ProgressToken.FINISHED_BUCKET);

-        }

-

-        assertFalse(src.isLosslessResetPossible());

-

-        VisitorIterator.BucketProgress[] bpp = new VisitorIterator.BucketProgress[6];

-        for (int i = 0; i < 6; ++i) {

-            bpp[i] = iter.getNext();

-        }

-        VisitorIterator.BucketProgress bpa = iter.getNext(); // Leave this hanging as active

-        for (int i = 0; i < 6; ++i) {

-            iter.update(bpp[i].getSuperbucket(), new BucketId());

-        }

-

-        assertEquals(p.getBucketCursor(), 10);

-

-        iter.setDistributionBitCount(db - 1);

-        assertEquals(iter.getDistributionBitCount(), db - 1);

-        assertEquals(p.getDistributionBitCount(), db);

-        assertEquals(iter.getBucketSource().getDistributionBitCount(), db - 1);

-        // The iterator is waiting patiently for all active buckets to be updated,

-        // at which point it will performed the merging and actually updating the

-        // progress token's distribution bit count

-        assertTrue(iter.getBucketSource().shouldYield());

-        assertFalse(iter.hasNext());

-        assertFalse(iter.isDone());

-        assertEquals(p.getActiveBucketCount(), 1);

-        iter.update(bpa.getSuperbucket(), new BucketId());

-

-        assertEquals(p.getDistributionBitCount(), db - 1);

-        assertEquals(p.getActiveBucketCount(), 0);

-        assertEquals(p.getPendingBucketCount(), 4); // 3 merges, P PP PP PP -> P P P P

-

-        assertEquals(p.getFinishedBucketCount(), 1);

-        assertEquals(p.getBucketCursor(), 5);

-    }

-

-    // Test that splitting and merging from and to the same db count gives

-    // back the initial state

-    public void testRangeDistributionBitIncreaseDecrease() throws ParseException {

-        int db = 16;

-        BucketIdFactory idFactory = new BucketIdFactory();

-        ProgressToken p = new ProgressToken();

-

-        VisitorIterator iter = VisitorIterator.createFromDocumentSelection(

-                "id.group != \"yahoo.com\"", idFactory, db, p);

-

-        VisitorIterator.DistributionRangeBucketSource src

-                = (VisitorIterator.DistributionRangeBucketSource)iter.getBucketSource();

-

-        assertTrue(src.isLosslessResetPossible());

-

-        // "Sabotage" resetting by having at least 1 finished

-        iter.update(iter.getNext().getSuperbucket(), ProgressToken.FINISHED_BUCKET);

-

-        VisitorIterator.BucketProgress[] bpp = new VisitorIterator.BucketProgress[4];

-        for (int i = 0; i < 4; ++i) {

-            bpp[i] = iter.getNext();

-        }

-        for (int i = 0; i < 4; ++i) {

-            iter.update(bpp[i].getSuperbucket(), new BucketId());

-        }

-

-        assertFalse(src.isLosslessResetPossible());

-

-        iter.setDistributionBitCount(20);

-        assertEquals(p.getDistributionBitCount(), 20);

-        assertEquals(p.getPendingBucketCount(), 4 << 4);

-        assertFalse(iter.getBucketSource().shouldYield());

-        assertEquals(p.getBucketCursor(), 5 << 4);

-

-        iter.setDistributionBitCount(16);

-

-        assertEquals(p.getDistributionBitCount(), 16);

-        assertEquals(p.getPendingBucketCount(), 4);

-        assertFalse(iter.getBucketSource().shouldYield());

-        assertEquals(p.getBucketCursor(), 5);

-    }

-

-    // Test that intermittent changes in distribution are handled properly, e.g.

-    // changing from 11 -> 9 with X active and then before all those are flushed,

-    // the distribution goes up to 12

-    public void testRangeDistributionBitChangeWithoutDone() throws ParseException {

-        int db = 11;

-        BucketIdFactory idFactory = new BucketIdFactory();

-        ProgressToken p = new ProgressToken();

-

-        VisitorIterator iter = VisitorIterator.createFromDocumentSelection(

-                "id.group != \"yahoo.com\"", idFactory, db, p);

-

-        VisitorIterator.DistributionRangeBucketSource src

-                = (VisitorIterator.DistributionRangeBucketSource)iter.getBucketSource();

-

-        VisitorIterator.BucketProgress[] bpp = new VisitorIterator.BucketProgress[4];

-        for (int i = 0; i < 4; ++i) {

-            bpp[i] = iter.getNext();

-        }

-        for (int i = 0; i < 2; ++i) {

-            iter.update(bpp[i].getSuperbucket(), new BucketId());

-        }

-

-        assertFalse(src.isLosslessResetPossible());

-

-        // Now 2 pending, 2 active

-

-        iter.setDistributionBitCount(9);

-        assertEquals(p.getDistributionBitCount(), 11);

-        assertEquals(p.getActiveBucketCount(), 2);

-        assertEquals(p.getPendingBucketCount(), 2);

-        assertTrue(iter.getBucketSource().shouldYield());

-        // Update as pending, still with old count since there's 1 more active

-        // with bpp[2]. Have progress so that lossless reset isn't possible

-        iter.update(bpp[3].getSuperbucket(), new BucketId(15, bpp[3].getSuperbucket().getId()));

-

-        iter.setDistributionBitCount(12);

-        assertEquals(p.getActiveBucketCount(), 1);

-        assertEquals(p.getPendingBucketCount(), 3);

-        assertTrue(iter.getBucketSource().shouldYield());

-

-        // Serialize before token is updated to 12 bits

-        String serialized = p.toString();

-

-        iter.update(bpp[2].getSuperbucket(), ProgressToken.FINISHED_BUCKET);

-

-        assertEquals(p.getActiveBucketCount(), 0);

-        // All active buckets are at db=11, so they should be split once each

-        assertEquals(p.getPendingBucketCount(), 3 * 2);

-        assertFalse(iter.getBucketSource().shouldYield());

-        assertEquals(p.getFinishedBucketCount(), 2);

-

-        // Ensure we get a consistent progress token imported

-        ProgressToken p2 = new ProgressToken(serialized);

-        assertEquals(p2.getDistributionBitCount(), 11); // Not yet updated

-

-        BucketIdFactory idFactory2 = new BucketIdFactory();

-        VisitorIterator iter2 = VisitorIterator.createFromDocumentSelection(

-                "id.group != \"yahoo.com\"", idFactory2, 1, p2);

-

-        // Not yet updated, since we don't trust the initial BucketIdFactory

-        assertEquals(iter2.getDistributionBitCount(), 11);

-        assertEquals(p2.getDistributionBitCount(), 11);

-        iter2.setDistributionBitCount(12);

-        // Now it has been updated

-        assertEquals(p2.getDistributionBitCount(), 12);

-        assertEquals(p2.getPendingBucketCount(), 8);

-        assertEquals(p2.getBucketCursor(), 8);

-        assertEquals(p2.getFinishedBucketCount(), 0);

-    }

-

-    // Test a drop from 31->11 bits upon iteration start

-    public void testRangeDistributionBitInitialDrop() throws ParseException {

-        int db = 31;

-        BucketIdFactory idFactory = new BucketIdFactory();

-        ProgressToken p = new ProgressToken();

-

-        VisitorIterator iter = VisitorIterator.createFromDocumentSelection(

-                "id.group != \"yahoo.com\"", idFactory, db, p);

-

-        VisitorIterator.BucketProgress[] bp = new VisitorIterator.BucketProgress[3];

-        bp[0] = iter.getNext();

-        bp[1] = iter.getNext();

-        bp[2] = iter.getNext();

-        iter.update(bp[2].getSuperbucket(), new BucketId());

-        iter.update(bp[1].getSuperbucket(), new BucketId());

-        assertEquals(p.getActiveBucketCount(), 1);

-

-        iter.setDistributionBitCount(11);

-

-        assertFalse(iter.hasNext());

-        assertFalse(iter.isDone());

-        assertEquals(p.getActiveBucketCount(), 1);

-

-        // Updating the active bucket allows the merging to take place

-        iter.update(new BucketId(31, 0), new BucketId());

-

-        assertTrue(iter.hasNext());

-        assertFalse(iter.isDone());

-

-        // All pending buckets should have been merged down to just 1 now

-        // Update: now rather gets reset

-        assertEquals(p.getPendingBucketCount(), 0);

-        assertEquals(p.getActiveBucketCount(), 0);

-        assertEquals(p.getFinishedBucketCount(), 0);

-        assertEquals(p.getBucketCursor(), 0);

-

-        bp[0] = iter.getNext();

-        assertEquals(bp[0].getSuperbucket(), new BucketId(11, 0));

-    }

-

-    // Similar to testRangeDistributionBitInitialDrop, but going from 1 to 11

-    // This tests that doing so may be done in an optimized way rather than

-    // attempting to split enough buckets to cover the entire bucket space!

-    public void testRangeDistributionLosslessReset() throws ParseException {

-        int db = 1;

-        BucketIdFactory idFactory = new BucketIdFactory();

-        ProgressToken p = new ProgressToken();

-

-        VisitorIterator iter = VisitorIterator.createFromDocumentSelection(

-                "id.group != \"yahoo.com\"", idFactory, db, p);

-

-        VisitorIterator.DistributionRangeBucketSource src

-                = (VisitorIterator.DistributionRangeBucketSource)iter.getBucketSource();

-

-        VisitorIterator.BucketProgress[] bp = new VisitorIterator.BucketProgress[2];

-        bp[0] = iter.getNext();

-        bp[1] = iter.getNext();

-

-        String serialized = p.toString();

-

-        assertFalse(src.isLosslessResetPossible());

-

-        iter.update(bp[1].getSuperbucket(), new BucketId());

-        assertEquals(p.getActiveBucketCount(), 1);

-

-        iter.setDistributionBitCount(11);

-

-        assertFalse(src.isLosslessResetPossible());

-        assertEquals(p.getDistributionBitCount(), 1); // Still at 1

-

-        assertFalse(iter.hasNext());

-        assertFalse(iter.isDone());

-        assertEquals(p.getActiveBucketCount(), 1);

-

-        // Updating the active bucket allows the reset to take place

-        iter.update(new BucketId(1, 0), new BucketId());

-

-        assertTrue(iter.hasNext());

-        assertFalse(iter.isDone());

-

-        // Should not be any buckets pending/active and the cursor should be

-        // back at 0

-        assertEquals(p.getPendingBucketCount(), 0);

-        assertEquals(p.getActiveBucketCount(), 0);

-        assertEquals(p.getFinishedBucketCount(), 0);

-        assertEquals(p.getBucketCursor(), 0);

-        assertEquals(p.getDistributionBitCount(), 11);

-

-        bp[0] = iter.getNext();

-        assertEquals(bp[0].getSuperbucket(), new BucketId(11, 0));

-

-        // Ensure resetting also works when you're importing existing

-        // progress

-        p = new ProgressToken(serialized);

-        idFactory = new BucketIdFactory();

-        iter = VisitorIterator.createFromDocumentSelection(

-                "id.group != \"yahoo.com\"", idFactory, 1, p);

-

-        iter.setDistributionBitCount(11);

-

-        assertEquals(p.getPendingBucketCount(), 0);

-        assertEquals(p.getActiveBucketCount(), 0);

-        assertEquals(p.getFinishedBucketCount(), 0);

-        assertEquals(p.getBucketCursor(), 0);

-        assertEquals(p.getDistributionBitCount(), 11);

-

-        bp[0] = iter.getNext();

-        assertEquals(bp[0].getSuperbucket(), new BucketId(11, 0));

-    }

-

-    public void testExplicitDistributionBitIncrease() throws ParseException {

-        int distBits = 12;

-

-        BucketIdFactory idFactory = new BucketIdFactory();

-        ProgressToken p = new ProgressToken();

-

-        VisitorIterator iter = VisitorIterator.createFromDocumentSelection(

-                "id.user == 1234 or id.user == 6789 or id.user == 8009", idFactory, distBits, p);

-

-        assertEquals(iter.getDistributionBitCount(), distBits);

-        assertEquals(p.getDistributionBitCount(), distBits);

-        assertEquals(iter.getBucketSource().getDistributionBitCount(), distBits);

-

-        iter.update(iter.getNext().getSuperbucket(), ProgressToken.FINISHED_BUCKET);

-        iter.setDistributionBitCount(16);

-

-        assertEquals(iter.getDistributionBitCount(), 16);

-        assertEquals(p.getDistributionBitCount(), 16);

-        assertEquals(iter.getBucketSource().getDistributionBitCount(), 16);

-        // Changing dist bits for explicit source should change nothing

-        assertEquals(p.getPendingBucketCount(), 2);

-        assertEquals(p.getFinishedBucketCount(), 1);

-        assertEquals(p.getTotalBucketCount(), 3);

-    }

-

-    public void testExplicitDistributionBitDecrease() throws ParseException {

-        int distBits = 20;

-

-        BucketIdFactory idFactory = new BucketIdFactory();

-        ProgressToken p = new ProgressToken();

-

-        VisitorIterator iter = VisitorIterator.createFromDocumentSelection(

-                "id.user == 1234 or id.user == 6789 or id.user == 8009", idFactory, distBits, p);

-

-        assertEquals(iter.getDistributionBitCount(), distBits);

-        assertEquals(p.getDistributionBitCount(), distBits);

-        assertEquals(iter.getBucketSource().getDistributionBitCount(), distBits);

-

-        iter.update(iter.getNext().getSuperbucket(), ProgressToken.FINISHED_BUCKET);

-        iter.setDistributionBitCount(16);

-

-        assertEquals(iter.getDistributionBitCount(), 16);

-        assertEquals(p.getDistributionBitCount(), 16);

-        assertEquals(iter.getBucketSource().getDistributionBitCount(), 16);

-        // Changing dist bits for explicit source should change nothing

-        assertEquals(p.getPendingBucketCount(), 2);

-        assertEquals(p.getFinishedBucketCount(), 1);

-        assertEquals(p.getTotalBucketCount(), 3);

-    }

-

-    public void testExplicitDistributionImportNoTruncation() throws ParseException {

-        BucketIdFactory idFactory = new BucketIdFactory();

-        ProgressToken p = new ProgressToken();

-

-        VisitorIterator iter = VisitorIterator.createFromDocumentSelection(

-                "id.user == 1234 or id.user == 6789 or id.user == 8009", idFactory, 20, p);

-        assertEquals(20, iter.getDistributionBitCount());

-        assertEquals(20, p.getDistributionBitCount());

-        assertEquals(20, iter.getBucketSource().getDistributionBitCount());

-

-        iter.update(iter.getNext().getSuperbucket(), ProgressToken.FINISHED_BUCKET);

-

-        // Make sure no truncation is done on import

-        String serialized = p.toString();

-        ProgressToken p2 = new ProgressToken(serialized);

-        BucketIdFactory idFactory2 = new BucketIdFactory();

-        VisitorIterator iter2 = VisitorIterator.createFromDocumentSelection(

-                "id.user == 1234 or id.user == 6789 or id.user == 8009", idFactory2, 1, p2);

-        assertEquals(20, iter2.getDistributionBitCount());

-        assertEquals(20, p2.getDistributionBitCount());

-        assertEquals(20, iter2.getBucketSource().getDistributionBitCount());

-        assertEquals(2, p2.getPendingBucketCount());

-        assertEquals(1, p2.getFinishedBucketCount());

-        assertEquals(3, p2.getTotalBucketCount());

-    }

-

-    public void testImportProgressWithOutdatedDistribution() throws ParseException {

-        String input = "VDS bucket progress file\n" +

-                "10\n" +

-                "503\n" +

-                "500\n" +

-                "1024\n" +

-                "28000000000000be:0\n" +

-                "28000000000002be:0\n" +

-                "28000000000001be:0\n";

-

-        int db = 12;

-        BucketIdFactory idFactory = new BucketIdFactory();

-        ProgressToken p = new ProgressToken(input);

-        assertEquals(10, p.getDistributionBitCount());

-

-        VisitorIterator iter = VisitorIterator.createFromDocumentSelection(

-                "id.group != \"yahoo.com\"", idFactory, 1, p);

-

-        iter.setDistributionBitCount(12);

-        assertEquals(iter.getDistributionBitCount(), 12);

-        assertEquals(p.getDistributionBitCount(), 12);

-        assertEquals(iter.getBucketSource().getDistributionBitCount(), 12);

-

-        assertEquals(p.getTotalBucketCount(), 1 << 12);

-        assertEquals(p.getFinishedBucketCount(), 500 << 2);

-        assertEquals(p.getPendingBucketCount(), 3 << 2);

-        assertEquals(p.getActiveBucketCount(), 0);

-        assertEquals(p.getBucketCursor(), 503 << 2);

-        assertTrue(iter.hasNext());

-        

-        ProgressToken p2 = new ProgressToken(p.serialize());

-        assertEquals(p2.getDistributionBitCount(), 12);

-        assertEquals(p2.getTotalBucketCount(), 1 << 12);

-        assertEquals(p2.getFinishedBucketCount(), 500 << 2);

-        assertEquals(p2.getPendingBucketCount(), 3 << 2);

-        assertEquals(p2.getActiveBucketCount(), 0);

-        assertEquals(p2.getBucketCursor(), 503 << 2);

-    }

-

-    public void testImportInconsistentProgressIncrease() throws ParseException {

-        // Bucket progress "file" that upon time of changing from 4 to 7

-        // distribution bits and writing the progress had an active bucket

-        String input = "VDS bucket progress file\n" +

-                "7\n" +

-                "32\n" +

-                "24\n" +

-                "128\n" +

-                "100000000000000c:0\n";

-        // Now we're at 8 distribution bits

-        int db = 8;

-        BucketIdFactory idFactory = new BucketIdFactory();

-        ProgressToken p = new ProgressToken(input);

-        assertEquals(7, p.getDistributionBitCount());

-        assertEquals(p.getTotalBucketCount(), 1 << 7);

-        assertEquals(p.getFinishedBucketCount(), 24);

-        // Not yet corrected

-        assertEquals(p.getPendingBucketCount(), 1);

-        assertEquals(p.getActiveBucketCount(), 0);

-        assertEquals(32, p.getBucketCursor());

-

-        VisitorIterator iter = VisitorIterator.createFromDocumentSelection(

-                "id.group != \"yahoo.com\"", idFactory, 1, p);

-

-        // Now the range handler should have corrected the progress

-        // (but not messed with the distribution bit count)

-        assertEquals(7, p.getDistributionBitCount());

-        assertEquals(p.getPendingBucketCount(), 1 << 3);

-        assertEquals(p.getActiveBucketCount(), 0);

-        assertEquals(24 + (1 << 3), p.getBucketCursor());

-

-        iter.setDistributionBitCount(8);

-

-        assertEquals(iter.getDistributionBitCount(), 8);

-        assertEquals(p.getDistributionBitCount(), 8);

-        assertEquals(iter.getBucketSource().getDistributionBitCount(), 8);

-

-        assertEquals(p.getTotalBucketCount(), 1 << 8);

-        assertEquals(p.getFinishedBucketCount(), 24 << 1);

-        assertEquals(p.getPendingBucketCount(), 1 << 4); // Split 4 -> 7 bits, then 7 -> 8 bits

-        assertEquals(p.getActiveBucketCount(), 0);

-        assertEquals(p.getBucketCursor(), 24*2 + (1 << 4));

-        assertTrue(iter.hasNext());

-    }

-

-    public void testImportInconsistentProgressDecrease() throws ParseException {

-        // Bucket progress "file" that upon time of changing from 4 to 7

-        // distribution bits and writing the progress had an active bucket

-        String input = "VDS bucket progress file\n" +

-                "7\n" +

-                "32\n" +

-                "24\n" +

-                "128\n" +

-                "100000000000000c:0\n";

-        BucketIdFactory idFactory = new BucketIdFactory();

-        ProgressToken p = new ProgressToken(input);

-

-        VisitorIterator iter = VisitorIterator.createFromDocumentSelection(

-                "id.group != \"yahoo.com\"", idFactory, 1, p);

-

-        assertEquals(iter.getDistributionBitCount(), 7);

-        // Now we're at 6 distribution bits

-        iter.setDistributionBitCount(6);

-

-        assertEquals(iter.getDistributionBitCount(), 6);

-        assertEquals(p.getDistributionBitCount(), 6);

-        assertEquals(iter.getBucketSource().getDistributionBitCount(), 6);

-

-        assertEquals(p.getTotalBucketCount(), 1 << 6);

-        assertEquals(p.getFinishedBucketCount(), 24 >> 1);

-        assertEquals(p.getPendingBucketCount(), 1 << 2); // Split 4 -> 7 bits, merge 7 -> 6 bits

-        assertEquals(p.getActiveBucketCount(), 0);

-        assertEquals(p.getBucketCursor(), 24/2 + (1 << 2));

-        assertTrue(iter.hasNext());

-    }

-

-    public void testEntireBucketSpaceCovered() throws ParseException {

-        int db = 4;

-        BucketIdFactory idFactory = new BucketIdFactory();

-        ProgressToken p = new ProgressToken();

-

-        VisitorIterator iter = VisitorIterator.createFromDocumentSelection(

-                "id.group != \"yahoo.com\"", idFactory, db, p);

-

-        VisitorIterator.BucketProgress[] bpp = new VisitorIterator.BucketProgress[3];

-

-        for (int i = 0; i < 3; ++i) {

-            bpp[i] = iter.getNext();

-        }

-        for (int i = 0; i < 3; ++i) {

-            // Must use non-zero progress or all pending will be optimized

-            // away by the reset-logic

-            iter.update(bpp[i].getSuperbucket(),

-                    new BucketId(db + 1, bpp[i].getSuperbucket().getId()));

-        }

-

-        Set<BucketId> buckets = new TreeSet<BucketId>();

-        db = 7;

-        for (int i = 0; i < (1 << db); ++i) {

-            buckets.add(new BucketId(db, i));

-        }

-

-        iter.setDistributionBitCount(db);

-        assertEquals(p.getFinishedBucketCount(), 0);

-        assertEquals(p.getPendingBucketCount(), 3 << 3);

-

-        // Ensure all buckets are visited once and only once

-        while (iter.hasNext()) {

-            VisitorIterator.BucketProgress bp = iter.getNext();

-            assertTrue(buckets.contains(bp.getSuperbucket()));

-            buckets.remove(bp.getSuperbucket());

-        }

-

-        assertTrue(buckets.isEmpty());

-    }

-

-    public void testExceptionOnWrongDocumentSelection() throws ParseException {

-        BucketIdFactory idFactory = new BucketIdFactory();

-        // Since we don't store the actual original document selection in the

-        // progress files, we can't really tell whether or not a "wrong" document

-        // selection has been given, so we just do a best effort by checking

-        // that the number of total buckets match up and that the bucket cursor

-        // isn't set for explicit sources

-

-        // Try to pass a known document selection to an unknown docsel iterator

-        boolean caughtIt = false;

-        try {

-            ProgressToken p = new ProgressToken("VDS bucket progress file\n16\n3\n1\n3\n"

-                    + "8000000000001f49:0\n8000000000001a85:0\n");

-

-            VisitorIterator.createFromDocumentSelection("id.group != \"yahoo.com\"", idFactory, 16, p);

-        }

-        catch (IllegalArgumentException e) {

-            caughtIt = true;

-        }

-        assertTrue(caughtIt);

-

-        // Now try it the other way around

-        caughtIt = false;

-        try {

-            ProgressToken p = new ProgressToken("VDS bucket progress file\n" +

-                    "10\n" +

-                    "503\n" +

-                    "500\n" +

-                    "1024\n" +

-                    "28000000000000be:0\n" +

-                    "28000000000002be:0\n" +

-                    "28000000000001be:0\n");

-

-            VisitorIterator.createFromDocumentSelection("id.group=\"yahoo.com\" or id.user=555", idFactory, 16, p);

-        }

-        catch (IllegalArgumentException e) {

-            caughtIt = true;

-        }

-        assertTrue(caughtIt);

-    }

-

-    public void testIsBucketFinished() throws ParseException {

-        BucketIdFactory idFactory = new BucketIdFactory();

-        ProgressToken p = new ProgressToken();

-        VisitorIterator iter = VisitorIterator.createFromDocumentSelection(

-                "id.group != \"yahoo.com\"", idFactory, 4, p);

-

-        assertFalse(p.isBucketFinished(new BucketId(32, 0)));

-        // Finish superbucket 0x0000

-        iter.update(iter.getNext().getSuperbucket(), ProgressToken.FINISHED_BUCKET);

-        assertTrue(p.isBucketFinished(new BucketId(32, 0)));

-        // Cursor is 1, but bucket 0x1000 not yet returned

-        assertFalse(p.isBucketFinished(new BucketId(32, 1 << 3)));

-        VisitorIterator.BucketProgress bp = iter.getNext();

-        // Cursor 2, 0x1000 returned but is contained in state, so not finished

-        assertFalse(p.isBucketFinished(new BucketId(32, 1 << 3)));

-        iter.update(bp.getSuperbucket(), ProgressToken.FINISHED_BUCKET);

-        assertTrue(p.isBucketFinished(new BucketId(32, 1 << 3)));

-        // Only superbucket part is used

-        assertTrue(p.isBucketFinished(new BucketId(32, 0x12345670)));  // ...0000

-        assertTrue(p.isBucketFinished(new BucketId(32, 0x12345678)));  // ...1000

-        assertFalse(p.isBucketFinished(new BucketId(32, 0x12345671))); // ...0001

-        assertFalse(p.isBucketFinished(new BucketId(32, 0x12345679))); // ...1001

-    }

-

-    // Test that altering distribution bit count sets ProgressToken as

-    // inconsistent when there are active buckets

-    public void testInconsistentState() throws ParseException {

-        int db = 16;

-        BucketIdFactory idFactory = new BucketIdFactory();

-        ProgressToken p = new ProgressToken();

-

-        VisitorIterator iter = VisitorIterator.createFromDocumentSelection(

-                "id.group != \"yahoo.com\"", idFactory, db, p);

-

-        // For this test, have 3 finished bucket, 2 pending and 1 active

-        for (int i = 0; i < 3; ++i) {

-            iter.update(iter.getNext().getSuperbucket(), ProgressToken.FINISHED_BUCKET);

-        }

-

-        VisitorIterator.BucketProgress[] bpp = new VisitorIterator.BucketProgress[2];

-        bpp[0] = iter.getNext();

-        bpp[1] = iter.getNext();

-        VisitorIterator.BucketProgress bpa = iter.getNext(); // Leave this hanging as active

-        iter.update(bpp[0].getSuperbucket(), new BucketId());

-        iter.update(bpp[1].getSuperbucket(), new BucketId());

-

-        assertFalse(p.isInconsistentState());

-        iter.setDistributionBitCount(20);

-        assertTrue(p.isInconsistentState());

-

-        // Finish active, triggering the consistency fixes

-        iter.update(bpa.getSuperbucket(), ProgressToken.FINISHED_BUCKET);

-

-        assertFalse(p.isInconsistentState());

-    }

-

-    public void testMalformedProgressFile() {

-        boolean caughtIt = false;

-        try {

-            new ProgressToken("VDS bucket progress file\n" +

-                    "10\n" +

-                    "503\n" +

-                    "500\n" +

-                    "1024\n" +

-                    "28000000000000be:0\n" +

-                    "28000000000002be:");

-        } catch (IllegalArgumentException e) {

-            caughtIt = true;

-        }

-        assertTrue(caughtIt);

-    }

-

-    public void testFailOnTooFewLinesInFile() {

-        boolean caughtIt = false;

-        try {

-            new ProgressToken("VDS bucket progress file\n" +

-                    "10\n" +

-                    "503\n");

-        } catch (IllegalArgumentException e) {

-            caughtIt = true;

-        }

-        assertTrue(caughtIt);

-    }

-

-    public void testUnknownFirstHeaderLine() {

-        boolean caughtIt = false;

-        try {

-            new ProgressToken("Smurf Time 3000\n" +

-                    "10\n" +

-                    "503\n" +

-                    "500\n" +

-                    "1024\n" +

-                    "28000000000000be:0\n" +

-                    "28000000000002be:0");

-        } catch (IllegalArgumentException e) {

-            caughtIt = true;

-        }

-        assertTrue(caughtIt);

-    }

-

-    public void testBinaryProgressSerialization() {

-        String input = "VDS bucket progress file (48.828125% completed)\n" +

-                "10\n" +

-                "503\n" +

-                "500\n" +

-                "1024\n" +

-                "28000000000000be:0\n" +

-                "28000000000002be:0\n" +

-                "28000000000001be:0\n";

-        ProgressToken p = new ProgressToken(input);

-        byte[] buf = p.serialize();

-        ProgressToken p2 = new ProgressToken(buf);

-        assertEquals(input, p2.toString());

-    }

- }

+package com.yahoo.documentapi;
+
+import com.yahoo.document.select.parser.ParseException;
+import com.yahoo.documentapi.ProgressToken;
+import com.yahoo.documentapi.VisitorIterator;
+import junit.framework.TestCase;
+import com.yahoo.document.BucketId;
+import com.yahoo.document.BucketIdFactory;
+
+import java.util.Arrays;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.Vector;
+
+/**
+ * Tests for VisitorIterator and ProgressToken (kept in one test case because their
+ * interactions are so tightly coupled)
+ * @author <a href="mailto:vekterli@yahoo-inc.com">Tor Brede Vekterli</a>
+ */
+public class VisitorIteratorTestCase extends TestCase {
+
+    public void testIterationSingleBucketUpdate() throws ParseException {
+        BucketIdFactory idFactory = new BucketIdFactory();
+        ProgressToken progress = new ProgressToken();
+
+        VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
+                "id.user = 1234", idFactory, 1, progress);
+
+        assertFalse(progress.hasActive());
+        assertEquals(progress.getPendingBucketCount(), 1);
+        assertEquals(progress.getFinishedBucketCount(), 0);
+        assertEquals(progress.getTotalBucketCount(), 1);
+        assertFalse(iter.isDone());
+        assertTrue(iter.hasNext());
+        assertEquals(iter.getRemainingBucketCount(), 1);
+        VisitorIterator.BucketProgress b1 = iter.getNext();
+        // Upon first getNext of a superbucket, progress == 0
+        assertEquals(b1.getSuperbucket(), new BucketId(32, 1234));
+        assertEquals(b1.getProgress(), new BucketId());
+        assertFalse(iter.hasNext());
+        assertFalse(iter.isDone());
+        assertEquals(iter.getRemainingBucketCount(), 1);
+        // Should only be one active bucket; the one we just got
+        assertEquals(progress.getActiveBucketCount(), 1);
+        // No pending yet
+        assertFalse(progress.hasPending());
+        // Update the bucket with a sub-bucket, moving it from active to pending
+        BucketId sub = new BucketId(b1.getSuperbucket().getUsedBits() + 1, b1.getSuperbucket().getId());
+        iter.update(b1.getSuperbucket(), sub);
+        assertFalse(progress.hasActive());
+        assertEquals(progress.getPendingBucketCount(), 1);
+        assertTrue(iter.hasNext());
+        assertFalse(iter.isDone());
+        assertEquals(iter.getRemainingBucketCount(), 1);
+        // Get the pending bucket
+        VisitorIterator.BucketProgress b2 = iter.getNext();
+        assertEquals(b2.getSuperbucket(), new BucketId(32, 1234));
+        assertEquals(b2.getProgress(), new BucketId(33, 1234));
+        assertFalse(iter.hasNext());
+        assertEquals(progress.getActiveBucketCount(), 1);
+        assertFalse(progress.hasPending());
+        // Now update with progress==super, signalling that the bucket is done
+        iter.update(b1.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
+        assertFalse(progress.hasActive());
+        assertFalse(progress.hasPending());
+        assertFalse(iter.hasNext());
+        assertTrue(iter.isDone());
+        assertTrue(progress.isFinished());
+        assertEquals(progress.getFinishedBucketCount(), 1);
+        assertEquals(iter.getRemainingBucketCount(), 0);
+    }
+
+    public void testProgressSerializationRange() throws ParseException {
+        int distBits = 4;
+
+        BucketIdFactory idFactory = new BucketIdFactory();
+        ProgressToken progress = new ProgressToken();
+
+        // docsel will be unknown --> entire bucket range will be covered
+        VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
+                "id.group != \"yahoo.com\"", idFactory, distBits, progress);
+
+        assertEquals(progress.getDistributionBitCount(), distBits);
+        assertTrue(iter.getBucketSource() instanceof VisitorIterator.DistributionRangeBucketSource);
+
+        assertEquals(progress.getFinishedBucketCount(), 0);
+        assertEquals(progress.getTotalBucketCount(), 1 << distBits);
+
+        // First, get+update half of the buckets, marking them as done
+        long bucketCount = 0;
+        long bucketStop = 1 << (distBits - 1);
+
+        while (iter.hasNext() && bucketCount != bucketStop) {
+            VisitorIterator.BucketProgress ids = iter.getNext();
+            iter.update(ids.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
+            ++bucketCount;
+        }
+        assertEquals(bucketCount, bucketStop);
+        // Should be no buckets in limbo at this point
+        assertFalse(progress.hasActive());
+        assertFalse(progress.hasPending());
+        assertFalse(iter.isDone());
+        assertTrue(iter.hasNext());
+        assertEquals(progress.getFinishedBucketCount(), bucketCount);
+        assertFalse(progress.isFinished());
+
+        StringBuilder desired = new StringBuilder();
+        desired.append("VDS bucket progress file (50.0% completed)\n");
+        desired.append(distBits);
+        desired.append('\n');
+        desired.append(bucketCount); // Finished == cursor for this
+        desired.append('\n');
+        desired.append(bucketCount);
+        desired.append('\n');
+        desired.append(1 << distBits);
+        desired.append('\n');
+
+        assertEquals(desired.toString(), progress.toString());
+
+        // Test import, in which case distribution bits are 1
+        BucketIdFactory idFactory2 = new BucketIdFactory();
+
+        // De-serialization with no pending buckets
+        {
+            ProgressToken progDs = new ProgressToken(progress.toString());
+
+            assertEquals(progDs.getDistributionBitCount(), distBits);
+            assertEquals(progDs.getTotalBucketCount(), 1 << distBits);
+            assertEquals(progDs.getFinishedBucketCount(), bucketCount);
+
+            VisitorIterator iterDs = VisitorIterator.createFromDocumentSelection(
+                "id.group != \"yahoo.com\"", idFactory2, 1, progDs);
+
+            assertFalse(progDs.hasPending());
+            assertFalse(progDs.hasActive());
+            assertTrue(iterDs.hasNext());
+            assertFalse(iterDs.isDone());
+            assertEquals(distBits, iterDs.getDistributionBitCount());
+            assertEquals(distBits, progDs.getDistributionBitCount());
+
+            // Iterator must start up on next bucket in range
+            VisitorIterator.BucketProgress idDs = iterDs.getNext();
+            long resumeKey = ProgressToken.makeNthBucketKey(bucketCount, distBits);
+            assertEquals(idDs.getSuperbucket(), new BucketId(ProgressToken.keyToBucketId(resumeKey)));
+            assertEquals(idDs.getProgress(), new BucketId());
+        }
+
+        // Now fetch a subset of the remaining buckets without finishing them,
+        // keeping some in the active set and some in pending
+        int pendingTotal = 1 << (distBits - 3);
+        int activeTotal = 1 << (distBits - 3);
+        Vector<VisitorIterator.BucketProgress> buckets = new Vector<VisitorIterator.BucketProgress>();
+
+        // Pre-fetch, since otherwise we'd reuse pending buckets
+        for (int i = 0; i < pendingTotal + activeTotal; ++i) {
+            buckets.add(iter.getNext());
+        }
+
+        for (int i = 0; i < pendingTotal + activeTotal; ++i) {
+            VisitorIterator.BucketProgress idTemp = buckets.get(i);
+            if (i < activeTotal) {
+                // Make them 50% done
+                iter.update(idTemp.getSuperbucket(),
+                        new BucketId(distBits + 2, idTemp.getSuperbucket().getId() | (2 << distBits)));
+            }
+            // else: leave hanging as active
+        }
+
+        assertEquals(progress.getActiveBucketCount(), activeTotal);
+        assertEquals(progress.getPendingBucketCount(), pendingTotal);
+
+        // we can't reuse the existing string builder, since the bucket cursor
+        // has changed
+        desired = new StringBuilder();
+        desired.append("VDS bucket progress file (").append(progress.percentFinished()).append("% completed)\n");
+        desired.append(distBits);
+        desired.append('\n');
+        desired.append(bucketCount + pendingTotal + activeTotal);
+        desired.append('\n');
+        desired.append(bucketCount);
+        desired.append('\n');
+        desired.append(1 << distBits);
+        desired.append('\n');
+
+        assertEquals(progress.getBuckets().entrySet().size(), pendingTotal + activeTotal);
+
+        for (Map.Entry<ProgressToken.BucketKeyWrapper, ProgressToken.BucketEntry> entry
+                : progress.getBuckets().entrySet()) {
+            desired.append(Long.toHexString(ProgressToken.keyToBucketId(entry.getKey().getKey())));
+            desired.append(':');
+            desired.append(Long.toHexString(entry.getValue().getProgress().getRawId()));
+            desired.append('\n');
+        }
+
+        assertEquals(progress.toString(), desired.toString());
+
+        {
+            // Deserialization with pending buckets
+            ProgressToken progDs = new ProgressToken(progress.toString());
+
+            assertEquals(progDs.getDistributionBitCount(), distBits);
+            assertEquals(progDs.getTotalBucketCount(), 1 << distBits);
+            assertEquals(progDs.getFinishedBucketCount(), bucketCount);
+
+            VisitorIterator iterDs = VisitorIterator.createFromDocumentSelection(
+                "id.group != \"yahoo.com\"", idFactory2, 1, progDs);
+
+            // All started but nonfinished buckets get placed in pending upon
+            // deserialization
+            assertEquals(progDs.getPendingBucketCount(), pendingTotal + activeTotal);
+            assertEquals(distBits, progDs.getDistributionBitCount());
+            assertEquals(distBits, iterDs.getDistributionBitCount());
+            assertFalse(progDs.hasActive());
+            assertTrue(iterDs.hasNext());
+            assertFalse(iterDs.isDone());
+            assertEquals(progDs.getBucketCursor(), bucketCount + pendingTotal + activeTotal);
+        }
+
+        // Finish all the active buckets
+        for (int i = activeTotal; i < activeTotal + pendingTotal; ++i) {
+            iter.update(buckets.get(i).getSuperbucket(), ProgressToken.FINISHED_BUCKET);
+            ++bucketCount;
+        }
+
+        assertEquals(progress.getActiveBucketCount(), 0);
+        boolean consistentNext = true;
+        // Get all pending/remaining sourced and finish them all
+        while (!iter.isDone()) {
+            if (!iter.hasNext()) {
+                consistentNext = false;
+                break;
+            }
+            VisitorIterator.BucketProgress bp = iter.getNext();
+            iter.update(bp.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
+            ++bucketCount;
+        }
+
+        assertTrue(consistentNext);
+        assertFalse(iter.hasNext());
+        assertTrue(progress.isFinished());
+        // Cumulative number of finished buckets must match 2^distbits
+        assertEquals(bucketCount, 1 << distBits);
+        StringBuilder finished = new StringBuilder();
+        finished.append("VDS bucket progress file (100.0% completed)\n");
+        finished.append(distBits);
+        finished.append('\n');
+        finished.append(1 << distBits); // Cursor
+        finished.append('\n');
+        finished.append(1 << distBits); // Finished
+        finished.append('\n');
+        finished.append(1 << distBits); // Total
+        finished.append('\n');
+
+        assertEquals(progress.toString(), finished.toString());
+    }
+
+    public void testProgressSerializationExplicit() throws ParseException {
+        int distBits = 16;
+
+        BucketIdFactory idFactory = new BucketIdFactory();
+        ProgressToken progress = new ProgressToken();
+
+        VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
+                "id.user == 1234 or id.user == 6789 or id.user == 8009", idFactory, distBits, progress);
+
+        assertEquals(progress.getDistributionBitCount(), distBits);
+        assertTrue(iter.getBucketSource() instanceof VisitorIterator.ExplicitBucketSource);
+
+        assertEquals(progress.getFinishedBucketCount(), 0);
+        assertEquals(progress.getTotalBucketCount(), 3);
+        assertEquals(progress.getPendingBucketCount(), 3);
+
+        VisitorIterator.BucketProgress bp1 = iter.getNext();
+        VisitorIterator.BucketProgress bp2 = iter.getNext();
+        assertEquals(progress.getPendingBucketCount(), 1);
+        assertEquals(progress.getActiveBucketCount(), 2);
+        // Buckets are ordered by their reverse bucket id key
+        assertEquals(bp1.getSuperbucket(), new BucketId(32, 1234));
+        assertEquals(bp1.getProgress(), new BucketId());
+        // Put bucket 1234 back into pending
+        iter.update(bp1.getSuperbucket(), new BucketId(36, 1234));
+        assertEquals(progress.getPendingBucketCount(), 2);
+
+        assertEquals(bp2.getSuperbucket(), new BucketId(32, 8009));
+        assertEquals(bp2.getProgress(), new BucketId());
+
+        {
+            StringBuilder desired = new StringBuilder();
+            desired.append("VDS bucket progress file (").append(progress.percentFinished()).append("% completed)\n");
+            desired.append(distBits);
+            desired.append('\n');
+            desired.append(0);
+            desired.append('\n');
+            desired.append(0);
+            desired.append('\n');
+            desired.append(3);
+            desired.append('\n');
+            // Pending/active buckets are written in an increasing (key, not
+            // bucket-id!) order
+            desired.append(Long.toHexString(new BucketId(32, 1234).getRawId()));
+            desired.append(':');
+            desired.append(Long.toHexString(new BucketId(36, 1234).getRawId()));
+            desired.append('\n');
+            desired.append(Long.toHexString(new BucketId(32, 8009).getRawId()));
+            desired.append(":0\n");
+            desired.append(Long.toHexString(new BucketId(32, 6789).getRawId()));
+            desired.append(":0\n");
+
+            assertEquals(desired.toString(), progress.toString());
+
+            ProgressToken prog2 = new ProgressToken(progress.toString());
+            assertEquals(prog2.getDistributionBitCount(), distBits);
+            assertEquals(prog2.getTotalBucketCount(), 3);
+            assertEquals(prog2.getFinishedBucketCount(), 0);
+
+            VisitorIterator iter2 = VisitorIterator.createFromDocumentSelection(
+                "id.user == 1234 or id.user == 6789 or id.user == 8009", idFactory, distBits, prog2);
+
+            assertEquals(prog2.getPendingBucketCount(), 3);
+            assertFalse(prog2.hasActive());
+            assertTrue(iter2.hasNext());
+            assertFalse(iter2.isDone());
+
+            assertTrue(iter2.getBucketSource() instanceof VisitorIterator.ExplicitBucketSource);
+            assertFalse(iter2.getBucketSource().hasNext());
+
+            VisitorIterator.BucketProgress bp = iter2.getNext();
+            assertEquals(bp.getSuperbucket(), new BucketId(32, 1234));
+            assertEquals(bp.getProgress(), new BucketId(36, 1234));
+            assertEquals(prog2.getPendingBucketCount(), 2);
+
+            assertTrue(iter2.hasNext());
+            assertFalse(iter2.isDone());
+            bp = iter2.getNext();
+            assertEquals(bp.getSuperbucket(), new BucketId(32, 8009));
+            assertEquals(bp.getProgress(), new BucketId());
+            assertEquals(prog2.getPendingBucketCount(), 1);
+
+            assertTrue(iter2.hasNext());
+            assertFalse(iter2.isDone());
+            bp = iter2.getNext();
+            assertEquals(prog2.getPendingBucketCount(), 0);
+            assertEquals(bp.getSuperbucket(), new BucketId(32, 6789));
+            assertEquals(bp.getProgress(), new BucketId());
+            assertFalse(iter2.hasNext());
+            assertFalse(iter2.isDone()); // Active buckets
+            assertEquals(prog2.getActiveBucketCount(), 3);
+        }
+
+        // Finish off all active buckets
+        assertTrue(iter.hasNext());
+        assertFalse(iter.isDone());
+        bp1 = iter.getNext();
+        assertEquals(bp1.getSuperbucket(), new BucketId(32, 1234));
+        assertEquals(bp1.getProgress(), new BucketId(36, 1234));
+
+        iter.update(bp1.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
+
+        assertTrue(iter.hasNext());
+        assertFalse(iter.isDone());
+        bp1 = iter.getNext();
+        assertEquals(bp1.getSuperbucket(), new BucketId(32, 6789));
+        assertEquals(bp1.getProgress(), new BucketId());
+
+        // Just to make sure Java serializes the long properly
+        assertEquals(
+                progress.toString(),
+                "VDS bucket progress file (" + progress.percentFinished() + "% completed)\n" +
+                "16\n" +
+                "0\n" +
+                "1\n" +
+                "3\n" +
+                "8000000000001f49:0\n" +
+                "8000000000001a85:0\n");
+
+        iter.update(bp1.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
+
+        // At this point, we've got one active but no pending, so hasNext == false,
+        // but isDone is also == false
+        assertFalse(iter.hasNext());
+        assertFalse(iter.isDone());
+        assertEquals(progress.getPendingBucketCount(), 0);
+        assertEquals(progress.getActiveBucketCount(), 1);
+
+        iter.update(bp2.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
+        assertFalse(iter.hasNext());
+        assertTrue(iter.isDone());
+        assertTrue(progress.isFinished());
+        assertEquals(progress.getActiveBucketCount(), 0);
+
+        {
+            StringBuilder finished = new StringBuilder();
+            finished.append("VDS bucket progress file (100.0% completed)\n");
+            finished.append(distBits);
+            finished.append('\n');
+            finished.append(0); // Cursor (not used by explicit)
+            finished.append('\n');
+            finished.append(3); // Finished
+            finished.append('\n');
+            finished.append(3); // Total
+            finished.append('\n');
+
+            assertEquals(finished.toString(), progress.toString());
+        }
+    }
+
+    /**
+     * Test that doing update() on a bucket several times in a row (without re-fetching
+     * from getNext first) works
+     * @throws ParseException
+     */
+    public void testActiveUpdate() throws ParseException {
+        BucketIdFactory idFactory = new BucketIdFactory();
+        ProgressToken progress = new ProgressToken();
+
+        VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
+                "id.group = \"yahoo.com\"", idFactory, 16, progress);
+
+        VisitorIterator.BucketProgress bp = iter.getNext();
+
+        assertEquals(progress.getPendingBucketCount(), 0);
+        assertEquals(progress.getActiveBucketCount(), 1);
+
+        BucketId superbucket = bp.getSuperbucket();
+        int usedBits = superbucket.getUsedBits();
+
+        iter.update(superbucket, new BucketId(usedBits + 2, superbucket.getId() | (2L << usedBits)));
+        assertEquals(progress.getPendingBucketCount(), 1);
+        assertEquals(progress.getActiveBucketCount(), 0);
+        iter.update(superbucket, new BucketId(usedBits + 2, superbucket.getId() | (1L << usedBits)));
+        assertEquals(progress.getPendingBucketCount(), 1);
+        assertEquals(progress.getActiveBucketCount(), 0);
+
+        bp = iter.getNext();
+        assertEquals(bp.getSuperbucket(), superbucket);
+        assertEquals(bp.getProgress(), new BucketId(usedBits + 2, superbucket.getId() | (1L << usedBits)));
+        assertEquals(progress.getPendingBucketCount(), 0);
+        assertEquals(progress.getActiveBucketCount(), 1);
+    }
+
+    /**
+     * Test that ensures doing update(superbucket, 0) simply puts the bucket back in
+     * pending
+     * @throws ParseException
+     */
+    public void testNullAndSuperUpdate() throws ParseException {
+        BucketIdFactory idFactory = new BucketIdFactory();
+        ProgressToken progress = new ProgressToken();
+
+        VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
+                "id.group = \"yahoo.com\"", idFactory, 16, progress);
+
+        assertEquals(progress.getPendingBucketCount(), 1);
+
+        VisitorIterator.BucketProgress bp = iter.getNext();
+        assertEquals(bp.getProgress(), new BucketId());
+        BucketId superbucket = bp.getSuperbucket();
+        BucketId sub = bp.getProgress();
+
+        assertFalse(iter.hasNext());
+        assertFalse(iter.isDone());
+        assertEquals(progress.getPendingBucketCount(), 0);
+        assertEquals(progress.getActiveBucketCount(), 1);
+
+        // 0-bucket
+        iter.update(superbucket, ProgressToken.NULL_BUCKET);
+        assertTrue(iter.hasNext());
+        assertFalse(iter.isDone());
+        assertEquals(progress.getPendingBucketCount(), 1);
+        assertEquals(progress.getActiveBucketCount(), 0);
+
+        VisitorIterator.BucketProgress bp2 = iter.getNext();
+        assertEquals(bp2.getSuperbucket(), superbucket);
+        assertEquals(bp2.getProgress(), ProgressToken.NULL_BUCKET);
+        assertEquals(progress.getPendingBucketCount(), 0);
+        assertEquals(progress.getActiveBucketCount(), 1);
+
+        // progress == super
+        iter.update(superbucket, superbucket);
+        assertTrue(iter.hasNext());
+        assertFalse(iter.isDone());
+        assertEquals(progress.getPendingBucketCount(), 1);
+        assertEquals(progress.getActiveBucketCount(), 0);
+
+        bp2 = iter.getNext();
+        assertEquals(bp2.getSuperbucket(), superbucket);
+        assertEquals(bp2.getProgress(), superbucket);
+        assertEquals(progress.getPendingBucketCount(), 0);
+        assertEquals(progress.getActiveBucketCount(), 1);
+    }
+
+    public void testDeserializedFinishedProgress() {
+        StringBuilder finished = new StringBuilder();
+        finished.append("VDS bucket progress file\n"); // legacy; no completion percentage
+        finished.append(17);
+        finished.append('\n');
+        finished.append(1L << 17); // Cursor
+        finished.append('\n');
+        finished.append(1L << 17); // Finished
+        finished.append('\n');
+        finished.append(1L << 17); // Total
+        finished.append('\n');
+
+        ProgressToken token = new ProgressToken(finished.toString());
+        assertEquals(token.getDistributionBitCount(), 17);
+        assertEquals(token.getTotalBucketCount(), 1L << 17);
+        assertEquals(token.getFinishedBucketCount(), 1L << 17);
+        assertEquals(token.getBucketCursor(), 1L << 17);
+        assertTrue(token.isFinished());
+
+        ProgressToken token2 = new ProgressToken(token.serialize());
+        assertEquals(17, token2.getDistributionBitCount());
+        assertEquals(1L << 17, token2.getTotalBucketCount());
+        assertEquals(1L << 17, token2.getFinishedBucketCount());
+        assertEquals(1L << 17, token2.getBucketCursor());
+        assertTrue(token2.isFinished());
+    }
+
+    public void testBucketProgressFraction() {
+        double epsilon = 0.00001;
+        // No progress
+        BucketId b_0 = new BucketId();
+        // No split; only superbucket (100%)
+        BucketId b_100_0 = new BucketId(16, 1234);
+        // 1 split (1/2)
+        BucketId b_50_1 = new BucketId(17,  1234);
+        BucketId b_100_1 = new BucketId(17, 1234 | (1 << 16));
+        // 2 splits (1/4)
+        BucketId b_25_2 = new BucketId(18, 1234);
+        BucketId b_50_2 = new BucketId(18, 1234 | (2 << 16));
+        BucketId b_75_2 = new BucketId(18, 1234 | (1 << 16));
+        BucketId b_100_2 = new BucketId(18, 1234 | (3 << 16));
+
+        ProgressToken p = new ProgressToken(16);
+
+        BucketId sb = new BucketId(16, 1234);
+
+        assertEquals(p.progressFraction(new BucketId(32, 1234), b_0), 0.0, epsilon);
+
+        assertEquals(p.progressFraction(sb, b_100_0), 1.0, epsilon);
+
+        assertEquals(p.progressFraction(sb, b_50_1),  0.5, epsilon);
+        assertEquals(p.progressFraction(sb, b_100_1), 1.0, epsilon);
+
+        assertEquals(p.progressFraction(sb, b_25_2),  0.25, epsilon);
+        assertEquals(p.progressFraction(sb, b_50_2),  0.5,  epsilon);
+        assertEquals(p.progressFraction(sb, b_75_2),  0.75, epsilon);
+        assertEquals(p.progressFraction(sb, b_100_2), 1.0,  epsilon);
+
+        assertEquals(p.progressFraction(new BucketId(0x8000000000000000L),
+                new BucketId(0xb0000fff00000000L)), 1.0, epsilon);
+    }
+
+    public void testProgressEstimation() throws ParseException {
+        int distBits = 4;
+
+        BucketIdFactory idFactory = new BucketIdFactory();
+        ProgressToken progress = new ProgressToken();
+
+        // Create a range of [0, 16) superbuckets
+        VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
+                "id.group != \"yahoo.com\"", idFactory, distBits, progress);
+
+        assertEquals(progress.getDistributionBitCount(), 4);
+
+        double epsilon = 0.00001;
+        assertEquals(progress.percentFinished(), 0, epsilon);
+        VisitorIterator.BucketProgress bp = iter.getNext();
+        // Finish first superbucket (6.25% total)
+        iter.update(bp.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
+        assertEquals(progress.percentFinished(), 6.25, epsilon);
+        assertEquals(progress.getFinishedBucketCount(), 1);
+
+        bp = iter.getNext();
+        VisitorIterator.BucketProgress bp3 = iter.getNext();
+        VisitorIterator.BucketProgress bp4 = iter.getNext();
+
+        // Finish second (12.5% total)
+        iter.update(bp.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
+        assertEquals(progress.percentFinished(), 12.5, epsilon);
+        assertEquals(progress.getFinishedBucketCount(), 2);
+
+        // Finish third bucket 75% through (17.1875% total)
+        iter.update(bp3.getSuperbucket(), new BucketId(distBits + 2, bp3.getSuperbucket().getId() | (1 << distBits)));
+        assertEquals(progress.percentFinished(), 17.1875, epsilon);
+        assertEquals(progress.getFinishedBucketCount(), 2);
+
+        // Finish fourth bucket 25% through (18.75% total)
+        iter.update(bp4.getSuperbucket(), new BucketId(distBits + 2, bp4.getSuperbucket().getId()));
+        assertEquals(progress.percentFinished(), 18.75, epsilon);
+        assertEquals(progress.getFinishedBucketCount(), 2);
+        // Finish all buckets
+        iter.update(bp4.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
+        iter.update(bp3.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
+        assertEquals(progress.percentFinished(), 25, epsilon);
+        assertEquals(progress.getFinishedBucketCount(), 4);
+
+        while (iter.hasNext()) {
+            bp = iter.getNext();
+            iter.update(bp.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
+        }
+
+        assertEquals(progress.getFinishedBucketCount(), 16);
+        assertEquals(progress.percentFinished(), 100, epsilon);
+    }
+
+    public void testBucketKeyWrapperOrdering() {
+        ProgressToken.BucketKeyWrapper bk1 = new ProgressToken.BucketKeyWrapper(0x0000000000000001L);
+        ProgressToken.BucketKeyWrapper bk2 = new ProgressToken.BucketKeyWrapper(0x7FFFFFFFFFFFFFFFL);
+        ProgressToken.BucketKeyWrapper bk3 = new ProgressToken.BucketKeyWrapper(0x8000000000000000L);
+        ProgressToken.BucketKeyWrapper bk4 = new ProgressToken.BucketKeyWrapper(0xFFFFFFFFFFFFFFFFL);
+        assertTrue(bk1.compareTo(bk2) < 0);
+        assertTrue(bk2.compareTo(bk3) < 0);
+        assertTrue(bk3.compareTo(bk4) < 0);
+        assertTrue(bk2.compareTo(bk1) > 0);
+        assertTrue(bk3.compareTo(bk2) > 0);
+        assertTrue(bk4.compareTo(bk3) > 0);
+        ProgressToken.BucketKeyWrapper bk5 = new ProgressToken.BucketKeyWrapper(0x7FFFFFFFFFFFFFFFL);
+        ProgressToken.BucketKeyWrapper bk6 = new ProgressToken.BucketKeyWrapper(0x8000000000000000L);
+        assertTrue(bk5.compareTo(bk2) == 0);
+        assertTrue(bk6.compareTo(bk3) == 0);
+    }
+
+    private void doTestBucketKeyGeneration(int db) {
+        // Can't use longs since they won't sort properly when MSB is set
+        ProgressToken.BucketKeyWrapper[] keys = new ProgressToken.BucketKeyWrapper[1 << db];
+
+        // Generate entire bucket space for db
+        for (int i = 0; i < (1 << db); ++i) {
+            keys[i] = new ProgressToken.BucketKeyWrapper(
+                    ProgressToken.bucketToKey(new BucketId(db, i).getId()));
+        }
+        Arrays.sort(keys);
+
+        boolean consistentKeys = true;
+        // Verify that makeNthBucketKey yields the same result as the equivalent
+        // ordered value in the array of keys
+        for (int i = 0; i < (1 << db); ++i) {
+            long genKey = ProgressToken.makeNthBucketKey(i, db);
+            long knownKey = keys[i].getKey();
+            if (genKey != knownKey) {
+                consistentKeys = false;
+                break;
+            }
+        }
+        assertTrue(consistentKeys);
+    }
+
+    public void testBucketKeyGeneration() {
+        // Due to the number of objects needed to be allocated, only test for a
+        // small set of distribution bits
+        for (int i = 1; i < 14; ++i) {
+            doTestBucketKeyGeneration(i);
+        }
+    }
+
+    public void testSingleBucketSplits() throws ParseException {
+        int db = 2;
+        BucketIdFactory idFactory = new BucketIdFactory();
+        ProgressToken p = new ProgressToken();
+
+        // Create a range of [0, 4) superbuckets
+        VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
+                "id.group != \"yahoo.com\"", idFactory, db, p);
+        VisitorIterator.BucketProgress bp = iter.getNext();
+        assertEquals(bp.getSuperbucket(), new BucketId(db, 0));
+        // Put back as pending
+        iter.update(bp.getSuperbucket(), new BucketId());
+        assertEquals(p.getPendingBucketCount(), 1);
+        p.splitPendingBucket(new BucketId(db, 0));
+        assertEquals(p.getPendingBucketCount(), 2);
+        bp = iter.getNext();
+        assertEquals(bp.getSuperbucket(), new BucketId(db + 1, 0)); // left split
+        assertEquals(bp.getProgress(), new BucketId(0));
+        bp = iter.getNext();
+        assertEquals(bp.getSuperbucket(), new BucketId(db + 1, 4)); // right split
+        assertEquals(bp.getProgress(), new BucketId(0));
+
+        bp = iter.getNext();
+        assertEquals(bp.getSuperbucket(), new BucketId(db, 2));
+        // Put back as pending, with a progress of 10010. This implies splitting
+        // the bucket should set both splits with a progress to 10010
+        iter.update(bp.getSuperbucket(), new BucketId(db + 3, 0x12));
+        assertEquals(p.getPendingBucketCount(), 1);
+        p.splitPendingBucket(new BucketId(db, 2));
+        assertEquals(p.getPendingBucketCount(), 2);
+        bp = iter.getNext();
+        assertEquals(bp.getSuperbucket(), new BucketId(db + 1, 2)); // left split
+        assertEquals(bp.getProgress(), new BucketId(db + 3, 0x12));
+        bp = iter.getNext();
+        assertEquals(bp.getSuperbucket(), new BucketId(db + 1, 6)); // right split
+        assertEquals(bp.getProgress(), new BucketId(db + 3, 0x12));
+
+        bp = iter.getNext();
+        // Put back as pending with a progress of 10101. This implies splitting the
+        // bucket should _discard_ left and set right's progress to 10101.
+        // Update: no it shouldn't, we now split with equal progress without
+        // discarding
+        assertEquals(bp.getSuperbucket(), new BucketId(db, 1));
+        iter.update(bp.getSuperbucket(), new BucketId(db + 3, 0x15));
+        assertEquals(p.getPendingBucketCount(), 1);
+        p.splitPendingBucket(new BucketId(db, 1));
+        assertEquals(p.getPendingBucketCount(), 2);
+        bp = iter.getNext();
+        assertEquals(bp.getSuperbucket(), new BucketId(db + 1, 1));
+        assertEquals(bp.getProgress(), new BucketId(db + 3, 0x15));
+        bp = iter.getNext();
+        assertEquals(bp.getSuperbucket(), new BucketId(db + 1, 5)); // right split
+        assertEquals(bp.getProgress(), new BucketId(db + 3, 0x15));
+    }
+
+    /**
+     * Test increasing the distribution bits for a full bucket space range
+     * source with no finished, active or pending buckets
+     * @throws ParseException upon docsel parse failure (shouldn't happen)
+     */
+    public void testRangeDistributionBitIncrease1NoPending() throws ParseException {
+        int db = 2;
+        BucketIdFactory idFactory = new BucketIdFactory();
+        ProgressToken p = new ProgressToken();
+
+        // Test for empty progress token. no splitting involved
+        VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
+                "id.group != \"yahoo.com\"", idFactory, db, p);
+
+        assertEquals(p.getTotalBucketCount(), 4);
+        iter.setDistributionBitCount(db + 1);
+        assertEquals(p.getTotalBucketCount(), 8);
+        assertEquals(p.getDistributionBitCount(), db + 1);
+        assertEquals(iter.getDistributionBitCount(), db + 1);
+        assertEquals(iter.getBucketSource().getDistributionBitCount(), db + 1);
+
+        int[] desired = new int[] { 0, 4, 2, 6, 1, 5, 3, 7 };
+        for (int i = 0; i < 8; ++i) {
+            VisitorIterator.BucketProgress bp = iter.getNext();
+            assertEquals(bp.getSuperbucket(), new BucketId(db + 1, desired[i]));
+        }
+    }
+
+    public void testRangeDistributionBitIncrease1AllBucketStates() throws ParseException {
+        int db = 3;
+        BucketIdFactory idFactory = new BucketIdFactory();
+        ProgressToken p = new ProgressToken();
+
+        VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
+                "id.group != \"yahoo.com\"", idFactory, db, p);
+
+        // For this test, have 1 finished bucket, 3 pending and 0 active (we
+        // want to have the splitting to be triggered immediately)
+        VisitorIterator.BucketProgress bp = iter.getNext();
+        iter.update(bp.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
+        VisitorIterator.BucketProgress[] bpp = new VisitorIterator.BucketProgress[3];
+        bpp[0] = iter.getNext();
+        bpp[1] = iter.getNext();
+        bpp[2] = iter.getNext();
+        iter.update(bpp[0].getSuperbucket(), new BucketId());
+        iter.update(bpp[1].getSuperbucket(), new BucketId());
+        iter.update(bpp[2].getSuperbucket(), new BucketId());
+
+        assertEquals(p.getFinishedBucketCount(), 1);
+        assertEquals(p.getPendingBucketCount(), 3);
+        assertEquals(p.getActiveBucketCount(), 0);
+
+        iter.setDistributionBitCount(db + 1);
+
+        assertEquals(p.getTotalBucketCount(), 16);
+        assertEquals(p.getFinishedBucketCount(), 2);
+        assertEquals(p.getPendingBucketCount(), 6);
+        assertEquals(p.getActiveBucketCount(), 0);
+        assertEquals(p.getDistributionBitCount(), db + 1);
+        assertEquals(iter.getDistributionBitCount(), db + 1);
+        assertEquals(iter.getBucketSource().getDistributionBitCount(), db + 1);
+
+        // Bucket 3:0x4 -> 4:0x4 & 4:0xC
+        assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x04));
+        assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x0C));
+        // Bucket 3:0x2 -> 4:0x2 & 4:0xA
+        assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x02));
+        assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x0A));
+        // Bucket 3:0x6 -> 4:0x6 & 4:0xE
+        assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x06));
+        assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x0E));
+
+        assertEquals(p.getPendingBucketCount(), 0);
+        // Bucket source should now begin returning from bucket 4:0x1
+        assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x01));
+        assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x09));
+        assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x05));
+        assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x0D));
+        assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x03));
+        // Assume correct from here on
+    }
+
+    public void testRangeDistributionIncreaseMultipleBits() throws ParseException {
+        int db = 16;
+        BucketIdFactory idFactory = new BucketIdFactory();
+        ProgressToken p = new ProgressToken();
+
+        VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
+                "id.group != \"yahoo.com\"", idFactory, db, p);
+
+        // For this test, have 3 finished bucket, 2 pending and 1 active
+        for (int i = 0; i < 3; ++i) {
+            iter.update(iter.getNext().getSuperbucket(), ProgressToken.FINISHED_BUCKET);
+        }
+
+        VisitorIterator.BucketProgress[] bpp = new VisitorIterator.BucketProgress[2];
+        bpp[0] = iter.getNext();
+        bpp[1] = iter.getNext();
+        VisitorIterator.BucketProgress bpa = iter.getNext(); // Leave this hanging as active
+        iter.update(bpp[0].getSuperbucket(), new BucketId());
+        iter.update(bpp[1].getSuperbucket(), new BucketId());
+
+        iter.setDistributionBitCount(20);
+        // ProgressToken doesn't change yet, since it had active buckets
+        assertEquals(p.getDistributionBitCount(), 16);
+        assertEquals(iter.getDistributionBitCount(), 20);
+        assertEquals(iter.getBucketSource().getDistributionBitCount(), 20);
+
+        assertFalse(iter.hasNext());
+        assertFalse(iter.isDone());
+        assertTrue(iter.getBucketSource().shouldYield());
+        assertEquals(p.getPendingBucketCount(), 2);
+        assertEquals(p.getActiveBucketCount(), 1);
+
+        // Finish active, triggering the consistency fixes
+        iter.update(bpa.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
+
+        assertEquals(p.getDistributionBitCount(), 20);
+        assertEquals(p.getPendingBucketCount(), 32);
+        assertEquals(p.getActiveBucketCount(), 0);
+        // Each bucket with db:16 becomes equal to 16 buckets with db:20, so
+        // the bucket space position must be 16 * 6 = 96
+        assertEquals(p.getBucketCursor(), 96);
+        // Each finished bucket also covers less ground, so count is upped
+        // accordingly
+        assertEquals(p.getFinishedBucketCount(), 16 * 4);
+
+        // Remove pending that came from the split
+        // Bucket space that should be covered by the 32 buckets is [48, 80)
+        // when using 20 distribution bits
+        for (int i = 0; i < 32; ++i) {
+            long testKey = ProgressToken.makeNthBucketKey(i + 48, 20);
+            VisitorIterator.BucketProgress bp = iter.getNext();
+            assertEquals(bp.getSuperbucket(), new BucketId(ProgressToken.keyToBucketId(testKey)));
+            iter.update(bp.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
+        }
+        assertEquals(p.getPendingBucketCount(), 0);
+        assertEquals(p.getFinishedBucketCount(), 16 * 6);
+
+        // Bucket source should now begin returning from bucket 20:0x6000
+        assertEquals(iter.getNext().getSuperbucket(), new BucketId(20, 0x6000));
+    }
+
+    public void testSingleBucketMerge() throws ParseException {
+        int db = 2;
+        BucketIdFactory idFactory = new BucketIdFactory();
+        ProgressToken p = new ProgressToken();
+
+        // Create a range of [0, 4) superbuckets
+        VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
+                "id.group != \"yahoo.com\"", idFactory, db, p);
+
+        VisitorIterator.BucketProgress bp = iter.getNext();
+        // Put back as pending and split it
+        iter.update(bp.getSuperbucket(), new BucketId());
+        p.splitPendingBucket(new BucketId(db, 0));
+        assertEquals(p.getPendingBucketCount(), 2);
+        // Merge both back into one node. Merge from left sibling with right present
+        p.mergePendingBucket(new BucketId(db + 1, 0));
+        assertEquals(p.getPendingBucketCount(), 1);
+        bp = iter.getNext();
+        assertEquals(bp.getSuperbucket(), new BucketId(db, 0));
+    }
+
+    public void testRangeDistributionBitDecrease1() throws ParseException {
+        int db = 16;
+        BucketIdFactory idFactory = new BucketIdFactory();
+        ProgressToken p = new ProgressToken();
+
+        VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
+                "id.group != \"yahoo.com\"", idFactory, db, p);
+
+        VisitorIterator.DistributionRangeBucketSource src
+                = (VisitorIterator.DistributionRangeBucketSource)iter.getBucketSource();
+
+        assertTrue(src.isLosslessResetPossible());
+
+        // For this test, have 3 finished buckets, 6 pending and 1 active
+        // This gives a sibling "distribution" of FF FP PP PP PA. When all
+        // active buckets have been updated, 3 merges should be triggered
+        for (int i = 0; i < 3; ++i) {
+            iter.update(iter.getNext().getSuperbucket(), ProgressToken.FINISHED_BUCKET);
+        }
+
+        assertFalse(src.isLosslessResetPossible());
+
+        VisitorIterator.BucketProgress[] bpp = new VisitorIterator.BucketProgress[6];
+        for (int i = 0; i < 6; ++i) {
+            bpp[i] = iter.getNext();
+        }
+        VisitorIterator.BucketProgress bpa = iter.getNext(); // Leave this hanging as active
+        for (int i = 0; i < 6; ++i) {
+            iter.update(bpp[i].getSuperbucket(), new BucketId());
+        }
+
+        assertEquals(p.getBucketCursor(), 10);
+
+        iter.setDistributionBitCount(db - 1);
+        assertEquals(iter.getDistributionBitCount(), db - 1);
+        assertEquals(p.getDistributionBitCount(), db);
+        assertEquals(iter.getBucketSource().getDistributionBitCount(), db - 1);
+        // The iterator is waiting patiently for all active buckets to be updated,
+        // at which point it will performed the merging and actually updating the
+        // progress token's distribution bit count
+        assertTrue(iter.getBucketSource().shouldYield());
+        assertFalse(iter.hasNext());
+        assertFalse(iter.isDone());
+        assertEquals(p.getActiveBucketCount(), 1);
+        iter.update(bpa.getSuperbucket(), new BucketId());
+
+        assertEquals(p.getDistributionBitCount(), db - 1);
+        assertEquals(p.getActiveBucketCount(), 0);
+        assertEquals(p.getPendingBucketCount(), 4); // 3 merges, P PP PP PP -> P P P P
+
+        assertEquals(p.getFinishedBucketCount(), 1);
+        assertEquals(p.getBucketCursor(), 5);
+    }
+
+    // Test that splitting and merging from and to the same db count gives
+    // back the initial state
+    public void testRangeDistributionBitIncreaseDecrease() throws ParseException {
+        int db = 16;
+        BucketIdFactory idFactory = new BucketIdFactory();
+        ProgressToken p = new ProgressToken();
+
+        VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
+                "id.group != \"yahoo.com\"", idFactory, db, p);
+
+        VisitorIterator.DistributionRangeBucketSource src
+                = (VisitorIterator.DistributionRangeBucketSource)iter.getBucketSource();
+
+        assertTrue(src.isLosslessResetPossible());
+
+        // "Sabotage" resetting by having at least 1 finished
+        iter.update(iter.getNext().getSuperbucket(), ProgressToken.FINISHED_BUCKET);
+
+        VisitorIterator.BucketProgress[] bpp = new VisitorIterator.BucketProgress[4];
+        for (int i = 0; i < 4; ++i) {
+            bpp[i] = iter.getNext();
+        }
+        for (int i = 0; i < 4; ++i) {
+            iter.update(bpp[i].getSuperbucket(), new BucketId());
+        }
+
+        assertFalse(src.isLosslessResetPossible());
+
+        iter.setDistributionBitCount(20);
+        assertEquals(p.getDistributionBitCount(), 20);
+        assertEquals(p.getPendingBucketCount(), 4 << 4);
+        assertFalse(iter.getBucketSource().shouldYield());
+        assertEquals(p.getBucketCursor(), 5 << 4);
+
+        iter.setDistributionBitCount(16);
+
+        assertEquals(p.getDistributionBitCount(), 16);
+        assertEquals(p.getPendingBucketCount(), 4);
+        assertFalse(iter.getBucketSource().shouldYield());
+        assertEquals(p.getBucketCursor(), 5);
+    }
+
+    // Test that intermittent changes in distribution are handled properly, e.g.
+    // changing from 11 -> 9 with X active and then before all those are flushed,
+    // the distribution goes up to 12
+    public void testRangeDistributionBitChangeWithoutDone() throws ParseException {
+        int db = 11;
+        BucketIdFactory idFactory = new BucketIdFactory();
+        ProgressToken p = new ProgressToken();
+
+        VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
+                "id.group != \"yahoo.com\"", idFactory, db, p);
+
+        VisitorIterator.DistributionRangeBucketSource src
+                = (VisitorIterator.DistributionRangeBucketSource)iter.getBucketSource();
+
+        VisitorIterator.BucketProgress[] bpp = new VisitorIterator.BucketProgress[4];
+        for (int i = 0; i < 4; ++i) {
+            bpp[i] = iter.getNext();
+        }
+        for (int i = 0; i < 2; ++i) {
+            iter.update(bpp[i].getSuperbucket(), new BucketId());
+        }
+
+        assertFalse(src.isLosslessResetPossible());
+
+        // Now 2 pending, 2 active
+
+        iter.setDistributionBitCount(9);
+        assertEquals(p.getDistributionBitCount(), 11);
+        assertEquals(p.getActiveBucketCount(), 2);
+        assertEquals(p.getPendingBucketCount(), 2);
+        assertTrue(iter.getBucketSource().shouldYield());
+        // Update as pending, still with old count since there's 1 more active
+        // with bpp[2]. Have progress so that lossless reset isn't possible
+        iter.update(bpp[3].getSuperbucket(), new BucketId(15, bpp[3].getSuperbucket().getId()));
+
+        iter.setDistributionBitCount(12);
+        assertEquals(p.getActiveBucketCount(), 1);
+        assertEquals(p.getPendingBucketCount(), 3);
+        assertTrue(iter.getBucketSource().shouldYield());
+
+        // Serialize before token is updated to 12 bits
+        String serialized = p.toString();
+
+        iter.update(bpp[2].getSuperbucket(), ProgressToken.FINISHED_BUCKET);
+
+        assertEquals(p.getActiveBucketCount(), 0);
+        // All active buckets are at db=11, so they should be split once each
+        assertEquals(p.getPendingBucketCount(), 3 * 2);
+        assertFalse(iter.getBucketSource().shouldYield());
+        assertEquals(p.getFinishedBucketCount(), 2);
+
+        // Ensure we get a consistent progress token imported
+        ProgressToken p2 = new ProgressToken(serialized);
+        assertEquals(p2.getDistributionBitCount(), 11); // Not yet updated
+
+        BucketIdFactory idFactory2 = new BucketIdFactory();
+        VisitorIterator iter2 = VisitorIterator.createFromDocumentSelection(
+                "id.group != \"yahoo.com\"", idFactory2, 1, p2);
+
+        // Not yet updated, since we don't trust the initial BucketIdFactory
+        assertEquals(iter2.getDistributionBitCount(), 11);
+        assertEquals(p2.getDistributionBitCount(), 11);
+        iter2.setDistributionBitCount(12);
+        // Now it has been updated
+        assertEquals(p2.getDistributionBitCount(), 12);
+        assertEquals(p2.getPendingBucketCount(), 8);
+        assertEquals(p2.getBucketCursor(), 8);
+        assertEquals(p2.getFinishedBucketCount(), 0);
+    }
+
+    // Test a drop from 31->11 bits upon iteration start
+    public void testRangeDistributionBitInitialDrop() throws ParseException {
+        int db = 31;
+        BucketIdFactory idFactory = new BucketIdFactory();
+        ProgressToken p = new ProgressToken();
+
+        VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
+                "id.group != \"yahoo.com\"", idFactory, db, p);
+
+        VisitorIterator.BucketProgress[] bp = new VisitorIterator.BucketProgress[3];
+        bp[0] = iter.getNext();
+        bp[1] = iter.getNext();
+        bp[2] = iter.getNext();
+        iter.update(bp[2].getSuperbucket(), new BucketId());
+        iter.update(bp[1].getSuperbucket(), new BucketId());
+        assertEquals(p.getActiveBucketCount(), 1);
+
+        iter.setDistributionBitCount(11);
+
+        assertFalse(iter.hasNext());
+        assertFalse(iter.isDone());
+        assertEquals(p.getActiveBucketCount(), 1);
+
+        // Updating the active bucket allows the merging to take place
+        iter.update(new BucketId(31, 0), new BucketId());
+
+        assertTrue(iter.hasNext());
+        assertFalse(iter.isDone());
+
+        // All pending buckets should have been merged down to just 1 now
+        // Update: now rather gets reset
+        assertEquals(p.getPendingBucketCount(), 0);
+        assertEquals(p.getActiveBucketCount(), 0);
+        assertEquals(p.getFinishedBucketCount(), 0);
+        assertEquals(p.getBucketCursor(), 0);
+
+        bp[0] = iter.getNext();
+        assertEquals(bp[0].getSuperbucket(), new BucketId(11, 0));
+    }
+
+    // Similar to testRangeDistributionBitInitialDrop, but going from 1 to 11
+    // This tests that doing so may be done in an optimized way rather than
+    // attempting to split enough buckets to cover the entire bucket space!
+    public void testRangeDistributionLosslessReset() throws ParseException {
+        int db = 1;
+        BucketIdFactory idFactory = new BucketIdFactory();
+        ProgressToken p = new ProgressToken();
+
+        VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
+                "id.group != \"yahoo.com\"", idFactory, db, p);
+
+        VisitorIterator.DistributionRangeBucketSource src
+                = (VisitorIterator.DistributionRangeBucketSource)iter.getBucketSource();
+
+        VisitorIterator.BucketProgress[] bp = new VisitorIterator.BucketProgress[2];
+        bp[0] = iter.getNext();
+        bp[1] = iter.getNext();
+
+        String serialized = p.toString();
+
+        assertFalse(src.isLosslessResetPossible());
+
+        iter.update(bp[1].getSuperbucket(), new BucketId());
+        assertEquals(p.getActiveBucketCount(), 1);
+
+        iter.setDistributionBitCount(11);
+
+        assertFalse(src.isLosslessResetPossible());
+        assertEquals(p.getDistributionBitCount(), 1); // Still at 1
+
+        assertFalse(iter.hasNext());
+        assertFalse(iter.isDone());
+        assertEquals(p.getActiveBucketCount(), 1);
+
+        // Updating the active bucket allows the reset to take place
+        iter.update(new BucketId(1, 0), new BucketId());
+
+        assertTrue(iter.hasNext());
+        assertFalse(iter.isDone());
+
+        // Should not be any buckets pending/active and the cursor should be
+        // back at 0
+        assertEquals(p.getPendingBucketCount(), 0);
+        assertEquals(p.getActiveBucketCount(), 0);
+        assertEquals(p.getFinishedBucketCount(), 0);
+        assertEquals(p.getBucketCursor(), 0);
+        assertEquals(p.getDistributionBitCount(), 11);
+
+        bp[0] = iter.getNext();
+        assertEquals(bp[0].getSuperbucket(), new BucketId(11, 0));
+
+        // Ensure resetting also works when you're importing existing
+        // progress
+        p = new ProgressToken(serialized);
+        idFactory = new BucketIdFactory();
+        iter = VisitorIterator.createFromDocumentSelection(
+                "id.group != \"yahoo.com\"", idFactory, 1, p);
+
+        iter.setDistributionBitCount(11);
+
+        assertEquals(p.getPendingBucketCount(), 0);
+        assertEquals(p.getActiveBucketCount(), 0);
+        assertEquals(p.getFinishedBucketCount(), 0);
+        assertEquals(p.getBucketCursor(), 0);
+        assertEquals(p.getDistributionBitCount(), 11);
+
+        bp[0] = iter.getNext();
+        assertEquals(bp[0].getSuperbucket(), new BucketId(11, 0));
+    }
+
+    public void testExplicitDistributionBitIncrease() throws ParseException {
+        int distBits = 12;
+
+        BucketIdFactory idFactory = new BucketIdFactory();
+        ProgressToken p = new ProgressToken();
+
+        VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
+                "id.user == 1234 or id.user == 6789 or id.user == 8009", idFactory, distBits, p);
+
+        assertEquals(iter.getDistributionBitCount(), distBits);
+        assertEquals(p.getDistributionBitCount(), distBits);
+        assertEquals(iter.getBucketSource().getDistributionBitCount(), distBits);
+
+        iter.update(iter.getNext().getSuperbucket(), ProgressToken.FINISHED_BUCKET);
+        iter.setDistributionBitCount(16);
+
+        assertEquals(iter.getDistributionBitCount(), 16);
+        assertEquals(p.getDistributionBitCount(), 16);
+        assertEquals(iter.getBucketSource().getDistributionBitCount(), 16);
+        // Changing dist bits for explicit source should change nothing
+        assertEquals(p.getPendingBucketCount(), 2);
+        assertEquals(p.getFinishedBucketCount(), 1);
+        assertEquals(p.getTotalBucketCount(), 3);
+    }
+
+    public void testExplicitDistributionBitDecrease() throws ParseException {
+        int distBits = 20;
+
+        BucketIdFactory idFactory = new BucketIdFactory();
+        ProgressToken p = new ProgressToken();
+
+        VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
+                "id.user == 1234 or id.user == 6789 or id.user == 8009", idFactory, distBits, p);
+
+        assertEquals(iter.getDistributionBitCount(), distBits);
+        assertEquals(p.getDistributionBitCount(), distBits);
+        assertEquals(iter.getBucketSource().getDistributionBitCount(), distBits);
+
+        iter.update(iter.getNext().getSuperbucket(), ProgressToken.FINISHED_BUCKET);
+        iter.setDistributionBitCount(16);
+
+        assertEquals(iter.getDistributionBitCount(), 16);
+        assertEquals(p.getDistributionBitCount(), 16);
+        assertEquals(iter.getBucketSource().getDistributionBitCount(), 16);
+        // Changing dist bits for explicit source should change nothing
+        assertEquals(p.getPendingBucketCount(), 2);
+        assertEquals(p.getFinishedBucketCount(), 1);
+        assertEquals(p.getTotalBucketCount(), 3);
+    }
+
+    public void testExplicitDistributionImportNoTruncation() throws ParseException {
+        BucketIdFactory idFactory = new BucketIdFactory();
+        ProgressToken p = new ProgressToken();
+
+        VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
+                "id.user == 1234 or id.user == 6789 or id.user == 8009", idFactory, 20, p);
+        assertEquals(20, iter.getDistributionBitCount());
+        assertEquals(20, p.getDistributionBitCount());
+        assertEquals(20, iter.getBucketSource().getDistributionBitCount());
+
+        iter.update(iter.getNext().getSuperbucket(), ProgressToken.FINISHED_BUCKET);
+
+        // Make sure no truncation is done on import
+        String serialized = p.toString();
+        ProgressToken p2 = new ProgressToken(serialized);
+        BucketIdFactory idFactory2 = new BucketIdFactory();
+        VisitorIterator iter2 = VisitorIterator.createFromDocumentSelection(
+                "id.user == 1234 or id.user == 6789 or id.user == 8009", idFactory2, 1, p2);
+        assertEquals(20, iter2.getDistributionBitCount());
+        assertEquals(20, p2.getDistributionBitCount());
+        assertEquals(20, iter2.getBucketSource().getDistributionBitCount());
+        assertEquals(2, p2.getPendingBucketCount());
+        assertEquals(1, p2.getFinishedBucketCount());
+        assertEquals(3, p2.getTotalBucketCount());
+    }
+
+    public void testImportProgressWithOutdatedDistribution() throws ParseException {
+        String input = "VDS bucket progress file\n" +
+                "10\n" +
+                "503\n" +
+                "500\n" +
+                "1024\n" +
+                "28000000000000be:0\n" +
+                "28000000000002be:0\n" +
+                "28000000000001be:0\n";
+
+        int db = 12;
+        BucketIdFactory idFactory = new BucketIdFactory();
+        ProgressToken p = new ProgressToken(input);
+        assertEquals(10, p.getDistributionBitCount());
+
+        VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
+                "id.group != \"yahoo.com\"", idFactory, 1, p);
+
+        iter.setDistributionBitCount(12);
+        assertEquals(iter.getDistributionBitCount(), 12);
+        assertEquals(p.getDistributionBitCount(), 12);
+        assertEquals(iter.getBucketSource().getDistributionBitCount(), 12);
+
+        assertEquals(p.getTotalBucketCount(), 1 << 12);
+        assertEquals(p.getFinishedBucketCount(), 500 << 2);
+        assertEquals(p.getPendingBucketCount(), 3 << 2);
+        assertEquals(p.getActiveBucketCount(), 0);
+        assertEquals(p.getBucketCursor(), 503 << 2);
+        assertTrue(iter.hasNext());
+
+        ProgressToken p2 = new ProgressToken(p.serialize());
+        assertEquals(p2.getDistributionBitCount(), 12);
+        assertEquals(p2.getTotalBucketCount(), 1 << 12);
+        assertEquals(p2.getFinishedBucketCount(), 500 << 2);
+        assertEquals(p2.getPendingBucketCount(), 3 << 2);
+        assertEquals(p2.getActiveBucketCount(), 0);
+        assertEquals(p2.getBucketCursor(), 503 << 2);
+    }
+
+    public void testImportInconsistentProgressIncrease() throws ParseException {
+        // Bucket progress "file" that upon time of changing from 4 to 7
+        // distribution bits and writing the progress had an active bucket
+        String input = "VDS bucket progress file\n" +
+                "7\n" +
+                "32\n" +
+                "24\n" +
+                "128\n" +
+                "100000000000000c:0\n";
+        // Now we're at 8 distribution bits
+        int db = 8;
+        BucketIdFactory idFactory = new BucketIdFactory();
+        ProgressToken p = new ProgressToken(input);
+        assertEquals(7, p.getDistributionBitCount());
+        assertEquals(p.getTotalBucketCount(), 1 << 7);
+        assertEquals(p.getFinishedBucketCount(), 24);
+        // Not yet corrected
+        assertEquals(p.getPendingBucketCount(), 1);
+        assertEquals(p.getActiveBucketCount(), 0);
+        assertEquals(32, p.getBucketCursor());
+
+        VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
+                "id.group != \"yahoo.com\"", idFactory, 1, p);
+
+        // Now the range handler should have corrected the progress
+        // (but not messed with the distribution bit count)
+        assertEquals(7, p.getDistributionBitCount());
+        assertEquals(p.getPendingBucketCount(), 1 << 3);
+        assertEquals(p.getActiveBucketCount(), 0);
+        assertEquals(24 + (1 << 3), p.getBucketCursor());
+
+        iter.setDistributionBitCount(8);
+
+        assertEquals(iter.getDistributionBitCount(), 8);
+        assertEquals(p.getDistributionBitCount(), 8);
+        assertEquals(iter.getBucketSource().getDistributionBitCount(), 8);
+
+        assertEquals(p.getTotalBucketCount(), 1 << 8);
+        assertEquals(p.getFinishedBucketCount(), 24 << 1);
+        assertEquals(p.getPendingBucketCount(), 1 << 4); // Split 4 -> 7 bits, then 7 -> 8 bits
+        assertEquals(p.getActiveBucketCount(), 0);
+        assertEquals(p.getBucketCursor(), 24*2 + (1 << 4));
+        assertTrue(iter.hasNext());
+    }
+
+    public void testImportInconsistentProgressDecrease() throws ParseException {
+        // Bucket progress "file" that upon time of changing from 4 to 7
+        // distribution bits and writing the progress had an active bucket
+        String input = "VDS bucket progress file\n" +
+                "7\n" +
+                "32\n" +
+                "24\n" +
+                "128\n" +
+                "100000000000000c:0\n";
+        BucketIdFactory idFactory = new BucketIdFactory();
+        ProgressToken p = new ProgressToken(input);
+
+        VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
+                "id.group != \"yahoo.com\"", idFactory, 1, p);
+
+        assertEquals(iter.getDistributionBitCount(), 7);
+        // Now we're at 6 distribution bits
+        iter.setDistributionBitCount(6);
+
+        assertEquals(iter.getDistributionBitCount(), 6);
+        assertEquals(p.getDistributionBitCount(), 6);
+        assertEquals(iter.getBucketSource().getDistributionBitCount(), 6);
+
+        assertEquals(p.getTotalBucketCount(), 1 << 6);
+        assertEquals(p.getFinishedBucketCount(), 24 >> 1);
+        assertEquals(p.getPendingBucketCount(), 1 << 2); // Split 4 -> 7 bits, merge 7 -> 6 bits
+        assertEquals(p.getActiveBucketCount(), 0);
+        assertEquals(p.getBucketCursor(), 24/2 + (1 << 2));
+        assertTrue(iter.hasNext());
+    }
+
+    public void testEntireBucketSpaceCovered() throws ParseException {
+        int db = 4;
+        BucketIdFactory idFactory = new BucketIdFactory();
+        ProgressToken p = new ProgressToken();
+
+        VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
+                "id.group != \"yahoo.com\"", idFactory, db, p);
+
+        VisitorIterator.BucketProgress[] bpp = new VisitorIterator.BucketProgress[3];
+
+        for (int i = 0; i < 3; ++i) {
+            bpp[i] = iter.getNext();
+        }
+        for (int i = 0; i < 3; ++i) {
+            // Must use non-zero progress or all pending will be optimized
+            // away by the reset-logic
+            iter.update(bpp[i].getSuperbucket(),
+                    new BucketId(db + 1, bpp[i].getSuperbucket().getId()));
+        }
+
+        Set<BucketId> buckets = new TreeSet<BucketId>();
+        db = 7;
+        for (int i = 0; i < (1 << db); ++i) {
+            buckets.add(new BucketId(db, i));
+        }
+
+        iter.setDistributionBitCount(db);
+        assertEquals(p.getFinishedBucketCount(), 0);
+        assertEquals(p.getPendingBucketCount(), 3 << 3);
+
+        // Ensure all buckets are visited once and only once
+        while (iter.hasNext()) {
+            VisitorIterator.BucketProgress bp = iter.getNext();
+            assertTrue(buckets.contains(bp.getSuperbucket()));
+            buckets.remove(bp.getSuperbucket());
+        }
+
+        assertTrue(buckets.isEmpty());
+    }
+
+    public void testExceptionOnWrongDocumentSelection() throws ParseException {
+        BucketIdFactory idFactory = new BucketIdFactory();
+        // Since we don't store the actual original document selection in the
+        // progress files, we can't really tell whether or not a "wrong" document
+        // selection has been given, so we just do a best effort by checking
+        // that the number of total buckets match up and that the bucket cursor
+        // isn't set for explicit sources
+
+        // Try to pass a known document selection to an unknown docsel iterator
+        boolean caughtIt = false;
+        try {
+            ProgressToken p = new ProgressToken("VDS bucket progress file\n16\n3\n1\n3\n"
+                    + "8000000000001f49:0\n8000000000001a85:0\n");
+
+            VisitorIterator.createFromDocumentSelection("id.group != \"yahoo.com\"", idFactory, 16, p);
+        }
+        catch (IllegalArgumentException e) {
+            caughtIt = true;
+        }
+        assertTrue(caughtIt);
+
+        // Now try it the other way around
+        caughtIt = false;
+        try {
+            ProgressToken p = new ProgressToken("VDS bucket progress file\n" +
+                    "10\n" +
+                    "503\n" +
+                    "500\n" +
+                    "1024\n" +
+                    "28000000000000be:0\n" +
+                    "28000000000002be:0\n" +
+                    "28000000000001be:0\n");
+
+            VisitorIterator.createFromDocumentSelection("id.group=\"yahoo.com\" or id.user=555", idFactory, 16, p);
+        }
+        catch (IllegalArgumentException e) {
+            caughtIt = true;
+        }
+        assertTrue(caughtIt);
+    }
+
+    public void testIsBucketFinished() throws ParseException {
+        BucketIdFactory idFactory = new BucketIdFactory();
+        ProgressToken p = new ProgressToken();
+        VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
+                "id.group != \"yahoo.com\"", idFactory, 4, p);
+
+        assertFalse(p.isBucketFinished(new BucketId(32, 0)));
+        // Finish superbucket 0x0000
+        iter.update(iter.getNext().getSuperbucket(), ProgressToken.FINISHED_BUCKET);
+        assertTrue(p.isBucketFinished(new BucketId(32, 0)));
+        // Cursor is 1, but bucket 0x1000 not yet returned
+        assertFalse(p.isBucketFinished(new BucketId(32, 1 << 3)));
+        VisitorIterator.BucketProgress bp = iter.getNext();
+        // Cursor 2, 0x1000 returned but is contained in state, so not finished
+        assertFalse(p.isBucketFinished(new BucketId(32, 1 << 3)));
+        iter.update(bp.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
+        assertTrue(p.isBucketFinished(new BucketId(32, 1 << 3)));
+        // Only superbucket part is used
+        assertTrue(p.isBucketFinished(new BucketId(32, 0x12345670)));  // ...0000
+        assertTrue(p.isBucketFinished(new BucketId(32, 0x12345678)));  // ...1000
+        assertFalse(p.isBucketFinished(new BucketId(32, 0x12345671))); // ...0001
+        assertFalse(p.isBucketFinished(new BucketId(32, 0x12345679))); // ...1001
+    }
+
+    // Test that altering distribution bit count sets ProgressToken as
+    // inconsistent when there are active buckets
+    public void testInconsistentState() throws ParseException {
+        int db = 16;
+        BucketIdFactory idFactory = new BucketIdFactory();
+        ProgressToken p = new ProgressToken();
+
+        VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
+                "id.group != \"yahoo.com\"", idFactory, db, p);
+
+        // For this test, have 3 finished bucket, 2 pending and 1 active
+        for (int i = 0; i < 3; ++i) {
+            iter.update(iter.getNext().getSuperbucket(), ProgressToken.FINISHED_BUCKET);
+        }
+
+        VisitorIterator.BucketProgress[] bpp = new VisitorIterator.BucketProgress[2];
+        bpp[0] = iter.getNext();
+        bpp[1] = iter.getNext();
+        VisitorIterator.BucketProgress bpa = iter.getNext(); // Leave this hanging as active
+        iter.update(bpp[0].getSuperbucket(), new BucketId());
+        iter.update(bpp[1].getSuperbucket(), new BucketId());
+
+        assertFalse(p.isInconsistentState());
+        iter.setDistributionBitCount(20);
+        assertTrue(p.isInconsistentState());
+
+        // Finish active, triggering the consistency fixes
+        iter.update(bpa.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
+
+        assertFalse(p.isInconsistentState());
+    }
+
+    public void testMalformedProgressFile() {
+        boolean caughtIt = false;
+        try {
+            new ProgressToken("VDS bucket progress file\n" +
+                    "10\n" +
+                    "503\n" +
+                    "500\n" +
+                    "1024\n" +
+                    "28000000000000be:0\n" +
+                    "28000000000002be:");
+        } catch (IllegalArgumentException e) {
+            caughtIt = true;
+        }
+        assertTrue(caughtIt);
+    }
+
+    public void testFailOnTooFewLinesInFile() {
+        boolean caughtIt = false;
+        try {
+            new ProgressToken("VDS bucket progress file\n" +
+                    "10\n" +
+                    "503\n");
+        } catch (IllegalArgumentException e) {
+            caughtIt = true;
+        }
+        assertTrue(caughtIt);
+    }
+
+    public void testUnknownFirstHeaderLine() {
+        boolean caughtIt = false;
+        try {
+            new ProgressToken("Smurf Time 3000\n" +
+                    "10\n" +
+                    "503\n" +
+                    "500\n" +
+                    "1024\n" +
+                    "28000000000000be:0\n" +
+                    "28000000000002be:0");
+        } catch (IllegalArgumentException e) {
+            caughtIt = true;
+        }
+        assertTrue(caughtIt);
+    }
+
+    public void testBinaryProgressSerialization() {
+        String input = "VDS bucket progress file (48.828125% completed)\n" +
+                "10\n" +
+                "503\n" +
+                "500\n" +
+                "1024\n" +
+                "28000000000000be:0\n" +
+                "28000000000002be:0\n" +
+                "28000000000001be:0\n";
+        ProgressToken p = new ProgressToken(input);
+        byte[] buf = p.serialize();
+        ProgressToken p2 = new ProgressToken(buf);
+        assertEquals(input, p2.toString());
+    }
+ }
diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/ReplyMergerTestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/ReplyMergerTestCase.java
index 9f4eace387c..e5db70c9967 100644
--- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/ReplyMergerTestCase.java
+++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/ReplyMergerTestCase.java
@@ -1,228 +1,228 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.documentapi.messagebus.protocol;

-

-import com.yahoo.collections.Tuple2;

-import com.yahoo.messagebus.*;

-import com.yahoo.messagebus.Error;

-import org.junit.Before;

-import org.junit.Test;

-import static org.junit.Assert.*;

-import static org.hamcrest.CoreMatchers.*;

-

-@SuppressWarnings("deprecation")

-public class ReplyMergerTestCase {

-

-    private ReplyMerger merger;

-

-    @Before

-    public void setUp() {

-        merger = new ReplyMerger();

-    }

-

-    @Test

-    public void mergingGenericRepliesWithNoErrorsPicksFirstReply() {

-        Reply r1 = new EmptyReply();

-        Reply r2 = new EmptyReply();

-        Reply r3 = new EmptyReply();

-        merger.merge(0, r1);

-        merger.merge(1, r2);

-        merger.merge(2, r3);

-        Tuple2<Integer, Reply> ret = merger.mergedReply();

-

-        assertThat(ret.first, is(0));

-        assertThat(ret.second, sameInstance(r1));

-    }

-

-    @Test

-    public void mergingSingleReplyWithOneErrorReturnsEmptyReplyWithError() {

-        Reply r1 = new EmptyReply();

-        Error error = new Error(1234, "oh no!");

-        r1.addError(error);

-        merger.merge(0, r1);

-        Tuple2<Integer, Reply> ret = merger.mergedReply();

-

-        assertThat(ret.first, nullValue());

-        assertThat(ret.second, not(sameInstance(r1)));

-        assertThatErrorsMatch(new Error[] { error }, ret);

-    }

-

-    @Test

-    public void mergingSingleReplyWithMultipleErrorsReturnsEmptyReplyWithAllErrors() {

-        Reply r1 = new EmptyReply();

-        Error errors[] = new Error[] {

-                new Error(1234, "oh no!"), new Error(4567, "oh dear!"),

-        };

-        r1.addError(errors[0]);

-        r1.addError(errors[1]);

-        merger.merge(0, r1);

-        Tuple2<Integer, Reply> ret = merger.mergedReply();

-

-        assertThat(ret.first, nullValue());

-        assertThat(ret.second, not(sameInstance(r1)));

-        assertThatErrorsMatch(errors, ret);

-    }

-

-    @Test

-    public void mergingMultipleRepliesWithMultipleErrorsReturnsEmptyReplyWithAllErrors() {

-        Reply r1 = new EmptyReply();

-        Reply r2 = new EmptyReply();

-        Error errors[] = new Error[] {

-                new Error(1234, "oh no!"), new Error(4567, "oh dear!"), new Error(678, "omg!"),

-        };

-        r1.addError(errors[0]);

-        r1.addError(errors[1]);

-        r2.addError(errors[2]);

-        merger.merge(0, r1);

-        merger.merge(1, r2);

-        Tuple2<Integer, Reply> ret = merger.mergedReply();

-

-        assertThat(ret.first, nullValue());

-        assertThat(ret.second, not(sameInstance(r1)));

-        assertThat(ret.second, not(sameInstance(r2)));

-        assertThatErrorsMatch(errors, ret);

-    }

-

-    @Test

-    public void returnIgnoredReplyWhenAllRepliesHaveOnlyIgnoredErrors() {

-        Reply r1 = new EmptyReply();

-        Reply r2 = new EmptyReply();

-        Error errors[] = new Error[] {

-                new Error(DocumentProtocol.ERROR_MESSAGE_IGNORED, "oh no!"),

-                new Error(DocumentProtocol.ERROR_MESSAGE_IGNORED, "oh dear!"),

-                new Error(DocumentProtocol.ERROR_MESSAGE_IGNORED, "omg!"),

-        };

-        r1.addError(errors[0]);

-        r1.addError(errors[1]);

-        r2.addError(errors[2]);

-

-        merger.merge(0, r1);

-        merger.merge(1, r2);

-        Tuple2<Integer, Reply> ret = merger.mergedReply();

-        assertThat(ret.first, nullValue());

-        assertThat(ret.second, not(sameInstance(r1)));

-        assertThat(ret.second, not(sameInstance(r2)));

-        // Only first ignore error from each reply

-        assertThatErrorsMatch(new Error[]{ errors[0], errors[2] }, ret);

-    }

-

-    @Test

-    public void successfulReplyTakesPrecedenceOverIgnoredReplyWhenNoErrors() {

-        Reply r1 = new EmptyReply();

-        Reply r2 = new EmptyReply();

-        Error errors[] = new Error[] {

-                new Error(DocumentProtocol.ERROR_MESSAGE_IGNORED, "oh no!"),

-        };

-        r1.addError(errors[0]);

-        merger.merge(0, r1);

-        merger.merge(1, r2);

-        Tuple2<Integer, Reply> ret = merger.mergedReply();

-        assertThat(ret.first, is(1));

-        assertThat(ret.second, sameInstance(r2));

-        // Only first ignore error from each reply

-        assertThatErrorsMatch(new Error[]{ }, ret);

-    }

-

-    @Test

-    public void nonIgnoredErrorTakesPrecedence() {

-        Reply r1 = new EmptyReply();

-        Reply r2 = new EmptyReply();

-        Error errors[] = new Error[] {

-                new Error(DocumentProtocol.ERROR_MESSAGE_IGNORED, "oh no!"),

-                new Error(DocumentProtocol.ERROR_ABORTED, "kablammo!"),

-                new Error(DocumentProtocol.ERROR_MESSAGE_IGNORED, "omg!"),

-        };

-        r1.addError(errors[0]);

-        r1.addError(errors[1]);

-        r2.addError(errors[2]);

-

-        merger.merge(0, r1);

-        merger.merge(1, r2);

-        Tuple2<Integer, Reply> ret = merger.mergedReply();

-        assertThat(ret.first, nullValue());

-        assertThat(ret.second, not(sameInstance(r1)));

-        assertThat(ret.second, not(sameInstance(r2)));

-        // All errors from replies with errors are included, not those that

-        // are fully ignored.

-        assertThatErrorsMatch(new Error[]{ errors[0], errors[1] }, ret);

-    }

-

-    @Test

-    public void returnRemoveDocumentReplyWhereDocWasFound() {

-        RemoveDocumentReply r1 = new RemoveDocumentReply();

-        RemoveDocumentReply r2 = new RemoveDocumentReply();

-        RemoveDocumentReply r3 = new RemoveDocumentReply();

-        r1.setWasFound(false);

-        r2.setWasFound(true);

-        r3.setWasFound(false);

-

-        merger.merge(0, r1);

-        merger.merge(1, r2);

-        merger.merge(2, r3);

-        Tuple2<Integer, Reply> ret = merger.mergedReply();

-        assertThat(ret.first, is(1));

-        assertThat(ret.second, sameInstance((Reply) r2));

-    }

-

-    @Test

-    public void returnFirstRemoveDocumentReplyIfNoDocsWereFound() {

-        RemoveDocumentReply r1 = new RemoveDocumentReply();

-        RemoveDocumentReply r2 = new RemoveDocumentReply();

-        r1.setWasFound(false);

-        r2.setWasFound(false);

-

-        merger.merge(0, r1);

-        merger.merge(1, r2);

-        Tuple2<Integer, Reply> ret = merger.mergedReply();

-        assertThat(ret.first, is(0));

-        assertThat(ret.second, sameInstance((Reply)r1));

-    }

-

-    @Test

-    public void returnUpdateDocumentReplyWhereDocWasFound() {

-        UpdateDocumentReply r1 = new UpdateDocumentReply();

-        UpdateDocumentReply r2 = new UpdateDocumentReply();

-        UpdateDocumentReply r3 = new UpdateDocumentReply();

-        r1.setWasFound(false);

-        r2.setWasFound(true); // return first reply

-        r3.setWasFound(true);

-

-        merger.merge(0, r1);

-        merger.merge(1, r2);

-        merger.merge(2, r3);

-        Tuple2<Integer, Reply> ret = merger.mergedReply();

-        assertThat(ret.first, is(1));

-        assertThat(ret.second, sameInstance((Reply)r2));

-    }

-

-    @Test

-    public void returnGetDocumentReplyWhereDocWasFound() {

-        GetDocumentReply r1 = new GetDocumentReply(null);

-        GetDocumentReply r2 = new GetDocumentReply(null);

-        GetDocumentReply r3 = new GetDocumentReply(null);

-        r2.setLastModified(12345L);

-

-        merger.merge(0, r1);

-        merger.merge(1, r2);

-        merger.merge(2, r3);

-        Tuple2<Integer, Reply> ret = merger.mergedReply();

-        assertThat(ret.first, is(1));

-        assertThat(ret.second, sameInstance((Reply)r2));

-    }

-

-    @Test

-    public void mergingZeroRepliesReturnsDefaultEmptyReply() {

-        Tuple2<Integer, Reply> ret = merger.mergedReply();

-        assertThat(ret.first, nullValue());

-        assertThat(ret.second, instanceOf(EmptyReply.class));

-        assertThatErrorsMatch(new Error[]{}, ret);

-    }

-

-    private void assertThatErrorsMatch(Error[] errors, Tuple2<Integer, Reply> ret) {

-        assertThat(ret.second.getNumErrors(), is(errors.length));

-        for (int i = 0; i < ret.second.getNumErrors(); ++i) {

-            assertThat(ret.second.getError(i).getCode(), is(errors[i].getCode()));

-            assertThat(ret.second.getError(i).getMessage(), is(errors[i].getMessage()));

-        }

-    }

-

-}

+package com.yahoo.documentapi.messagebus.protocol;
+
+import com.yahoo.collections.Tuple2;
+import com.yahoo.messagebus.*;
+import com.yahoo.messagebus.Error;
+import org.junit.Before;
+import org.junit.Test;
+import static org.junit.Assert.*;
+import static org.hamcrest.CoreMatchers.*;
+
+@SuppressWarnings("deprecation")
+public class ReplyMergerTestCase {
+
+    private ReplyMerger merger;
+
+    @Before
+    public void setUp() {
+        merger = new ReplyMerger();
+    }
+
+    @Test
+    public void mergingGenericRepliesWithNoErrorsPicksFirstReply() {
+        Reply r1 = new EmptyReply();
+        Reply r2 = new EmptyReply();
+        Reply r3 = new EmptyReply();
+        merger.merge(0, r1);
+        merger.merge(1, r2);
+        merger.merge(2, r3);
+        Tuple2<Integer, Reply> ret = merger.mergedReply();
+
+        assertThat(ret.first, is(0));
+        assertThat(ret.second, sameInstance(r1));
+    }
+
+    @Test
+    public void mergingSingleReplyWithOneErrorReturnsEmptyReplyWithError() {
+        Reply r1 = new EmptyReply();
+        Error error = new Error(1234, "oh no!");
+        r1.addError(error);
+        merger.merge(0, r1);
+        Tuple2<Integer, Reply> ret = merger.mergedReply();
+
+        assertThat(ret.first, nullValue());
+        assertThat(ret.second, not(sameInstance(r1)));
+        assertThatErrorsMatch(new Error[] { error }, ret);
+    }
+
+    @Test
+    public void mergingSingleReplyWithMultipleErrorsReturnsEmptyReplyWithAllErrors() {
+        Reply r1 = new EmptyReply();
+        Error errors[] = new Error[] {
+                new Error(1234, "oh no!"), new Error(4567, "oh dear!"),
+        };
+        r1.addError(errors[0]);
+        r1.addError(errors[1]);
+        merger.merge(0, r1);
+        Tuple2<Integer, Reply> ret = merger.mergedReply();
+
+        assertThat(ret.first, nullValue());
+        assertThat(ret.second, not(sameInstance(r1)));
+        assertThatErrorsMatch(errors, ret);
+    }
+
+    @Test
+    public void mergingMultipleRepliesWithMultipleErrorsReturnsEmptyReplyWithAllErrors() {
+        Reply r1 = new EmptyReply();
+        Reply r2 = new EmptyReply();
+        Error errors[] = new Error[] {
+                new Error(1234, "oh no!"), new Error(4567, "oh dear!"), new Error(678, "omg!"),
+        };
+        r1.addError(errors[0]);
+        r1.addError(errors[1]);
+        r2.addError(errors[2]);
+        merger.merge(0, r1);
+        merger.merge(1, r2);
+        Tuple2<Integer, Reply> ret = merger.mergedReply();
+
+        assertThat(ret.first, nullValue());
+        assertThat(ret.second, not(sameInstance(r1)));
+        assertThat(ret.second, not(sameInstance(r2)));
+        assertThatErrorsMatch(errors, ret);
+    }
+
+    @Test
+    public void returnIgnoredReplyWhenAllRepliesHaveOnlyIgnoredErrors() {
+        Reply r1 = new EmptyReply();
+        Reply r2 = new EmptyReply();
+        Error errors[] = new Error[] {
+                new Error(DocumentProtocol.ERROR_MESSAGE_IGNORED, "oh no!"),
+                new Error(DocumentProtocol.ERROR_MESSAGE_IGNORED, "oh dear!"),
+                new Error(DocumentProtocol.ERROR_MESSAGE_IGNORED, "omg!"),
+        };
+        r1.addError(errors[0]);
+        r1.addError(errors[1]);
+        r2.addError(errors[2]);
+
+        merger.merge(0, r1);
+        merger.merge(1, r2);
+        Tuple2<Integer, Reply> ret = merger.mergedReply();
+        assertThat(ret.first, nullValue());
+        assertThat(ret.second, not(sameInstance(r1)));
+        assertThat(ret.second, not(sameInstance(r2)));
+        // Only first ignore error from each reply
+        assertThatErrorsMatch(new Error[]{ errors[0], errors[2] }, ret);
+    }
+
+    @Test
+    public void successfulReplyTakesPrecedenceOverIgnoredReplyWhenNoErrors() {
+        Reply r1 = new EmptyReply();
+        Reply r2 = new EmptyReply();
+        Error errors[] = new Error[] {
+                new Error(DocumentProtocol.ERROR_MESSAGE_IGNORED, "oh no!"),
+        };
+        r1.addError(errors[0]);
+        merger.merge(0, r1);
+        merger.merge(1, r2);
+        Tuple2<Integer, Reply> ret = merger.mergedReply();
+        assertThat(ret.first, is(1));
+        assertThat(ret.second, sameInstance(r2));
+        // Only first ignore error from each reply
+        assertThatErrorsMatch(new Error[]{ }, ret);
+    }
+
+    @Test
+    public void nonIgnoredErrorTakesPrecedence() {
+        Reply r1 = new EmptyReply();
+        Reply r2 = new EmptyReply();
+        Error errors[] = new Error[] {
+                new Error(DocumentProtocol.ERROR_MESSAGE_IGNORED, "oh no!"),
+                new Error(DocumentProtocol.ERROR_ABORTED, "kablammo!"),
+                new Error(DocumentProtocol.ERROR_MESSAGE_IGNORED, "omg!"),
+        };
+        r1.addError(errors[0]);
+        r1.addError(errors[1]);
+        r2.addError(errors[2]);
+
+        merger.merge(0, r1);
+        merger.merge(1, r2);
+        Tuple2<Integer, Reply> ret = merger.mergedReply();
+        assertThat(ret.first, nullValue());
+        assertThat(ret.second, not(sameInstance(r1)));
+        assertThat(ret.second, not(sameInstance(r2)));
+        // All errors from replies with errors are included, not those that
+        // are fully ignored.
+        assertThatErrorsMatch(new Error[]{ errors[0], errors[1] }, ret);
+    }
+
+    @Test
+    public void returnRemoveDocumentReplyWhereDocWasFound() {
+        RemoveDocumentReply r1 = new RemoveDocumentReply();
+        RemoveDocumentReply r2 = new RemoveDocumentReply();
+        RemoveDocumentReply r3 = new RemoveDocumentReply();
+        r1.setWasFound(false);
+        r2.setWasFound(true);
+        r3.setWasFound(false);
+
+        merger.merge(0, r1);
+        merger.merge(1, r2);
+        merger.merge(2, r3);
+        Tuple2<Integer, Reply> ret = merger.mergedReply();
+        assertThat(ret.first, is(1));
+        assertThat(ret.second, sameInstance((Reply) r2));
+    }
+
+    @Test
+    public void returnFirstRemoveDocumentReplyIfNoDocsWereFound() {
+        RemoveDocumentReply r1 = new RemoveDocumentReply();
+        RemoveDocumentReply r2 = new RemoveDocumentReply();
+        r1.setWasFound(false);
+        r2.setWasFound(false);
+
+        merger.merge(0, r1);
+        merger.merge(1, r2);
+        Tuple2<Integer, Reply> ret = merger.mergedReply();
+        assertThat(ret.first, is(0));
+        assertThat(ret.second, sameInstance((Reply)r1));
+    }
+
+    @Test
+    public void returnUpdateDocumentReplyWhereDocWasFound() {
+        UpdateDocumentReply r1 = new UpdateDocumentReply();
+        UpdateDocumentReply r2 = new UpdateDocumentReply();
+        UpdateDocumentReply r3 = new UpdateDocumentReply();
+        r1.setWasFound(false);
+        r2.setWasFound(true); // return first reply
+        r3.setWasFound(true);
+
+        merger.merge(0, r1);
+        merger.merge(1, r2);
+        merger.merge(2, r3);
+        Tuple2<Integer, Reply> ret = merger.mergedReply();
+        assertThat(ret.first, is(1));
+        assertThat(ret.second, sameInstance((Reply)r2));
+    }
+
+    @Test
+    public void returnGetDocumentReplyWhereDocWasFound() {
+        GetDocumentReply r1 = new GetDocumentReply(null);
+        GetDocumentReply r2 = new GetDocumentReply(null);
+        GetDocumentReply r3 = new GetDocumentReply(null);
+        r2.setLastModified(12345L);
+
+        merger.merge(0, r1);
+        merger.merge(1, r2);
+        merger.merge(2, r3);
+        Tuple2<Integer, Reply> ret = merger.mergedReply();
+        assertThat(ret.first, is(1));
+        assertThat(ret.second, sameInstance((Reply)r2));
+    }
+
+    @Test
+    public void mergingZeroRepliesReturnsDefaultEmptyReply() {
+        Tuple2<Integer, Reply> ret = merger.mergedReply();
+        assertThat(ret.first, nullValue());
+        assertThat(ret.second, instanceOf(EmptyReply.class));
+        assertThatErrorsMatch(new Error[]{}, ret);
+    }
+
+    private void assertThatErrorsMatch(Error[] errors, Tuple2<Integer, Reply> ret) {
+        assertThat(ret.second.getNumErrors(), is(errors.length));
+        for (int i = 0; i < ret.second.getNumErrors(); ++i) {
+            assertThat(ret.second.getError(i).getCode(), is(errors[i].getCode()));
+            assertThat(ret.second.getError(i).getMessage(), is(errors[i].getMessage()));
+        }
+    }
+
+}
diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages50TestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages50TestCase.java
index 226f96f6553..9faed67ba8d 100644
--- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages50TestCase.java
+++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages50TestCase.java
@@ -1,975 +1,975 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.documentapi.messagebus.protocol.test;

-

-import com.yahoo.component.Version;

-import com.yahoo.document.*;

-import com.yahoo.document.fieldpathupdate.RemoveFieldPathUpdate;

-import com.yahoo.document.idstring.IdString;

-import com.yahoo.document.select.OrderingSpecification;

-import com.yahoo.documentapi.messagebus.protocol.*;

-import com.yahoo.messagebus.Routable;

-import com.yahoo.text.Utf8;

-import com.yahoo.vdslib.DocumentList;

-import com.yahoo.vdslib.Entry;

-import com.yahoo.vdslib.SearchResult;

-

-import java.io.FileInputStream;

-import java.io.IOException;

-import java.util.ArrayList;

-import java.util.List;

-import java.util.Map;

-

-import static org.junit.Assert.*;

-

-/**

- * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>

- */

-public class Messages50TestCase extends MessagesTestBase {

-

-    @Override

-    protected void registerTests(Map<Integer, RunnableTest> out) {

-        // This list MUST mirror the list of routable factories from the DocumentProtocol constructor that support

-        // version 5.0. When adding tests to this list, please KEEP THEM ORDERED alphabetically like they are now.

-        out.put(DocumentProtocol.MESSAGE_BATCHDOCUMENTUPDATE, new testBatchDocumentUpdateMessage());

-        out.put(DocumentProtocol.MESSAGE_CREATEVISITOR, new testCreateVisitorMessage());

-        out.put(DocumentProtocol.MESSAGE_DESTROYVISITOR, new testDestroyVisitorMessage());

-        out.put(DocumentProtocol.MESSAGE_DOCUMENTLIST, new testDocumentListMessage());

-        out.put(DocumentProtocol.MESSAGE_DOCUMENTSUMMARY, new testDocumentSummaryMessage());

-        out.put(DocumentProtocol.MESSAGE_EMPTYBUCKETS, new testEmptyBucketsMessage());

-        out.put(DocumentProtocol.MESSAGE_GETBUCKETLIST, new testGetBucketListMessage());

-        out.put(DocumentProtocol.MESSAGE_GETBUCKETSTATE, new testGetBucketStateMessage());

-        out.put(DocumentProtocol.MESSAGE_GETDOCUMENT, new testGetDocumentMessage());

-        out.put(DocumentProtocol.MESSAGE_MAPVISITOR, new testMapVisitorMessage());

-        out.put(DocumentProtocol.MESSAGE_PUTDOCUMENT, new testPutDocumentMessage());

-        out.put(DocumentProtocol.MESSAGE_QUERYRESULT, new testQueryResultMessage());

-        out.put(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, new testRemoveDocumentMessage());

-        out.put(DocumentProtocol.MESSAGE_REMOVELOCATION, new testRemoveLocationMessage());

-        out.put(DocumentProtocol.MESSAGE_SEARCHRESULT, new testSearchResultMessage());

-        out.put(DocumentProtocol.MESSAGE_STATBUCKET, new testStatBucketMessage());

-        out.put(DocumentProtocol.MESSAGE_UPDATEDOCUMENT, new testUpdateDocumentMessage());

-        out.put(DocumentProtocol.MESSAGE_VISITORINFO, new testVisitorInfoMessage());

-        out.put(DocumentProtocol.REPLY_BATCHDOCUMENTUPDATE, new testBatchDocumentUpdateReply());

-        out.put(DocumentProtocol.REPLY_CREATEVISITOR, new testCreateVisitorReply());

-        out.put(DocumentProtocol.REPLY_DESTROYVISITOR, new testDestroyVisitorReply());

-        out.put(DocumentProtocol.REPLY_DOCUMENTLIST, new testDocumentListReply());

-        out.put(DocumentProtocol.REPLY_DOCUMENTSUMMARY, new testDocumentSummaryReply());

-        out.put(DocumentProtocol.REPLY_EMPTYBUCKETS, new testEmptyBucketsReply());

-        out.put(DocumentProtocol.REPLY_GETBUCKETLIST, new testGetBucketListReply());

-        out.put(DocumentProtocol.REPLY_GETBUCKETSTATE, new testGetBucketStateReply());

-        out.put(DocumentProtocol.REPLY_GETDOCUMENT, new testGetDocumentReply());

-        out.put(DocumentProtocol.REPLY_MAPVISITOR, new testMapVisitorReply());

-        out.put(DocumentProtocol.REPLY_PUTDOCUMENT, new testPutDocumentReply());

-        out.put(DocumentProtocol.REPLY_QUERYRESULT, new testQueryResultReply());

-        out.put(DocumentProtocol.REPLY_REMOVEDOCUMENT, new testRemoveDocumentReply());

-        out.put(DocumentProtocol.REPLY_REMOVELOCATION, new testRemoveLocationReply());

-        out.put(DocumentProtocol.REPLY_SEARCHRESULT, new testSearchResultReply());

-        out.put(DocumentProtocol.REPLY_STATBUCKET, new testStatBucketReply());

-        out.put(DocumentProtocol.REPLY_UPDATEDOCUMENT, new testUpdateDocumentReply());

-        out.put(DocumentProtocol.REPLY_VISITORINFO, new testVisitorInfoReply());

-        out.put(DocumentProtocol.REPLY_WRONGDISTRIBUTION, new testWrongDistributionReply());

-    }

-

-    @Override

-    protected Version version() {

-        return new Version(5, 0);

-    }

-

-    @Override

-    protected boolean shouldTestCoverage() {

-        return false;

-    }

-

-    ////////////////////////////////////////////////////////////////////////////////

-    //

-    // Tests

-    //

-    ////////////////////////////////////////////////////////////////////////////////

-

-    private static int BASE_MESSAGE_LENGTH = 5;

-

-    public class testRemoveLocationMessage implements RunnableTest {

-

-        @Override

-        public void run() {

-            {

-                RemoveLocationMessage msg = new RemoveLocationMessage("id.group == \"mygroup\"");

-                assertEquals(BASE_MESSAGE_LENGTH + 29, serialize("RemoveLocationMessage", msg));

-

-                for (Language lang : LANGUAGES) {

-                    msg = (RemoveLocationMessage)deserialize("RemoveLocationMessage", DocumentProtocol.MESSAGE_REMOVELOCATION, lang);

-                    assertEquals("id.group == \"mygroup\"", msg.getDocumentSelection());

-                }

-            }

-        }

-    }

-

-    public class testGetBucketListMessage implements RunnableTest {

-

-        @Override

-        public void run() {

-            GetBucketListMessage msg = new GetBucketListMessage(new BucketId(16, 123));

-            msg.setLoadType(loadTypes.getNameMap().get("foo"));

-            assertEquals(BASE_MESSAGE_LENGTH + 12, serialize("GetBucketListMessage", msg));

-

-            for (Language lang : LANGUAGES) {

-                msg = (GetBucketListMessage)deserialize("GetBucketListMessage", DocumentProtocol.MESSAGE_GETBUCKETLIST, lang);

-                assertEquals(new BucketId(16, 123), msg.getBucketId());

-                assertEquals("foo", msg.getLoadType().getName());

-            }

-        }

-    }

-

-

-    public class testStatBucketMessage implements RunnableTest {

-

-        @Override

-        public void run() {

-            StatBucketMessage msg = new StatBucketMessage(new BucketId(16, 123), "id.user=123");

-            msg.setLoadType(null);

-            assertEquals(BASE_MESSAGE_LENGTH + 27, serialize("StatBucketMessage", msg));

-

-            for (Language lang : LANGUAGES) {

-                msg = (StatBucketMessage)deserialize("StatBucketMessage", DocumentProtocol.MESSAGE_STATBUCKET, lang);

-                assertEquals(new BucketId(16, 123), msg.getBucketId());

-                assertEquals("id.user=123", msg.getDocumentSelection());

-                assertEquals("default", msg.getLoadType().getName());

-            }

-        }

-    }

-

-    public class testGetBucketStateMessage implements RunnableTest {

-

-        @Override

-        public void run() {

-            GetBucketStateMessage msg = new GetBucketStateMessage(new BucketId(16, 666));

-            assertEquals(BASE_MESSAGE_LENGTH + 12, serialize("GetBucketStateMessage", msg));

-

-            for (Language lang : LANGUAGES) {

-                msg = (GetBucketStateMessage)deserialize("GetBucketStateMessage", DocumentProtocol.MESSAGE_GETBUCKETSTATE, lang);

-                assertEquals(16, msg.getBucketId().getUsedBits());

-                assertEquals(4611686018427388570l, msg.getBucketId().getId());

-            }

-        }

-    }

-

-    public class testCreateVisitorMessage implements RunnableTest {

-

-        @Override

-        @SuppressWarnings("deprecation")

-        public void run() {

-            CreateVisitorMessage msg = new CreateVisitorMessage("SomeLibrary", "myvisitor", "newyork", "london");

-            msg.setDocumentSelection("true and false or true");

-            msg.getParameters().put("myvar", Utf8.toBytes("somevalue"));

-            msg.getParameters().put("anothervar", Utf8.toBytes("34"));

-            msg.getBuckets().add(new BucketId(16, 1234));

-            msg.setVisitRemoves(true);

-            msg.setVisitorOrdering(OrderingSpecification.DESCENDING);

-            msg.setMaxBucketsPerVisitor(2);

-            assertEquals(BASE_MESSAGE_LENGTH + 168, serialize("CreateVisitorMessage", msg));

-

-            for (Language lang : LANGUAGES) {

-                msg = (CreateVisitorMessage)deserialize("CreateVisitorMessage", DocumentProtocol.MESSAGE_CREATEVISITOR, lang);

-                assertEquals("SomeLibrary", msg.getLibraryName());

-                assertEquals("myvisitor", msg.getInstanceId());

-                assertEquals("newyork", msg.getControlDestination());

-                assertEquals("london", msg.getDataDestination());

-                assertEquals("true and false or true", msg.getDocumentSelection());

-                assertEquals(8, msg.getMaxPendingReplyCount());

-                assertEquals(true, msg.getVisitRemoves());

-                assertEquals(false, msg.getVisitInconsistentBuckets());

-                assertEquals(1, msg.getBuckets().size());

-                assertEquals(new BucketId(16, 1234), msg.getBuckets().iterator().next());

-                assertEquals("somevalue", Utf8.toString(msg.getParameters().get("myvar")));

-                assertEquals("34", Utf8.toString(msg.getParameters().get("anothervar")));

-                assertEquals(OrderingSpecification.DESCENDING, msg.getVisitorOrdering());

-                assertEquals(2, msg.getMaxBucketsPerVisitor());

-            }

-

-            msg.getBuckets().clear();

-

-            assertEquals("CreateVisitorMessage(" +

-                    "No buckets, " +

-                    "selection 'true and false or true', " +

-                    "library SomeLibrary, including removes, " +

-                    "get fields: [all]" +

-                    ")",

-                    msg.toString());

-

-            msg.getBuckets().add(new BucketId(16, 1234));

-

-            assertEquals("CreateVisitorMessage(" +

-                    "Bucket BucketId(0x40000000000004d2), " +

-                    "selection 'true and false or true', " +

-                    "library SomeLibrary, including removes, " +

-                    "get fields: [all]" +

-                    ")",

-                    msg.toString());

-

-            msg.getBuckets().add(new BucketId(16, 1235));

-            msg.getBuckets().add(new BucketId(16, 1236));

-            msg.getBuckets().add(new BucketId(16, 1237));

-            msg.getBuckets().add(new BucketId(16, 1238));

-            msg.setFromTimestamp(10001);

-            msg.setToTimestamp(20002);

-            msg.setVisitInconsistentBuckets(true);

-            assertEquals("CreateVisitorMessage(" +

-                    "5 buckets: BucketId(0x40000000000004d2) BucketId(0x40000000000004d3) BucketId(0x40000000000004d4) ..., " +

-                    "time 10001-20002, " +

-                    "selection 'true and false or true', " +

-                    "library SomeLibrary, including removes, " +

-                    "get fields: [all], " +

-                    "visit inconsistent buckets" +

-                    ")",

-                    msg.toString());

-        }

-    }

-

-    public class testCreateVisitorReply implements RunnableTest {

-

-        @Override

-        public void run() {

-            CreateVisitorReply reply = new CreateVisitorReply(DocumentProtocol.REPLY_CREATEVISITOR);

-            reply.setLastBucket(new BucketId(16, 123));

-            reply.getVisitorStatistics().setBucketsVisited(3);

-            reply.getVisitorStatistics().setDocumentsVisited(1000);

-            reply.getVisitorStatistics().setBytesVisited(1024000);

-            reply.getVisitorStatistics().setDocumentsReturned(123);

-            reply.getVisitorStatistics().setBytesReturned(512000);

-            reply.getVisitorStatistics().setSecondPassDocumentsReturned(456);

-            reply.getVisitorStatistics().setSecondPassBytesReturned(789100);

-

-            assertEquals(65, serialize("CreateVisitorReply", reply));

-

-            for (Language lang : LANGUAGES) {

-                reply = (CreateVisitorReply)deserialize("CreateVisitorReply", DocumentProtocol.REPLY_CREATEVISITOR, lang);

-                assertNotNull(reply);

-                assertEquals(new BucketId(16, 123), reply.getLastBucket());

-                assertEquals(3, reply.getVisitorStatistics().getBucketsVisited());

-                assertEquals(1000, reply.getVisitorStatistics().getDocumentsVisited());

-                assertEquals(1024000, reply.getVisitorStatistics().getBytesVisited());

-                assertEquals(123, reply.getVisitorStatistics().getDocumentsReturned());

-                assertEquals(512000, reply.getVisitorStatistics().getBytesReturned());

-                assertEquals(456, reply.getVisitorStatistics().getSecondPassDocumentsReturned());

-                assertEquals(789100, reply.getVisitorStatistics().getSecondPassBytesReturned());

-            }

-        }

-    }

-

-    public class testDestroyVisitorReply implements RunnableTest {

-

-        @Override

-        public void run() {

-            testVisitorReply("DestroyVisitorReply", DocumentProtocol.REPLY_DESTROYVISITOR);

-        }

-    }

-

-    public class testDocumentListReply implements RunnableTest {

-

-        @Override

-        public void run() {

-            testVisitorReply("DocumentListReply", DocumentProtocol.REPLY_DOCUMENTLIST);

-        }

-    }

-

-    public class testDocumentSummaryReply implements RunnableTest {

-

-        @Override

-        public void run() {

-            testVisitorReply("DocumentSummaryReply", DocumentProtocol.REPLY_DOCUMENTSUMMARY);

-        }

-    }

-

-    public class testEmptyBucketsReply implements RunnableTest {

-

-        @Override

-        public void run() {

-            testVisitorReply("EmptyBucketsReply", DocumentProtocol.REPLY_EMPTYBUCKETS);

-        }

-    }

-

-    public class testDestroyVisitorMessage implements RunnableTest {

-

-        @Override

-        public void run() {

-            DestroyVisitorMessage msg = new DestroyVisitorMessage("myvisitor");

-            assertEquals(BASE_MESSAGE_LENGTH + 17, serialize("DestroyVisitorMessage", msg));

-

-            for (Language lang : LANGUAGES) {

-                msg = (DestroyVisitorMessage)deserialize("DestroyVisitorMessage", DocumentProtocol.MESSAGE_DESTROYVISITOR, lang);

-                assertEquals("myvisitor", msg.getInstanceId());

-            }

-        }

-    }

-

-    public class testDocumentListMessage implements RunnableTest {

-

-        @Override

-        public void run() {

-            DocumentListMessage msg = (DocumentListMessage)deserialize("DocumentListMessage", DocumentProtocol.MESSAGE_DOCUMENTLIST, Language.CPP);

-            assertEquals("userdoc:scheme:1234:", msg.getDocuments().get(0).getDocument().getId().toString());

-            assertEquals(1234, msg.getDocuments().get(0).getTimestamp());

-            assertFalse(msg.getDocuments().get(0).isRemoveEntry());

-

-            assertEquals(BASE_MESSAGE_LENGTH + 63, serialize("DocumentListMessage", msg));

-            msg = (DocumentListMessage)deserialize("DocumentListMessage", DocumentProtocol.MESSAGE_DOCUMENTLIST, Language.JAVA);

-            assertEquals("userdoc:scheme:1234:", msg.getDocuments().get(0).getDocument().getId().toString());

-            assertEquals(1234, msg.getDocuments().get(0).getTimestamp());

-            assertFalse(msg.getDocuments().get(0).isRemoveEntry());

-

-        }

-    }

-

-    public class testEmptyBucketsMessage implements RunnableTest {

-

-        @Override

-        public void run() {

-            List<BucketId> bids = new ArrayList<>();

-            for (int i = 0; i < 13; ++i) {

-                bids.add(new BucketId(16, i));

-            }

-

-            EmptyBucketsMessage ebm = new EmptyBucketsMessage(bids);

-            assertEquals(BASE_MESSAGE_LENGTH + 112, serialize("EmptyBucketsMessage", ebm));

-            for (Language lang : LANGUAGES) {

-                ebm = (EmptyBucketsMessage)deserialize("EmptyBucketsMessage", DocumentProtocol.MESSAGE_EMPTYBUCKETS, lang);

-                for (int i = 0; i < 13; ++i) {

-                    assertEquals(new BucketId(16, i), ebm.getBucketIds().get(i));

-                }

-            }

-        }

-    }

-

-    public class testDocumentSummaryMessage implements RunnableTest {

-

-        @Override

-        public void run() {

-            try {

-                FileInputStream stream = new FileInputStream(getPath("5-cpp-DocumentSummaryMessage-1.dat"));

-                byte[] data = new byte[stream.available()];

-                assertEquals(data.length, stream.read(data));

-

-                Routable routable = decode(data);

-                assertTrue(routable instanceof DocumentSummaryMessage);

-

-                DocumentSummaryMessage msg = (DocumentSummaryMessage)routable;

-                assertEquals(0, msg.getResult().getSummaryCount());

-

-                stream = new FileInputStream(getPath("5-cpp-DocumentSummaryMessage-2.dat"));

-                data = new byte[stream.available()];

-                assertEquals(data.length, stream.read(data));

-

-                routable = decode(data);

-                assertTrue(routable instanceof DocumentSummaryMessage);

-

-                msg = (DocumentSummaryMessage)routable;

-                assertEquals(2, msg.getResult().getSummaryCount());

-                com.yahoo.vdslib.DocumentSummary.Summary s = msg.getResult().getSummary(0);

-                assertEquals("doc1", s.getDocId());

-                byte[] b = s.getSummary();

-                assertEquals(8, b.length);

-                byte[] c = { 's', 'u', 'm', 'm', 'a', 'r', 'y', '1' };

-                for (int i = 0; i < b.length; i++) {

-                    assertEquals(c[i], b[i]);

-                }

-

-                s = msg.getResult().getSummary(1);

-                assertEquals("aoc17", s.getDocId());

-                b = s.getSummary();

-                assertEquals(9, b.length);

-                byte[] d = { 's', 'u', 'm', 'm', 'a', 'r', 'y', '4', '5' };

-                for (int i = 0; i < b.length; i++) {

-                    assertEquals(d[i], b[i]);

-                }

-

-                stream = new FileInputStream(getPath("5-cpp-DocumentSummaryMessage-3.dat"));

-                data = new byte[stream.available()];

-                assertEquals(data.length, stream.read(data));

-

-                routable = decode(data);

-                assertTrue(routable instanceof DocumentSummaryMessage);

-

-                msg = (DocumentSummaryMessage)routable;

-                assertEquals(2, msg.getResult().getSummaryCount());

-

-                s = msg.getResult().getSummary(0);

-                assertEquals("aoc17", s.getDocId());

-                b = s.getSummary();

-                assertEquals(9, b.length);

-                byte[] e = { 's', 'u', 'm', 'm', 'a', 'r', 'y', '4', '5' };

-                for (int i = 0; i < b.length; i++) {

-                    assertEquals(e[i], b[i]);

-                }

-

-                s = msg.getResult().getSummary(1);

-                assertEquals("doc1", s.getDocId());

-                b = s.getSummary();

-                assertEquals(8, b.length);

-                byte[] f = { 's', 'u', 'm', 'm', 'a', 'r', 'y', '1' };

-                for (int i = 0; i < b.length; i++) {

-                    assertEquals(f[i], b[i]);

-                }

-            } catch (IOException e) {

-                fail(e.toString());

-            }

-        }

-    }

-

-

-    public class testGetDocumentMessage implements RunnableTest {

-

-        @Override

-        @SuppressWarnings("deprecation")

-        public void run() {

-            GetDocumentMessage msg = new GetDocumentMessage(new DocumentId("doc:scheme:"));

-            assertEquals(BASE_MESSAGE_LENGTH + 20, serialize("GetDocumentMessage", msg));

-

-            for (Language lang : LANGUAGES) {

-                msg = (GetDocumentMessage)deserialize("GetDocumentMessage", DocumentProtocol.MESSAGE_GETDOCUMENT, lang);

-                assertEquals("doc:scheme:", msg.getDocumentId().toString());

-            }

-        }

-    }

-

-

-    public class testRemoveDocumentMessage implements RunnableTest {

-

-        @Override

-        public void run() {

-            RemoveDocumentMessage msg = new RemoveDocumentMessage(new DocumentId("doc:scheme:"));

-            assertEquals(BASE_MESSAGE_LENGTH + 16, serialize("RemoveDocumentMessage", msg));

-

-            for (Language lang : LANGUAGES) {

-                msg = (RemoveDocumentMessage)deserialize("RemoveDocumentMessage", DocumentProtocol.MESSAGE_REMOVEDOCUMENT, lang);

-                assertEquals("doc:scheme:", msg.getDocumentId().toString());

-            }

-        }

-    }

-

-    public class testMapVisitorMessage implements RunnableTest {

-

-        @Override

-        public void run() {

-            MapVisitorMessage msg = (MapVisitorMessage)deserialize("MapVisitorMessage", DocumentProtocol.MESSAGE_MAPVISITOR, Language.CPP);

-            assertEquals("3", msg.getData().get("foo"));

-            assertEquals("5", msg.getData().get("bar"));

-

-            assertEquals(BASE_MESSAGE_LENGTH + 32, serialize("MapVisitorMessage", msg));

-

-            msg = (MapVisitorMessage)deserialize("MapVisitorMessage", DocumentProtocol.MESSAGE_MAPVISITOR, Language.JAVA);

-            assertEquals("3", msg.getData().get("foo"));

-            assertEquals("5", msg.getData().get("bar"));

-        }

-    }

-

-

-    public class testVisitorInfoMessage implements RunnableTest {

-

-        @Override

-        public void run() {

-            VisitorInfoMessage msg = new VisitorInfoMessage();

-            msg.getFinishedBuckets().add(new BucketId(16, 1));

-            msg.getFinishedBuckets().add(new BucketId(16, 2));

-            msg.getFinishedBuckets().add(new BucketId(16, 4));

-            msg.setErrorMessage("error message: \u00e6\u00c6\u00f8\u00d8\u00e5\u00c5\u00f6\u00d6");

-            assertEquals(BASE_MESSAGE_LENGTH + 67, serialize("VisitorInfoMessage", msg));

-

-            for (Language lang : LANGUAGES) {

-                msg = (VisitorInfoMessage)deserialize("VisitorInfoMessage", DocumentProtocol.MESSAGE_VISITORINFO, lang);

-                assertTrue(msg.getFinishedBuckets().contains(new BucketId(16, 1)));

-                assertTrue(msg.getFinishedBuckets().contains(new BucketId(16, 2)));

-                assertTrue(msg.getFinishedBuckets().contains(new BucketId(16, 4)));

-                assertEquals("error message: \u00e6\u00c6\u00f8\u00d8\u00e5\u00c5\u00f6\u00d6", msg.getErrorMessage());

-            }

-        }

-    }

-

-    public class testSearchResultMessage implements RunnableTest {

-

-        @Override

-        public void run() throws Exception {

-            FileInputStream stream = new FileInputStream(getPath("5-cpp-SearchResultMessage-1.dat"));

-            byte[] data = new byte[stream.available()];

-            assertEquals(data.length, stream.read(data));

-

-            Routable routable = decode(data);

-            assertTrue(routable instanceof SearchResultMessage);

-

-            SearchResultMessage msg = (SearchResultMessage)routable;

-            assertEquals(0, msg.getResult().getHitCount());

-

-            stream = new FileInputStream(getPath("5-cpp-SearchResultMessage-2.dat"));

-            data = new byte[stream.available()];

-            assertEquals(data.length, stream.read(data));

-

-            routable = decode(data);

-            assertTrue(routable instanceof SearchResultMessage);

-

-            msg = (SearchResultMessage)routable;

-            assertEquals(2, msg.getResult().getHitCount());

-            com.yahoo.vdslib.SearchResult.Hit h = msg.getResult().getHit(0);

-            assertEquals(89.0, h.getRank(), 1E-6);

-            assertEquals("doc1", h.getDocId());

-            h = msg.getResult().getHit(1);

-            assertEquals(109.0, h.getRank(), 1E-6);

-            assertEquals("doc17", h.getDocId());

-

-            stream = new FileInputStream(getPath("5-cpp-SearchResultMessage-3.dat"));

-            data = new byte[stream.available()];

-            assertEquals(data.length, stream.read(data));

-

-            routable = decode(data);

-            assertTrue(routable instanceof SearchResultMessage);

-

-            msg = (SearchResultMessage)routable;

-            assertEquals(2, msg.getResult().getHitCount());

-            h = msg.getResult().getHit(0);

-            assertEquals(109.0, h.getRank(), 1E-6);

-            assertEquals("doc17", h.getDocId());

-            h = msg.getResult().getHit(1);

-            assertEquals(89.0, h.getRank(), 1E-6);

-            assertEquals("doc1", h.getDocId());

-

-            stream = new FileInputStream(getPath("5-cpp-SearchResultMessage-4.dat"));

-            data = new byte[stream.available()];

-            assertEquals(data.length, stream.read(data));

-

-            routable = decode(data);

-            assertTrue(routable instanceof SearchResultMessage);

-

-            msg = (SearchResultMessage)routable;

-            assertEquals(3, msg.getResult().getHitCount());

-            h = msg.getResult().getHit(0);

-            assertTrue(h instanceof SearchResult.HitWithSortBlob);

-            assertEquals(89.0, h.getRank(), 1E-6);

-            assertEquals("doc1", h.getDocId());

-            byte[] b = ((SearchResult.HitWithSortBlob)h).getSortBlob();

-            assertEquals(9, b.length);

-            byte[] e = { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '2' };

-            for (int i = 0; i < b.length; i++) {

-                assertEquals(e[i], b[i]);

-            }

-            h = msg.getResult().getHit(1);

-            assertTrue(h instanceof SearchResult.HitWithSortBlob);

-            assertEquals(109.0, h.getRank(), 1E-6);

-            assertEquals("doc17", h.getDocId());

-            b = ((SearchResult.HitWithSortBlob)h).getSortBlob();

-            assertEquals(9, b.length);

-            byte[] d = { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '1' };

-            for (int i = 0; i < b.length; i++) {

-                assertEquals(d[i], b[i]);

-            }

-            h = msg.getResult().getHit(2);

-            assertTrue(h instanceof SearchResult.HitWithSortBlob);

-            assertEquals(90.0, h.getRank(), 1E-6);

-            assertEquals("doc18", h.getDocId());

-            b = ((SearchResult.HitWithSortBlob)h).getSortBlob();

-            assertEquals(9, b.length);

-            byte[] c = { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '3' };

-            for (int i = 0; i < b.length; i++) {

-                assertEquals(c[i], b[i]);

-            }

-        }

-    }

-

-    public class testPutDocumentMessage implements RunnableTest {

-

-        @Override

-        public void run() {

-            PutDocumentMessage msg = new PutDocumentMessage(new DocumentPut(new Document(protocol.getDocumentTypeManager().getDocumentType("testdoc"), "doc:scheme:")));

-            msg.setTimestamp(666);

-            assertEquals(BASE_MESSAGE_LENGTH + 41, serialize("PutDocumentMessage", msg));

-

-            for (Language lang : LANGUAGES) {

-                msg = (PutDocumentMessage)deserialize("PutDocumentMessage", DocumentProtocol.MESSAGE_PUTDOCUMENT, lang);

-                assertEquals("testdoc", msg.getDocumentPut().getDocument().getDataType().getName());

-                assertEquals("doc:scheme:", msg.getDocumentPut().getDocument().getId().toString());

-                assertEquals(666, msg.getTimestamp());

-            }

-        }

-    }

-

-    public class testPutDocumentReply implements RunnableTest {

-

-        @Override

-        public void run() {

-            WriteDocumentReply reply = new WriteDocumentReply(DocumentProtocol.REPLY_PUTDOCUMENT);

-            reply.setHighestModificationTimestamp(30);

-

-            assertEquals(13, serialize("PutDocumentReply", reply));

-

-            for (Language lang : LANGUAGES) {

-                WriteDocumentReply obj = (WriteDocumentReply)deserialize("PutDocumentReply", DocumentProtocol.REPLY_PUTDOCUMENT, lang);

-                assertNotNull(obj);

-                assertEquals(30, obj.getHighestModificationTimestamp());

-            }

-        }

-    }

-

-    public class testUpdateDocumentMessage implements RunnableTest {

-

-        @Override

-        public void run() {

-            DocumentType docType = protocol.getDocumentTypeManager().getDocumentType("testdoc");

-            DocumentUpdate update = new DocumentUpdate(docType, new DocumentId("doc:scheme:"));

-            update.addFieldPathUpdate(new RemoveFieldPathUpdate(docType, "intfield", "testdoc.intfield > 0"));

-            UpdateDocumentMessage msg = new UpdateDocumentMessage(update);

-            msg.setNewTimestamp(777);

-            msg.setOldTimestamp(666);

-

-            assertEquals(BASE_MESSAGE_LENGTH + 89, serialize("UpdateDocumentMessage", msg));

-

-            for (Language lang : LANGUAGES) {

-                msg = (UpdateDocumentMessage)deserialize("UpdateDocumentMessage", DocumentProtocol.MESSAGE_UPDATEDOCUMENT, lang);

-                assertEquals(update, msg.getDocumentUpdate());

-                assertEquals(777, msg.getNewTimestamp());

-                assertEquals(666, msg.getOldTimestamp());

-            }

-        }

-    }

-

-    public class testUpdateDocumentReply implements RunnableTest {

-

-        @Override

-        public void run() {

-            UpdateDocumentReply reply = new UpdateDocumentReply();

-            reply.setHighestModificationTimestamp(30);

-            reply.setWasFound(false);

-

-            assertEquals(14, serialize("UpdateDocumentReply", reply));

-

-            for (Language lang : LANGUAGES) {

-                UpdateDocumentReply obj = (UpdateDocumentReply)deserialize("UpdateDocumentReply", DocumentProtocol.REPLY_UPDATEDOCUMENT, lang);

-                assertNotNull(obj);

-                assertEquals(30, reply.getHighestModificationTimestamp());

-                assertEquals(false, obj.wasFound());

-            }

-        }

-    }

-

-    public class testVisitorInfoReply implements RunnableTest {

-

-        @Override

-        public void run() {

-            testVisitorReply("VisitorInfoReply", DocumentProtocol.REPLY_VISITORINFO);

-        }

-    }

-

-    public class testWrongDistributionReply implements RunnableTest {

-

-        @Override

-        public void run() {

-            WrongDistributionReply reply = new WrongDistributionReply("distributor:3 storage:2");

-            assertEquals(32, serialize("WrongDistributionReply", reply));

-

-            for (Language lang : LANGUAGES) {

-                reply = (WrongDistributionReply)deserialize("WrongDistributionReply", DocumentProtocol.REPLY_WRONGDISTRIBUTION, lang);

-                assertEquals("distributor:3 storage:2", reply.getSystemState());

-            }

-        }

-    }

-

-    public class testRemoveDocumentReply implements RunnableTest {

-

-        @Override

-        public void run() {

-            RemoveDocumentReply reply = new RemoveDocumentReply();

-            reply.setHighestModificationTimestamp(30);

-            reply.setWasFound(false);

-

-            assertEquals(14, serialize("RemoveDocumentReply", reply));

-

-            for (Language lang : LANGUAGES) {

-                RemoveDocumentReply obj = (RemoveDocumentReply)deserialize("RemoveDocumentReply", DocumentProtocol.REPLY_REMOVEDOCUMENT, lang);

-                assertNotNull(obj);

-                assertEquals(30, obj.getHighestModificationTimestamp());

-                assertEquals(false, obj.wasFound());

-            }

-        }

-    }

-

-    public class testRemoveLocationReply implements RunnableTest {

-

-        @Override

-        public void run() {

-            testDocumentReply("RemoveLocationReply", DocumentProtocol.REPLY_REMOVELOCATION);

-        }

-    }

-

-    public class testSearchResultReply implements RunnableTest {

-

-        @Override

-        public void run() {

-            testVisitorReply("SearchResultReply", DocumentProtocol.REPLY_SEARCHRESULT);

-        }

-    }

-

-    public class testStatBucketReply implements RunnableTest {

-

-        @Override

-        public void run() {

-            StatBucketReply msg = new StatBucketReply();

-            msg.setResults("These are the votes of the Norwegian jury");

-

-            assertEquals(50, serialize("StatBucketReply", msg));

-

-            for (Language lang : LANGUAGES) {

-                msg = (StatBucketReply)deserialize("StatBucketReply", DocumentProtocol.REPLY_STATBUCKET, lang);

-                assertEquals("These are the votes of the Norwegian jury", msg.getResults());

-            }

-        }

-    }

-

-    public class testBatchDocumentUpdateMessage implements RunnableTest {

-

-        @Override

-        public void run() {

-            DocumentType docType = protocol.getDocumentTypeManager().getDocumentType("testdoc");

-            BatchDocumentUpdateMessage msg = new BatchDocumentUpdateMessage(1234);

-

-            {

-                DocumentUpdate update = new DocumentUpdate(docType, new DocumentId("userdoc:footype:1234:foo"));

-                update.addFieldPathUpdate(new RemoveFieldPathUpdate(docType, "intfield", "testdoc.intfield > 0"));

-                msg.addUpdate(update);

-            }

-            {

-                DocumentUpdate update = new DocumentUpdate(docType, new DocumentId("orderdoc(32,17):footype:1234:123456789:foo"));

-                update.addFieldPathUpdate(new RemoveFieldPathUpdate(docType, "intfield", "testdoc.intfield > 0"));

-                msg.addUpdate(update);

-            }

-

-            try {

-                DocumentUpdate update = new DocumentUpdate(docType, new DocumentId("orderdoc:footype:5678:foo"));

-                update.addFieldPathUpdate(new RemoveFieldPathUpdate(docType, "intfield", "testdoc.intfield > 0"));

-                msg.addUpdate(update);

-                fail();

-            } catch (Exception e) {

-

-            }

-

-            try {

-                DocumentUpdate update = new DocumentUpdate(docType, new DocumentId("groupdoc:footype:hable:foo"));

-                update.addFieldPathUpdate(new RemoveFieldPathUpdate(docType, "intfield", "testdoc.intfield > 0"));

-                msg.addUpdate(update);

-                fail();

-            } catch (Exception e) {

-

-            }

-

-            assertEquals(2, msg.getUpdates().size());

-

-            assertEquals(BASE_MESSAGE_LENGTH + 202, serialize("BatchDocumentUpdateMessage", msg));

-

-            for (Language lang : LANGUAGES) {

-                msg = (BatchDocumentUpdateMessage)deserialize("BatchDocumentUpdateMessage", DocumentProtocol.MESSAGE_BATCHDOCUMENTUPDATE, lang);

-                assertEquals(2, msg.getUpdates().size());

-            }

-        }

-    }

-

-    public class testBatchDocumentUpdateReply implements RunnableTest {

-

-        @Override

-        public void run() {

-            BatchDocumentUpdateReply reply = new BatchDocumentUpdateReply();

-            reply.setHighestModificationTimestamp(30);

-            reply.getDocumentsNotFound().add(false);

-            reply.getDocumentsNotFound().add(true);

-            reply.getDocumentsNotFound().add(true);

-

-            assertEquals(20, serialize("BatchDocumentUpdateReply", reply));

-

-            for (Language lang : LANGUAGES) {

-                BatchDocumentUpdateReply obj = (BatchDocumentUpdateReply)deserialize("BatchDocumentUpdateReply", DocumentProtocol.REPLY_BATCHDOCUMENTUPDATE, lang);

-                assertNotNull(obj);

-                assertEquals(30, obj.getHighestModificationTimestamp());

-                assertEquals(3, obj.getDocumentsNotFound().size());

-                assertFalse(obj.getDocumentsNotFound().get(0));

-                assertTrue(obj.getDocumentsNotFound().get(1));

-                assertTrue(obj.getDocumentsNotFound().get(2));

-            }

-        }

-    }

-

-

-    public class testQueryResultReply implements RunnableTest {

-

-        @Override

-        public void run() {

-            testVisitorReply("QueryResultReply", DocumentProtocol.REPLY_QUERYRESULT);

-        }

-    }

-

-    public class testQueryResultMessage implements RunnableTest {

-

-        @Override

-        public void run() throws Exception {

-            FileInputStream stream = new FileInputStream(getPath("5-cpp-QueryResultMessage-1.dat"));

-            byte[] data = new byte[stream.available()];

-            assertEquals(data.length, stream.read(data));

-

-            Routable routable = decode(data);

-            assertTrue(routable instanceof QueryResultMessage);

-

-            QueryResultMessage msg = (QueryResultMessage)routable;

-            assertEquals(0, msg.getResult().getHitCount());

-

-            stream = new FileInputStream(getPath("5-cpp-QueryResultMessage-2.dat"));

-            data = new byte[stream.available()];

-            assertEquals(data.length, stream.read(data));

-

-            routable = decode(data);

-            assertTrue(routable instanceof QueryResultMessage);

-

-            msg = (QueryResultMessage)routable;

-            assertEquals(2, msg.getResult().getHitCount());

-            com.yahoo.vdslib.SearchResult.Hit h = msg.getResult().getHit(0);

-            assertEquals(89.0, h.getRank(), 1E-6);

-            assertEquals("doc1", h.getDocId());

-            h = msg.getResult().getHit(1);

-            assertEquals(109.0, h.getRank(), 1E-6);

-            assertEquals("doc17", h.getDocId());

-

-            stream = new FileInputStream(getPath("5-cpp-QueryResultMessage-3.dat"));

-            data = new byte[stream.available()];

-            assertEquals(data.length, stream.read(data));

-

-            routable = decode(data);

-            assertTrue(routable instanceof QueryResultMessage);

-

-            msg = (QueryResultMessage)routable;

-            assertEquals(2, msg.getResult().getHitCount());

-            h = msg.getResult().getHit(0);

-            assertEquals(109.0, h.getRank(), 1E-6);

-            assertEquals("doc17", h.getDocId());

-            h = msg.getResult().getHit(1);

-            assertEquals(89.0, h.getRank(), 1E-6);

-            assertEquals("doc1", h.getDocId());

-

-            stream = new FileInputStream(getPath("5-cpp-QueryResultMessage-4.dat"));

-            data = new byte[stream.available()];

-            assertEquals(data.length, stream.read(data));

-

-            routable = decode(data);

-            assertTrue(routable instanceof QueryResultMessage);

-

-            msg = (QueryResultMessage)routable;

-            assertEquals(3, msg.getResult().getHitCount());

-            h = msg.getResult().getHit(0);

-            assertTrue(h instanceof SearchResult.HitWithSortBlob);

-            assertEquals(89.0, h.getRank(), 1E-6);

-            assertEquals("doc1", h.getDocId());

-            byte[] b = ((SearchResult.HitWithSortBlob)h).getSortBlob();

-            assertEquals(9, b.length);

-            byte[] e = { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '2' };

-            for (int i = 0; i < b.length; i++) {

-                assertEquals(e[i], b[i]);

-            }

-            h = msg.getResult().getHit(1);

-            assertTrue(h instanceof SearchResult.HitWithSortBlob);

-            assertEquals(109.0, h.getRank(), 1E-6);

-            assertEquals("doc17", h.getDocId());

-            b = ((SearchResult.HitWithSortBlob)h).getSortBlob();

-            assertEquals(9, b.length);

-            byte[] d = { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '1' };

-            for (int i = 0; i < b.length; i++) {

-                assertEquals(d[i], b[i]);

-            }

-            h = msg.getResult().getHit(2);

-            assertTrue(h instanceof SearchResult.HitWithSortBlob);

-            assertEquals(90.0, h.getRank(), 1E-6);

-            assertEquals("doc18", h.getDocId());

-            b = ((SearchResult.HitWithSortBlob)h).getSortBlob();

-            assertEquals(9, b.length);

-            byte[] c = { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '3' };

-            for (int i = 0; i < b.length; i++) {

-                assertEquals(c[i], b[i]);

-            }

-        }

-    }

-

-    public class testGetBucketListReply implements RunnableTest {

-

-        public void run() {

-            GetBucketListReply reply = new GetBucketListReply();

-            reply.getBuckets().add(new GetBucketListReply.BucketInfo(new BucketId(16, 123), "foo"));

-            reply.getBuckets().add(new GetBucketListReply.BucketInfo(new BucketId(17, 1123), "bar"));

-            reply.getBuckets().add(new GetBucketListReply.BucketInfo(new BucketId(18, 11123), "zoink"));

-

-            assertEquals(56, serialize("GetBucketListReply", reply));

-

-            for (Language lang : LANGUAGES) {

-                reply = (GetBucketListReply)deserialize("GetBucketListReply", DocumentProtocol.REPLY_GETBUCKETLIST, lang);

-                assertEquals(reply.getBuckets().get(0), new GetBucketListReply.BucketInfo(new BucketId(16, 123), "foo"));

-                assertEquals(reply.getBuckets().get(1), new GetBucketListReply.BucketInfo(new BucketId(17, 1123), "bar"));

-                assertEquals(reply.getBuckets().get(2), new GetBucketListReply.BucketInfo(new BucketId(18, 11123), "zoink"));

-            }

-        }

-    }

-

-    public class testGetBucketStateReply implements RunnableTest {

-

-        public void run() {

-            GlobalId foo = new GlobalId(IdString.createIdString("doc:scheme:foo"));

-            GlobalId bar = new GlobalId(IdString.createIdString("doc:scheme:bar"));

-

-            GetBucketStateReply reply = new GetBucketStateReply();

-            List<DocumentState> state = new ArrayList<>(2);

-            state.add(new DocumentState(foo, 777, false));

-            state.add(new DocumentState(bar, 888, true));

-            reply.setBucketState(state);

-            assertEquals(53, serialize("GetBucketStateReply", reply));

-

-            for (Language lang : LANGUAGES) {

-                reply = (GetBucketStateReply)deserialize("GetBucketStateReply", DocumentProtocol.REPLY_GETBUCKETSTATE, lang);

-                assertEquals(777, reply.getBucketState().get(0).getTimestamp());

-                assertEquals(foo, reply.getBucketState().get(0).getGid());

-                assertEquals(false, reply.getBucketState().get(0).isRemoveEntry());

-                assertEquals(888, reply.getBucketState().get(1).getTimestamp());

-                assertEquals(bar, reply.getBucketState().get(1).getGid());

-                assertEquals(true, reply.getBucketState().get(1).isRemoveEntry());

-            }

-        }

-    }

-

-    public class testGetDocumentReply implements RunnableTest {

-

-        public void run() {

-            GetDocumentReply reply = new GetDocumentReply(new Document(protocol.getDocumentTypeManager().getDocumentType("testdoc"), "doc:scheme:"));

-            assertEquals(43, serialize("GetDocumentReply", reply));

-

-            for (Language lang : LANGUAGES) {

-                reply = (GetDocumentReply)deserialize("GetDocumentReply", DocumentProtocol.REPLY_GETDOCUMENT, lang);

-                assertEquals("testdoc", reply.getDocument().getDataType().getName());

-                assertEquals("doc:scheme:", reply.getDocument().getId().toString());

-            }

-        }

-    }

-

-    public class testMapVisitorReply implements RunnableTest {

-

-        public void run() {

-            testVisitorReply("MapVisitorReply", DocumentProtocol.REPLY_MAPVISITOR);

-        }

-    }

-

-    protected void testDocumentReply(String filename, int type) {

-        DocumentReply reply = new DocumentReply(type);

-        assertEquals(5, serialize(filename, reply));

-

-        for (Language lang : LANGUAGES) {

-            reply = (DocumentReply)deserialize(filename, type, lang);

-            assertNotNull(reply);

-        }

-    }

-

-    protected void testVisitorReply(String filename, int type) {

-        VisitorReply reply = new VisitorReply(type);

-        assertEquals(5, serialize(filename, reply));

-

-        for (Language lang : LANGUAGES) {

-            reply = (VisitorReply)deserialize(filename, type, lang);

-            assertNotNull(reply);

-        }

-    }

-

-}

+package com.yahoo.documentapi.messagebus.protocol.test;
+
+import com.yahoo.component.Version;
+import com.yahoo.document.*;
+import com.yahoo.document.fieldpathupdate.RemoveFieldPathUpdate;
+import com.yahoo.document.idstring.IdString;
+import com.yahoo.document.select.OrderingSpecification;
+import com.yahoo.documentapi.messagebus.protocol.*;
+import com.yahoo.messagebus.Routable;
+import com.yahoo.text.Utf8;
+import com.yahoo.vdslib.DocumentList;
+import com.yahoo.vdslib.Entry;
+import com.yahoo.vdslib.SearchResult;
+
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.Assert.*;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class Messages50TestCase extends MessagesTestBase {
+
+    @Override
+    protected void registerTests(Map<Integer, RunnableTest> out) {
+        // This list MUST mirror the list of routable factories from the DocumentProtocol constructor that support
+        // version 5.0. When adding tests to this list, please KEEP THEM ORDERED alphabetically like they are now.
+        out.put(DocumentProtocol.MESSAGE_BATCHDOCUMENTUPDATE, new testBatchDocumentUpdateMessage());
+        out.put(DocumentProtocol.MESSAGE_CREATEVISITOR, new testCreateVisitorMessage());
+        out.put(DocumentProtocol.MESSAGE_DESTROYVISITOR, new testDestroyVisitorMessage());
+        out.put(DocumentProtocol.MESSAGE_DOCUMENTLIST, new testDocumentListMessage());
+        out.put(DocumentProtocol.MESSAGE_DOCUMENTSUMMARY, new testDocumentSummaryMessage());
+        out.put(DocumentProtocol.MESSAGE_EMPTYBUCKETS, new testEmptyBucketsMessage());
+        out.put(DocumentProtocol.MESSAGE_GETBUCKETLIST, new testGetBucketListMessage());
+        out.put(DocumentProtocol.MESSAGE_GETBUCKETSTATE, new testGetBucketStateMessage());
+        out.put(DocumentProtocol.MESSAGE_GETDOCUMENT, new testGetDocumentMessage());
+        out.put(DocumentProtocol.MESSAGE_MAPVISITOR, new testMapVisitorMessage());
+        out.put(DocumentProtocol.MESSAGE_PUTDOCUMENT, new testPutDocumentMessage());
+        out.put(DocumentProtocol.MESSAGE_QUERYRESULT, new testQueryResultMessage());
+        out.put(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, new testRemoveDocumentMessage());
+        out.put(DocumentProtocol.MESSAGE_REMOVELOCATION, new testRemoveLocationMessage());
+        out.put(DocumentProtocol.MESSAGE_SEARCHRESULT, new testSearchResultMessage());
+        out.put(DocumentProtocol.MESSAGE_STATBUCKET, new testStatBucketMessage());
+        out.put(DocumentProtocol.MESSAGE_UPDATEDOCUMENT, new testUpdateDocumentMessage());
+        out.put(DocumentProtocol.MESSAGE_VISITORINFO, new testVisitorInfoMessage());
+        out.put(DocumentProtocol.REPLY_BATCHDOCUMENTUPDATE, new testBatchDocumentUpdateReply());
+        out.put(DocumentProtocol.REPLY_CREATEVISITOR, new testCreateVisitorReply());
+        out.put(DocumentProtocol.REPLY_DESTROYVISITOR, new testDestroyVisitorReply());
+        out.put(DocumentProtocol.REPLY_DOCUMENTLIST, new testDocumentListReply());
+        out.put(DocumentProtocol.REPLY_DOCUMENTSUMMARY, new testDocumentSummaryReply());
+        out.put(DocumentProtocol.REPLY_EMPTYBUCKETS, new testEmptyBucketsReply());
+        out.put(DocumentProtocol.REPLY_GETBUCKETLIST, new testGetBucketListReply());
+        out.put(DocumentProtocol.REPLY_GETBUCKETSTATE, new testGetBucketStateReply());
+        out.put(DocumentProtocol.REPLY_GETDOCUMENT, new testGetDocumentReply());
+        out.put(DocumentProtocol.REPLY_MAPVISITOR, new testMapVisitorReply());
+        out.put(DocumentProtocol.REPLY_PUTDOCUMENT, new testPutDocumentReply());
+        out.put(DocumentProtocol.REPLY_QUERYRESULT, new testQueryResultReply());
+        out.put(DocumentProtocol.REPLY_REMOVEDOCUMENT, new testRemoveDocumentReply());
+        out.put(DocumentProtocol.REPLY_REMOVELOCATION, new testRemoveLocationReply());
+        out.put(DocumentProtocol.REPLY_SEARCHRESULT, new testSearchResultReply());
+        out.put(DocumentProtocol.REPLY_STATBUCKET, new testStatBucketReply());
+        out.put(DocumentProtocol.REPLY_UPDATEDOCUMENT, new testUpdateDocumentReply());
+        out.put(DocumentProtocol.REPLY_VISITORINFO, new testVisitorInfoReply());
+        out.put(DocumentProtocol.REPLY_WRONGDISTRIBUTION, new testWrongDistributionReply());
+    }
+
+    @Override
+    protected Version version() {
+        return new Version(5, 0);
+    }
+
+    @Override
+    protected boolean shouldTestCoverage() {
+        return false;
+    }
+
+    ////////////////////////////////////////////////////////////////////////////////
+    //
+    // Tests
+    //
+    ////////////////////////////////////////////////////////////////////////////////
+
+    private static int BASE_MESSAGE_LENGTH = 5;
+
+    public class testRemoveLocationMessage implements RunnableTest {
+
+        @Override
+        public void run() {
+            {
+                RemoveLocationMessage msg = new RemoveLocationMessage("id.group == \"mygroup\"");
+                assertEquals(BASE_MESSAGE_LENGTH + 29, serialize("RemoveLocationMessage", msg));
+
+                for (Language lang : LANGUAGES) {
+                    msg = (RemoveLocationMessage)deserialize("RemoveLocationMessage", DocumentProtocol.MESSAGE_REMOVELOCATION, lang);
+                    assertEquals("id.group == \"mygroup\"", msg.getDocumentSelection());
+                }
+            }
+        }
+    }
+
+    public class testGetBucketListMessage implements RunnableTest {
+
+        @Override
+        public void run() {
+            GetBucketListMessage msg = new GetBucketListMessage(new BucketId(16, 123));
+            msg.setLoadType(loadTypes.getNameMap().get("foo"));
+            assertEquals(BASE_MESSAGE_LENGTH + 12, serialize("GetBucketListMessage", msg));
+
+            for (Language lang : LANGUAGES) {
+                msg = (GetBucketListMessage)deserialize("GetBucketListMessage", DocumentProtocol.MESSAGE_GETBUCKETLIST, lang);
+                assertEquals(new BucketId(16, 123), msg.getBucketId());
+                assertEquals("foo", msg.getLoadType().getName());
+            }
+        }
+    }
+
+
+    public class testStatBucketMessage implements RunnableTest {
+
+        @Override
+        public void run() {
+            StatBucketMessage msg = new StatBucketMessage(new BucketId(16, 123), "id.user=123");
+            msg.setLoadType(null);
+            assertEquals(BASE_MESSAGE_LENGTH + 27, serialize("StatBucketMessage", msg));
+
+            for (Language lang : LANGUAGES) {
+                msg = (StatBucketMessage)deserialize("StatBucketMessage", DocumentProtocol.MESSAGE_STATBUCKET, lang);
+                assertEquals(new BucketId(16, 123), msg.getBucketId());
+                assertEquals("id.user=123", msg.getDocumentSelection());
+                assertEquals("default", msg.getLoadType().getName());
+            }
+        }
+    }
+
+    public class testGetBucketStateMessage implements RunnableTest {
+
+        @Override
+        public void run() {
+            GetBucketStateMessage msg = new GetBucketStateMessage(new BucketId(16, 666));
+            assertEquals(BASE_MESSAGE_LENGTH + 12, serialize("GetBucketStateMessage", msg));
+
+            for (Language lang : LANGUAGES) {
+                msg = (GetBucketStateMessage)deserialize("GetBucketStateMessage", DocumentProtocol.MESSAGE_GETBUCKETSTATE, lang);
+                assertEquals(16, msg.getBucketId().getUsedBits());
+                assertEquals(4611686018427388570l, msg.getBucketId().getId());
+            }
+        }
+    }
+
+    public class testCreateVisitorMessage implements RunnableTest {
+
+        @Override
+        @SuppressWarnings("deprecation")
+        public void run() {
+            CreateVisitorMessage msg = new CreateVisitorMessage("SomeLibrary", "myvisitor", "newyork", "london");
+            msg.setDocumentSelection("true and false or true");
+            msg.getParameters().put("myvar", Utf8.toBytes("somevalue"));
+            msg.getParameters().put("anothervar", Utf8.toBytes("34"));
+            msg.getBuckets().add(new BucketId(16, 1234));
+            msg.setVisitRemoves(true);
+            msg.setVisitorOrdering(OrderingSpecification.DESCENDING);
+            msg.setMaxBucketsPerVisitor(2);
+            assertEquals(BASE_MESSAGE_LENGTH + 168, serialize("CreateVisitorMessage", msg));
+
+            for (Language lang : LANGUAGES) {
+                msg = (CreateVisitorMessage)deserialize("CreateVisitorMessage", DocumentProtocol.MESSAGE_CREATEVISITOR, lang);
+                assertEquals("SomeLibrary", msg.getLibraryName());
+                assertEquals("myvisitor", msg.getInstanceId());
+                assertEquals("newyork", msg.getControlDestination());
+                assertEquals("london", msg.getDataDestination());
+                assertEquals("true and false or true", msg.getDocumentSelection());
+                assertEquals(8, msg.getMaxPendingReplyCount());
+                assertEquals(true, msg.getVisitRemoves());
+                assertEquals(false, msg.getVisitInconsistentBuckets());
+                assertEquals(1, msg.getBuckets().size());
+                assertEquals(new BucketId(16, 1234), msg.getBuckets().iterator().next());
+                assertEquals("somevalue", Utf8.toString(msg.getParameters().get("myvar")));
+                assertEquals("34", Utf8.toString(msg.getParameters().get("anothervar")));
+                assertEquals(OrderingSpecification.DESCENDING, msg.getVisitorOrdering());
+                assertEquals(2, msg.getMaxBucketsPerVisitor());
+            }
+
+            msg.getBuckets().clear();
+
+            assertEquals("CreateVisitorMessage(" +
+                    "No buckets, " +
+                    "selection 'true and false or true', " +
+                    "library SomeLibrary, including removes, " +
+                    "get fields: [all]" +
+                    ")",
+                    msg.toString());
+
+            msg.getBuckets().add(new BucketId(16, 1234));
+
+            assertEquals("CreateVisitorMessage(" +
+                    "Bucket BucketId(0x40000000000004d2), " +
+                    "selection 'true and false or true', " +
+                    "library SomeLibrary, including removes, " +
+                    "get fields: [all]" +
+                    ")",
+                    msg.toString());
+
+            msg.getBuckets().add(new BucketId(16, 1235));
+            msg.getBuckets().add(new BucketId(16, 1236));
+            msg.getBuckets().add(new BucketId(16, 1237));
+            msg.getBuckets().add(new BucketId(16, 1238));
+            msg.setFromTimestamp(10001);
+            msg.setToTimestamp(20002);
+            msg.setVisitInconsistentBuckets(true);
+            assertEquals("CreateVisitorMessage(" +
+                    "5 buckets: BucketId(0x40000000000004d2) BucketId(0x40000000000004d3) BucketId(0x40000000000004d4) ..., " +
+                    "time 10001-20002, " +
+                    "selection 'true and false or true', " +
+                    "library SomeLibrary, including removes, " +
+                    "get fields: [all], " +
+                    "visit inconsistent buckets" +
+                    ")",
+                    msg.toString());
+        }
+    }
+
+    public class testCreateVisitorReply implements RunnableTest {
+
+        @Override
+        public void run() {
+            CreateVisitorReply reply = new CreateVisitorReply(DocumentProtocol.REPLY_CREATEVISITOR);
+            reply.setLastBucket(new BucketId(16, 123));
+            reply.getVisitorStatistics().setBucketsVisited(3);
+            reply.getVisitorStatistics().setDocumentsVisited(1000);
+            reply.getVisitorStatistics().setBytesVisited(1024000);
+            reply.getVisitorStatistics().setDocumentsReturned(123);
+            reply.getVisitorStatistics().setBytesReturned(512000);
+            reply.getVisitorStatistics().setSecondPassDocumentsReturned(456);
+            reply.getVisitorStatistics().setSecondPassBytesReturned(789100);
+
+            assertEquals(65, serialize("CreateVisitorReply", reply));
+
+            for (Language lang : LANGUAGES) {
+                reply = (CreateVisitorReply)deserialize("CreateVisitorReply", DocumentProtocol.REPLY_CREATEVISITOR, lang);
+                assertNotNull(reply);
+                assertEquals(new BucketId(16, 123), reply.getLastBucket());
+                assertEquals(3, reply.getVisitorStatistics().getBucketsVisited());
+                assertEquals(1000, reply.getVisitorStatistics().getDocumentsVisited());
+                assertEquals(1024000, reply.getVisitorStatistics().getBytesVisited());
+                assertEquals(123, reply.getVisitorStatistics().getDocumentsReturned());
+                assertEquals(512000, reply.getVisitorStatistics().getBytesReturned());
+                assertEquals(456, reply.getVisitorStatistics().getSecondPassDocumentsReturned());
+                assertEquals(789100, reply.getVisitorStatistics().getSecondPassBytesReturned());
+            }
+        }
+    }
+
+    public class testDestroyVisitorReply implements RunnableTest {
+
+        @Override
+        public void run() {
+            testVisitorReply("DestroyVisitorReply", DocumentProtocol.REPLY_DESTROYVISITOR);
+        }
+    }
+
+    public class testDocumentListReply implements RunnableTest {
+
+        @Override
+        public void run() {
+            testVisitorReply("DocumentListReply", DocumentProtocol.REPLY_DOCUMENTLIST);
+        }
+    }
+
+    public class testDocumentSummaryReply implements RunnableTest {
+
+        @Override
+        public void run() {
+            testVisitorReply("DocumentSummaryReply", DocumentProtocol.REPLY_DOCUMENTSUMMARY);
+        }
+    }
+
+    public class testEmptyBucketsReply implements RunnableTest {
+
+        @Override
+        public void run() {
+            testVisitorReply("EmptyBucketsReply", DocumentProtocol.REPLY_EMPTYBUCKETS);
+        }
+    }
+
+    public class testDestroyVisitorMessage implements RunnableTest {
+
+        @Override
+        public void run() {
+            DestroyVisitorMessage msg = new DestroyVisitorMessage("myvisitor");
+            assertEquals(BASE_MESSAGE_LENGTH + 17, serialize("DestroyVisitorMessage", msg));
+
+            for (Language lang : LANGUAGES) {
+                msg = (DestroyVisitorMessage)deserialize("DestroyVisitorMessage", DocumentProtocol.MESSAGE_DESTROYVISITOR, lang);
+                assertEquals("myvisitor", msg.getInstanceId());
+            }
+        }
+    }
+
+    public class testDocumentListMessage implements RunnableTest {
+
+        @Override
+        public void run() {
+            DocumentListMessage msg = (DocumentListMessage)deserialize("DocumentListMessage", DocumentProtocol.MESSAGE_DOCUMENTLIST, Language.CPP);
+            assertEquals("userdoc:scheme:1234:", msg.getDocuments().get(0).getDocument().getId().toString());
+            assertEquals(1234, msg.getDocuments().get(0).getTimestamp());
+            assertFalse(msg.getDocuments().get(0).isRemoveEntry());
+
+            assertEquals(BASE_MESSAGE_LENGTH + 63, serialize("DocumentListMessage", msg));
+            msg = (DocumentListMessage)deserialize("DocumentListMessage", DocumentProtocol.MESSAGE_DOCUMENTLIST, Language.JAVA);
+            assertEquals("userdoc:scheme:1234:", msg.getDocuments().get(0).getDocument().getId().toString());
+            assertEquals(1234, msg.getDocuments().get(0).getTimestamp());
+            assertFalse(msg.getDocuments().get(0).isRemoveEntry());
+
+        }
+    }
+
+    public class testEmptyBucketsMessage implements RunnableTest {
+
+        @Override
+        public void run() {
+            List<BucketId> bids = new ArrayList<>();
+            for (int i = 0; i < 13; ++i) {
+                bids.add(new BucketId(16, i));
+            }
+
+            EmptyBucketsMessage ebm = new EmptyBucketsMessage(bids);
+            assertEquals(BASE_MESSAGE_LENGTH + 112, serialize("EmptyBucketsMessage", ebm));
+            for (Language lang : LANGUAGES) {
+                ebm = (EmptyBucketsMessage)deserialize("EmptyBucketsMessage", DocumentProtocol.MESSAGE_EMPTYBUCKETS, lang);
+                for (int i = 0; i < 13; ++i) {
+                    assertEquals(new BucketId(16, i), ebm.getBucketIds().get(i));
+                }
+            }
+        }
+    }
+
+    public class testDocumentSummaryMessage implements RunnableTest {
+
+        @Override
+        public void run() {
+            try {
+                FileInputStream stream = new FileInputStream(getPath("5-cpp-DocumentSummaryMessage-1.dat"));
+                byte[] data = new byte[stream.available()];
+                assertEquals(data.length, stream.read(data));
+
+                Routable routable = decode(data);
+                assertTrue(routable instanceof DocumentSummaryMessage);
+
+                DocumentSummaryMessage msg = (DocumentSummaryMessage)routable;
+                assertEquals(0, msg.getResult().getSummaryCount());
+
+                stream = new FileInputStream(getPath("5-cpp-DocumentSummaryMessage-2.dat"));
+                data = new byte[stream.available()];
+                assertEquals(data.length, stream.read(data));
+
+                routable = decode(data);
+                assertTrue(routable instanceof DocumentSummaryMessage);
+
+                msg = (DocumentSummaryMessage)routable;
+                assertEquals(2, msg.getResult().getSummaryCount());
+                com.yahoo.vdslib.DocumentSummary.Summary s = msg.getResult().getSummary(0);
+                assertEquals("doc1", s.getDocId());
+                byte[] b = s.getSummary();
+                assertEquals(8, b.length);
+                byte[] c = { 's', 'u', 'm', 'm', 'a', 'r', 'y', '1' };
+                for (int i = 0; i < b.length; i++) {
+                    assertEquals(c[i], b[i]);
+                }
+
+                s = msg.getResult().getSummary(1);
+                assertEquals("aoc17", s.getDocId());
+                b = s.getSummary();
+                assertEquals(9, b.length);
+                byte[] d = { 's', 'u', 'm', 'm', 'a', 'r', 'y', '4', '5' };
+                for (int i = 0; i < b.length; i++) {
+                    assertEquals(d[i], b[i]);
+                }
+
+                stream = new FileInputStream(getPath("5-cpp-DocumentSummaryMessage-3.dat"));
+                data = new byte[stream.available()];
+                assertEquals(data.length, stream.read(data));
+
+                routable = decode(data);
+                assertTrue(routable instanceof DocumentSummaryMessage);
+
+                msg = (DocumentSummaryMessage)routable;
+                assertEquals(2, msg.getResult().getSummaryCount());
+
+                s = msg.getResult().getSummary(0);
+                assertEquals("aoc17", s.getDocId());
+                b = s.getSummary();
+                assertEquals(9, b.length);
+                byte[] e = { 's', 'u', 'm', 'm', 'a', 'r', 'y', '4', '5' };
+                for (int i = 0; i < b.length; i++) {
+                    assertEquals(e[i], b[i]);
+                }
+
+                s = msg.getResult().getSummary(1);
+                assertEquals("doc1", s.getDocId());
+                b = s.getSummary();
+                assertEquals(8, b.length);
+                byte[] f = { 's', 'u', 'm', 'm', 'a', 'r', 'y', '1' };
+                for (int i = 0; i < b.length; i++) {
+                    assertEquals(f[i], b[i]);
+                }
+            } catch (IOException e) {
+                fail(e.toString());
+            }
+        }
+    }
+
+
+    public class testGetDocumentMessage implements RunnableTest {
+
+        @Override
+        @SuppressWarnings("deprecation")
+        public void run() {
+            GetDocumentMessage msg = new GetDocumentMessage(new DocumentId("doc:scheme:"));
+            assertEquals(BASE_MESSAGE_LENGTH + 20, serialize("GetDocumentMessage", msg));
+
+            for (Language lang : LANGUAGES) {
+                msg = (GetDocumentMessage)deserialize("GetDocumentMessage", DocumentProtocol.MESSAGE_GETDOCUMENT, lang);
+                assertEquals("doc:scheme:", msg.getDocumentId().toString());
+            }
+        }
+    }
+
+
+    public class testRemoveDocumentMessage implements RunnableTest {
+
+        @Override
+        public void run() {
+            RemoveDocumentMessage msg = new RemoveDocumentMessage(new DocumentId("doc:scheme:"));
+            assertEquals(BASE_MESSAGE_LENGTH + 16, serialize("RemoveDocumentMessage", msg));
+
+            for (Language lang : LANGUAGES) {
+                msg = (RemoveDocumentMessage)deserialize("RemoveDocumentMessage", DocumentProtocol.MESSAGE_REMOVEDOCUMENT, lang);
+                assertEquals("doc:scheme:", msg.getDocumentId().toString());
+            }
+        }
+    }
+
+    public class testMapVisitorMessage implements RunnableTest {
+
+        @Override
+        public void run() {
+            MapVisitorMessage msg = (MapVisitorMessage)deserialize("MapVisitorMessage", DocumentProtocol.MESSAGE_MAPVISITOR, Language.CPP);
+            assertEquals("3", msg.getData().get("foo"));
+            assertEquals("5", msg.getData().get("bar"));
+
+            assertEquals(BASE_MESSAGE_LENGTH + 32, serialize("MapVisitorMessage", msg));
+
+            msg = (MapVisitorMessage)deserialize("MapVisitorMessage", DocumentProtocol.MESSAGE_MAPVISITOR, Language.JAVA);
+            assertEquals("3", msg.getData().get("foo"));
+            assertEquals("5", msg.getData().get("bar"));
+        }
+    }
+
+
+    public class testVisitorInfoMessage implements RunnableTest {
+
+        @Override
+        public void run() {
+            VisitorInfoMessage msg = new VisitorInfoMessage();
+            msg.getFinishedBuckets().add(new BucketId(16, 1));
+            msg.getFinishedBuckets().add(new BucketId(16, 2));
+            msg.getFinishedBuckets().add(new BucketId(16, 4));
+            msg.setErrorMessage("error message: \u00e6\u00c6\u00f8\u00d8\u00e5\u00c5\u00f6\u00d6");
+            assertEquals(BASE_MESSAGE_LENGTH + 67, serialize("VisitorInfoMessage", msg));
+
+            for (Language lang : LANGUAGES) {
+                msg = (VisitorInfoMessage)deserialize("VisitorInfoMessage", DocumentProtocol.MESSAGE_VISITORINFO, lang);
+                assertTrue(msg.getFinishedBuckets().contains(new BucketId(16, 1)));
+                assertTrue(msg.getFinishedBuckets().contains(new BucketId(16, 2)));
+                assertTrue(msg.getFinishedBuckets().contains(new BucketId(16, 4)));
+                assertEquals("error message: \u00e6\u00c6\u00f8\u00d8\u00e5\u00c5\u00f6\u00d6", msg.getErrorMessage());
+            }
+        }
+    }
+
+    public class testSearchResultMessage implements RunnableTest {
+
+        @Override
+        public void run() throws Exception {
+            FileInputStream stream = new FileInputStream(getPath("5-cpp-SearchResultMessage-1.dat"));
+            byte[] data = new byte[stream.available()];
+            assertEquals(data.length, stream.read(data));
+
+            Routable routable = decode(data);
+            assertTrue(routable instanceof SearchResultMessage);
+
+            SearchResultMessage msg = (SearchResultMessage)routable;
+            assertEquals(0, msg.getResult().getHitCount());
+
+            stream = new FileInputStream(getPath("5-cpp-SearchResultMessage-2.dat"));
+            data = new byte[stream.available()];
+            assertEquals(data.length, stream.read(data));
+
+            routable = decode(data);
+            assertTrue(routable instanceof SearchResultMessage);
+
+            msg = (SearchResultMessage)routable;
+            assertEquals(2, msg.getResult().getHitCount());
+            com.yahoo.vdslib.SearchResult.Hit h = msg.getResult().getHit(0);
+            assertEquals(89.0, h.getRank(), 1E-6);
+            assertEquals("doc1", h.getDocId());
+            h = msg.getResult().getHit(1);
+            assertEquals(109.0, h.getRank(), 1E-6);
+            assertEquals("doc17", h.getDocId());
+
+            stream = new FileInputStream(getPath("5-cpp-SearchResultMessage-3.dat"));
+            data = new byte[stream.available()];
+            assertEquals(data.length, stream.read(data));
+
+            routable = decode(data);
+            assertTrue(routable instanceof SearchResultMessage);
+
+            msg = (SearchResultMessage)routable;
+            assertEquals(2, msg.getResult().getHitCount());
+            h = msg.getResult().getHit(0);
+            assertEquals(109.0, h.getRank(), 1E-6);
+            assertEquals("doc17", h.getDocId());
+            h = msg.getResult().getHit(1);
+            assertEquals(89.0, h.getRank(), 1E-6);
+            assertEquals("doc1", h.getDocId());
+
+            stream = new FileInputStream(getPath("5-cpp-SearchResultMessage-4.dat"));
+            data = new byte[stream.available()];
+            assertEquals(data.length, stream.read(data));
+
+            routable = decode(data);
+            assertTrue(routable instanceof SearchResultMessage);
+
+            msg = (SearchResultMessage)routable;
+            assertEquals(3, msg.getResult().getHitCount());
+            h = msg.getResult().getHit(0);
+            assertTrue(h instanceof SearchResult.HitWithSortBlob);
+            assertEquals(89.0, h.getRank(), 1E-6);
+            assertEquals("doc1", h.getDocId());
+            byte[] b = ((SearchResult.HitWithSortBlob)h).getSortBlob();
+            assertEquals(9, b.length);
+            byte[] e = { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '2' };
+            for (int i = 0; i < b.length; i++) {
+                assertEquals(e[i], b[i]);
+            }
+            h = msg.getResult().getHit(1);
+            assertTrue(h instanceof SearchResult.HitWithSortBlob);
+            assertEquals(109.0, h.getRank(), 1E-6);
+            assertEquals("doc17", h.getDocId());
+            b = ((SearchResult.HitWithSortBlob)h).getSortBlob();
+            assertEquals(9, b.length);
+            byte[] d = { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '1' };
+            for (int i = 0; i < b.length; i++) {
+                assertEquals(d[i], b[i]);
+            }
+            h = msg.getResult().getHit(2);
+            assertTrue(h instanceof SearchResult.HitWithSortBlob);
+            assertEquals(90.0, h.getRank(), 1E-6);
+            assertEquals("doc18", h.getDocId());
+            b = ((SearchResult.HitWithSortBlob)h).getSortBlob();
+            assertEquals(9, b.length);
+            byte[] c = { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '3' };
+            for (int i = 0; i < b.length; i++) {
+                assertEquals(c[i], b[i]);
+            }
+        }
+    }
+
+    public class testPutDocumentMessage implements RunnableTest {
+
+        @Override
+        public void run() {
+            PutDocumentMessage msg = new PutDocumentMessage(new DocumentPut(new Document(protocol.getDocumentTypeManager().getDocumentType("testdoc"), "doc:scheme:")));
+            msg.setTimestamp(666);
+            assertEquals(BASE_MESSAGE_LENGTH + 41, serialize("PutDocumentMessage", msg));
+
+            for (Language lang : LANGUAGES) {
+                msg = (PutDocumentMessage)deserialize("PutDocumentMessage", DocumentProtocol.MESSAGE_PUTDOCUMENT, lang);
+                assertEquals("testdoc", msg.getDocumentPut().getDocument().getDataType().getName());
+                assertEquals("doc:scheme:", msg.getDocumentPut().getDocument().getId().toString());
+                assertEquals(666, msg.getTimestamp());
+            }
+        }
+    }
+
+    public class testPutDocumentReply implements RunnableTest {
+
+        @Override
+        public void run() {
+            WriteDocumentReply reply = new WriteDocumentReply(DocumentProtocol.REPLY_PUTDOCUMENT);
+            reply.setHighestModificationTimestamp(30);
+
+            assertEquals(13, serialize("PutDocumentReply", reply));
+
+            for (Language lang : LANGUAGES) {
+                WriteDocumentReply obj = (WriteDocumentReply)deserialize("PutDocumentReply", DocumentProtocol.REPLY_PUTDOCUMENT, lang);
+                assertNotNull(obj);
+                assertEquals(30, obj.getHighestModificationTimestamp());
+            }
+        }
+    }
+
+    public class testUpdateDocumentMessage implements RunnableTest {
+
+        @Override
+        public void run() {
+            DocumentType docType = protocol.getDocumentTypeManager().getDocumentType("testdoc");
+            DocumentUpdate update = new DocumentUpdate(docType, new DocumentId("doc:scheme:"));
+            update.addFieldPathUpdate(new RemoveFieldPathUpdate(docType, "intfield", "testdoc.intfield > 0"));
+            UpdateDocumentMessage msg = new UpdateDocumentMessage(update);
+            msg.setNewTimestamp(777);
+            msg.setOldTimestamp(666);
+
+            assertEquals(BASE_MESSAGE_LENGTH + 89, serialize("UpdateDocumentMessage", msg));
+
+            for (Language lang : LANGUAGES) {
+                msg = (UpdateDocumentMessage)deserialize("UpdateDocumentMessage", DocumentProtocol.MESSAGE_UPDATEDOCUMENT, lang);
+                assertEquals(update, msg.getDocumentUpdate());
+                assertEquals(777, msg.getNewTimestamp());
+                assertEquals(666, msg.getOldTimestamp());
+            }
+        }
+    }
+
+    public class testUpdateDocumentReply implements RunnableTest {
+
+        @Override
+        public void run() {
+            UpdateDocumentReply reply = new UpdateDocumentReply();
+            reply.setHighestModificationTimestamp(30);
+            reply.setWasFound(false);
+
+            assertEquals(14, serialize("UpdateDocumentReply", reply));
+
+            for (Language lang : LANGUAGES) {
+                UpdateDocumentReply obj = (UpdateDocumentReply)deserialize("UpdateDocumentReply", DocumentProtocol.REPLY_UPDATEDOCUMENT, lang);
+                assertNotNull(obj);
+                assertEquals(30, reply.getHighestModificationTimestamp());
+                assertEquals(false, obj.wasFound());
+            }
+        }
+    }
+
+    public class testVisitorInfoReply implements RunnableTest {
+
+        @Override
+        public void run() {
+            testVisitorReply("VisitorInfoReply", DocumentProtocol.REPLY_VISITORINFO);
+        }
+    }
+
+    public class testWrongDistributionReply implements RunnableTest {
+
+        @Override
+        public void run() {
+            WrongDistributionReply reply = new WrongDistributionReply("distributor:3 storage:2");
+            assertEquals(32, serialize("WrongDistributionReply", reply));
+
+            for (Language lang : LANGUAGES) {
+                reply = (WrongDistributionReply)deserialize("WrongDistributionReply", DocumentProtocol.REPLY_WRONGDISTRIBUTION, lang);
+                assertEquals("distributor:3 storage:2", reply.getSystemState());
+            }
+        }
+    }
+
+    public class testRemoveDocumentReply implements RunnableTest {
+
+        @Override
+        public void run() {
+            RemoveDocumentReply reply = new RemoveDocumentReply();
+            reply.setHighestModificationTimestamp(30);
+            reply.setWasFound(false);
+
+            assertEquals(14, serialize("RemoveDocumentReply", reply));
+
+            for (Language lang : LANGUAGES) {
+                RemoveDocumentReply obj = (RemoveDocumentReply)deserialize("RemoveDocumentReply", DocumentProtocol.REPLY_REMOVEDOCUMENT, lang);
+                assertNotNull(obj);
+                assertEquals(30, obj.getHighestModificationTimestamp());
+                assertEquals(false, obj.wasFound());
+            }
+        }
+    }
+
+    public class testRemoveLocationReply implements RunnableTest {
+
+        @Override
+        public void run() {
+            testDocumentReply("RemoveLocationReply", DocumentProtocol.REPLY_REMOVELOCATION);
+        }
+    }
+
+    public class testSearchResultReply implements RunnableTest {
+
+        @Override
+        public void run() {
+            testVisitorReply("SearchResultReply", DocumentProtocol.REPLY_SEARCHRESULT);
+        }
+    }
+
+    public class testStatBucketReply implements RunnableTest {
+
+        @Override
+        public void run() {
+            StatBucketReply msg = new StatBucketReply();
+            msg.setResults("These are the votes of the Norwegian jury");
+
+            assertEquals(50, serialize("StatBucketReply", msg));
+
+            for (Language lang : LANGUAGES) {
+                msg = (StatBucketReply)deserialize("StatBucketReply", DocumentProtocol.REPLY_STATBUCKET, lang);
+                assertEquals("These are the votes of the Norwegian jury", msg.getResults());
+            }
+        }
+    }
+
+    public class testBatchDocumentUpdateMessage implements RunnableTest {
+
+        @Override
+        public void run() {
+            DocumentType docType = protocol.getDocumentTypeManager().getDocumentType("testdoc");
+            BatchDocumentUpdateMessage msg = new BatchDocumentUpdateMessage(1234);
+
+            {
+                DocumentUpdate update = new DocumentUpdate(docType, new DocumentId("userdoc:footype:1234:foo"));
+                update.addFieldPathUpdate(new RemoveFieldPathUpdate(docType, "intfield", "testdoc.intfield > 0"));
+                msg.addUpdate(update);
+            }
+            {
+                DocumentUpdate update = new DocumentUpdate(docType, new DocumentId("orderdoc(32,17):footype:1234:123456789:foo"));
+                update.addFieldPathUpdate(new RemoveFieldPathUpdate(docType, "intfield", "testdoc.intfield > 0"));
+                msg.addUpdate(update);
+            }
+
+            try {
+                DocumentUpdate update = new DocumentUpdate(docType, new DocumentId("orderdoc:footype:5678:foo"));
+                update.addFieldPathUpdate(new RemoveFieldPathUpdate(docType, "intfield", "testdoc.intfield > 0"));
+                msg.addUpdate(update);
+                fail();
+            } catch (Exception e) {
+
+            }
+
+            try {
+                DocumentUpdate update = new DocumentUpdate(docType, new DocumentId("groupdoc:footype:hable:foo"));
+                update.addFieldPathUpdate(new RemoveFieldPathUpdate(docType, "intfield", "testdoc.intfield > 0"));
+                msg.addUpdate(update);
+                fail();
+            } catch (Exception e) {
+
+            }
+
+            assertEquals(2, msg.getUpdates().size());
+
+            assertEquals(BASE_MESSAGE_LENGTH + 202, serialize("BatchDocumentUpdateMessage", msg));
+
+            for (Language lang : LANGUAGES) {
+                msg = (BatchDocumentUpdateMessage)deserialize("BatchDocumentUpdateMessage", DocumentProtocol.MESSAGE_BATCHDOCUMENTUPDATE, lang);
+                assertEquals(2, msg.getUpdates().size());
+            }
+        }
+    }
+
+    public class testBatchDocumentUpdateReply implements RunnableTest {
+
+        @Override
+        public void run() {
+            BatchDocumentUpdateReply reply = new BatchDocumentUpdateReply();
+            reply.setHighestModificationTimestamp(30);
+            reply.getDocumentsNotFound().add(false);
+            reply.getDocumentsNotFound().add(true);
+            reply.getDocumentsNotFound().add(true);
+
+            assertEquals(20, serialize("BatchDocumentUpdateReply", reply));
+
+            for (Language lang : LANGUAGES) {
+                BatchDocumentUpdateReply obj = (BatchDocumentUpdateReply)deserialize("BatchDocumentUpdateReply", DocumentProtocol.REPLY_BATCHDOCUMENTUPDATE, lang);
+                assertNotNull(obj);
+                assertEquals(30, obj.getHighestModificationTimestamp());
+                assertEquals(3, obj.getDocumentsNotFound().size());
+                assertFalse(obj.getDocumentsNotFound().get(0));
+                assertTrue(obj.getDocumentsNotFound().get(1));
+                assertTrue(obj.getDocumentsNotFound().get(2));
+            }
+        }
+    }
+
+
+    public class testQueryResultReply implements RunnableTest {
+
+        @Override
+        public void run() {
+            testVisitorReply("QueryResultReply", DocumentProtocol.REPLY_QUERYRESULT);
+        }
+    }
+
+    public class testQueryResultMessage implements RunnableTest {
+
+        @Override
+        public void run() throws Exception {
+            FileInputStream stream = new FileInputStream(getPath("5-cpp-QueryResultMessage-1.dat"));
+            byte[] data = new byte[stream.available()];
+            assertEquals(data.length, stream.read(data));
+
+            Routable routable = decode(data);
+            assertTrue(routable instanceof QueryResultMessage);
+
+            QueryResultMessage msg = (QueryResultMessage)routable;
+            assertEquals(0, msg.getResult().getHitCount());
+
+            stream = new FileInputStream(getPath("5-cpp-QueryResultMessage-2.dat"));
+            data = new byte[stream.available()];
+            assertEquals(data.length, stream.read(data));
+
+            routable = decode(data);
+            assertTrue(routable instanceof QueryResultMessage);
+
+            msg = (QueryResultMessage)routable;
+            assertEquals(2, msg.getResult().getHitCount());
+            com.yahoo.vdslib.SearchResult.Hit h = msg.getResult().getHit(0);
+            assertEquals(89.0, h.getRank(), 1E-6);
+            assertEquals("doc1", h.getDocId());
+            h = msg.getResult().getHit(1);
+            assertEquals(109.0, h.getRank(), 1E-6);
+            assertEquals("doc17", h.getDocId());
+
+            stream = new FileInputStream(getPath("5-cpp-QueryResultMessage-3.dat"));
+            data = new byte[stream.available()];
+            assertEquals(data.length, stream.read(data));
+
+            routable = decode(data);
+            assertTrue(routable instanceof QueryResultMessage);
+
+            msg = (QueryResultMessage)routable;
+            assertEquals(2, msg.getResult().getHitCount());
+            h = msg.getResult().getHit(0);
+            assertEquals(109.0, h.getRank(), 1E-6);
+            assertEquals("doc17", h.getDocId());
+            h = msg.getResult().getHit(1);
+            assertEquals(89.0, h.getRank(), 1E-6);
+            assertEquals("doc1", h.getDocId());
+
+            stream = new FileInputStream(getPath("5-cpp-QueryResultMessage-4.dat"));
+            data = new byte[stream.available()];
+            assertEquals(data.length, stream.read(data));
+
+            routable = decode(data);
+            assertTrue(routable instanceof QueryResultMessage);
+
+            msg = (QueryResultMessage)routable;
+            assertEquals(3, msg.getResult().getHitCount());
+            h = msg.getResult().getHit(0);
+            assertTrue(h instanceof SearchResult.HitWithSortBlob);
+            assertEquals(89.0, h.getRank(), 1E-6);
+            assertEquals("doc1", h.getDocId());
+            byte[] b = ((SearchResult.HitWithSortBlob)h).getSortBlob();
+            assertEquals(9, b.length);
+            byte[] e = { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '2' };
+            for (int i = 0; i < b.length; i++) {
+                assertEquals(e[i], b[i]);
+            }
+            h = msg.getResult().getHit(1);
+            assertTrue(h instanceof SearchResult.HitWithSortBlob);
+            assertEquals(109.0, h.getRank(), 1E-6);
+            assertEquals("doc17", h.getDocId());
+            b = ((SearchResult.HitWithSortBlob)h).getSortBlob();
+            assertEquals(9, b.length);
+            byte[] d = { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '1' };
+            for (int i = 0; i < b.length; i++) {
+                assertEquals(d[i], b[i]);
+            }
+            h = msg.getResult().getHit(2);
+            assertTrue(h instanceof SearchResult.HitWithSortBlob);
+            assertEquals(90.0, h.getRank(), 1E-6);
+            assertEquals("doc18", h.getDocId());
+            b = ((SearchResult.HitWithSortBlob)h).getSortBlob();
+            assertEquals(9, b.length);
+            byte[] c = { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '3' };
+            for (int i = 0; i < b.length; i++) {
+                assertEquals(c[i], b[i]);
+            }
+        }
+    }
+
+    public class testGetBucketListReply implements RunnableTest {
+
+        public void run() {
+            GetBucketListReply reply = new GetBucketListReply();
+            reply.getBuckets().add(new GetBucketListReply.BucketInfo(new BucketId(16, 123), "foo"));
+            reply.getBuckets().add(new GetBucketListReply.BucketInfo(new BucketId(17, 1123), "bar"));
+            reply.getBuckets().add(new GetBucketListReply.BucketInfo(new BucketId(18, 11123), "zoink"));
+
+            assertEquals(56, serialize("GetBucketListReply", reply));
+
+            for (Language lang : LANGUAGES) {
+                reply = (GetBucketListReply)deserialize("GetBucketListReply", DocumentProtocol.REPLY_GETBUCKETLIST, lang);
+                assertEquals(reply.getBuckets().get(0), new GetBucketListReply.BucketInfo(new BucketId(16, 123), "foo"));
+                assertEquals(reply.getBuckets().get(1), new GetBucketListReply.BucketInfo(new BucketId(17, 1123), "bar"));
+                assertEquals(reply.getBuckets().get(2), new GetBucketListReply.BucketInfo(new BucketId(18, 11123), "zoink"));
+            }
+        }
+    }
+
+    public class testGetBucketStateReply implements RunnableTest {
+
+        public void run() {
+            GlobalId foo = new GlobalId(IdString.createIdString("doc:scheme:foo"));
+            GlobalId bar = new GlobalId(IdString.createIdString("doc:scheme:bar"));
+
+            GetBucketStateReply reply = new GetBucketStateReply();
+            List<DocumentState> state = new ArrayList<>(2);
+            state.add(new DocumentState(foo, 777, false));
+            state.add(new DocumentState(bar, 888, true));
+            reply.setBucketState(state);
+            assertEquals(53, serialize("GetBucketStateReply", reply));
+
+            for (Language lang : LANGUAGES) {
+                reply = (GetBucketStateReply)deserialize("GetBucketStateReply", DocumentProtocol.REPLY_GETBUCKETSTATE, lang);
+                assertEquals(777, reply.getBucketState().get(0).getTimestamp());
+                assertEquals(foo, reply.getBucketState().get(0).getGid());
+                assertEquals(false, reply.getBucketState().get(0).isRemoveEntry());
+                assertEquals(888, reply.getBucketState().get(1).getTimestamp());
+                assertEquals(bar, reply.getBucketState().get(1).getGid());
+                assertEquals(true, reply.getBucketState().get(1).isRemoveEntry());
+            }
+        }
+    }
+
+    public class testGetDocumentReply implements RunnableTest {
+
+        public void run() {
+            GetDocumentReply reply = new GetDocumentReply(new Document(protocol.getDocumentTypeManager().getDocumentType("testdoc"), "doc:scheme:"));
+            assertEquals(43, serialize("GetDocumentReply", reply));
+
+            for (Language lang : LANGUAGES) {
+                reply = (GetDocumentReply)deserialize("GetDocumentReply", DocumentProtocol.REPLY_GETDOCUMENT, lang);
+                assertEquals("testdoc", reply.getDocument().getDataType().getName());
+                assertEquals("doc:scheme:", reply.getDocument().getId().toString());
+            }
+        }
+    }
+
+    public class testMapVisitorReply implements RunnableTest {
+
+        public void run() {
+            testVisitorReply("MapVisitorReply", DocumentProtocol.REPLY_MAPVISITOR);
+        }
+    }
+
+    protected void testDocumentReply(String filename, int type) {
+        DocumentReply reply = new DocumentReply(type);
+        assertEquals(5, serialize(filename, reply));
+
+        for (Language lang : LANGUAGES) {
+            reply = (DocumentReply)deserialize(filename, type, lang);
+            assertNotNull(reply);
+        }
+    }
+
+    protected void testVisitorReply(String filename, int type) {
+        VisitorReply reply = new VisitorReply(type);
+        assertEquals(5, serialize(filename, reply));
+
+        for (Language lang : LANGUAGES) {
+            reply = (VisitorReply)deserialize(filename, type, lang);
+            assertNotNull(reply);
+        }
+    }
+
+}
diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages51TestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages51TestCase.java
index 68448adb8fc..aa97355e0c4 100644
--- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages51TestCase.java
+++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages51TestCase.java
@@ -1,121 +1,121 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.documentapi.messagebus.protocol.test;

-

-import com.yahoo.component.Version;

-import com.yahoo.document.BucketId;

-import com.yahoo.document.DocumentId;

-import com.yahoo.document.select.OrderingSpecification;

-import com.yahoo.documentapi.messagebus.protocol.CreateVisitorMessage;

-import com.yahoo.documentapi.messagebus.protocol.DocumentIgnoredReply;

-import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol;

-import com.yahoo.documentapi.messagebus.protocol.GetDocumentMessage;

-import com.yahoo.text.Utf8;

-

-import java.util.Map;

-

-import static org.junit.Assert.assertEquals;

-

-/**

- * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>

- */

-public class Messages51TestCase extends Messages50TestCase {

-

-    ////////////////////////////////////////////////////////////////////////////////

-    //

-    // Setup

-    //

-    ///////////////////////////////////////////////////////////////////////////////

-

-    @Override

-    protected void registerTests(Map<Integer, RunnableTest> out) {

-        super.registerTests(out);

-

-        // This list MUST mirror the list of routable factories from the DocumentProtocol constructor that support

-        // version 5.0. When adding tests to this list, please KEEP THEM ORDERED alphabetically like they are now.

-        out.put(DocumentProtocol.MESSAGE_CREATEVISITOR, new testCreateVisitorMessage());

-        out.put(DocumentProtocol.MESSAGE_GETDOCUMENT, new testGetDocumentMessage());

-        out.put(DocumentProtocol.REPLY_DOCUMENTIGNORED, new testDocumentIgnoredReply());

-    }

-

-    @Override

-    protected Version version() {

-        return new Version(5, 1);

-    }

-

-    @Override

-    protected boolean shouldTestCoverage() {

-        return true;

-    }

-

-    ////////////////////////////////////////////////////////////////////////////////

-    //

-    // Tests

-    //

-    ////////////////////////////////////////////////////////////////////////////////

-

-    private static int BASE_MESSAGE_LENGTH = 5;

-

-    public class testCreateVisitorMessage implements RunnableTest {

-

-        @Override

-        public void run() {

-            CreateVisitorMessage msg = new CreateVisitorMessage("SomeLibrary", "myvisitor", "newyork", "london");

-            msg.setDocumentSelection("true and false or true");

-            msg.getParameters().put("myvar", Utf8.toBytes("somevalue"));

-            msg.getParameters().put("anothervar", Utf8.toBytes("34"));

-            msg.getBuckets().add(new BucketId(16, 1234));

-            msg.setVisitRemoves(true);

-            msg.setFieldSet("foo bar");

-            msg.setVisitorOrdering(OrderingSpecification.DESCENDING);

-            msg.setMaxBucketsPerVisitor(2);

-            assertEquals(BASE_MESSAGE_LENGTH + 178, serialize("CreateVisitorMessage", msg));

-

-            for (Language lang : LANGUAGES) {

-                msg = (CreateVisitorMessage)deserialize("CreateVisitorMessage", DocumentProtocol.MESSAGE_CREATEVISITOR, lang);

-                assertEquals("SomeLibrary", msg.getLibraryName());

-                assertEquals("myvisitor", msg.getInstanceId());

-                assertEquals("newyork", msg.getControlDestination());

-                assertEquals("london", msg.getDataDestination());

-                assertEquals("true and false or true", msg.getDocumentSelection());

-                assertEquals(8, msg.getMaxPendingReplyCount());

-                assertEquals(true, msg.getVisitRemoves());

-                assertEquals("foo bar", msg.getFieldSet());

-                assertEquals(false, msg.getVisitInconsistentBuckets());

-                assertEquals(1, msg.getBuckets().size());

-                assertEquals(new BucketId(16, 1234), msg.getBuckets().iterator().next());

-                assertEquals("somevalue", Utf8.toString(msg.getParameters().get("myvar")));

-                assertEquals("34", Utf8.toString(msg.getParameters().get("anothervar")));

-                assertEquals(OrderingSpecification.DESCENDING, msg.getVisitorOrdering());

-                assertEquals(2, msg.getMaxBucketsPerVisitor());

-            }

-        }

-    }

-

-    public class testGetDocumentMessage implements RunnableTest {

-

-        @Override

-        public void run() {

-            GetDocumentMessage msg = new GetDocumentMessage(new DocumentId("doc:scheme:"), "foo bar");

-            assertEquals(BASE_MESSAGE_LENGTH + 27, serialize("GetDocumentMessage", msg));

-

-            for (Language lang : LANGUAGES) {

-                msg = (GetDocumentMessage)deserialize("GetDocumentMessage", DocumentProtocol.MESSAGE_GETDOCUMENT, lang);

-                assertEquals("doc:scheme:", msg.getDocumentId().toString());

-                assertEquals("foo bar", msg.getFieldSet());

-            }

-        }

-    }

-

-    public class testDocumentIgnoredReply implements RunnableTest {

-

-        @Override

-        public void run() {

-            DocumentIgnoredReply reply = new DocumentIgnoredReply();

-            assertEquals(BASE_MESSAGE_LENGTH, serialize("DocumentIgnoredReply", reply));

-

-            for (Language lang : LANGUAGES) {

-                reply = (DocumentIgnoredReply)deserialize("DocumentIgnoredReply", DocumentProtocol.REPLY_DOCUMENTIGNORED, lang);

-            }

-        }

-    }

-}

+package com.yahoo.documentapi.messagebus.protocol.test;
+
+import com.yahoo.component.Version;
+import com.yahoo.document.BucketId;
+import com.yahoo.document.DocumentId;
+import com.yahoo.document.select.OrderingSpecification;
+import com.yahoo.documentapi.messagebus.protocol.CreateVisitorMessage;
+import com.yahoo.documentapi.messagebus.protocol.DocumentIgnoredReply;
+import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol;
+import com.yahoo.documentapi.messagebus.protocol.GetDocumentMessage;
+import com.yahoo.text.Utf8;
+
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class Messages51TestCase extends Messages50TestCase {
+
+    ////////////////////////////////////////////////////////////////////////////////
+    //
+    // Setup
+    //
+    ///////////////////////////////////////////////////////////////////////////////
+
+    @Override
+    protected void registerTests(Map<Integer, RunnableTest> out) {
+        super.registerTests(out);
+
+        // This list MUST mirror the list of routable factories from the DocumentProtocol constructor that support
+        // version 5.0. When adding tests to this list, please KEEP THEM ORDERED alphabetically like they are now.
+        out.put(DocumentProtocol.MESSAGE_CREATEVISITOR, new testCreateVisitorMessage());
+        out.put(DocumentProtocol.MESSAGE_GETDOCUMENT, new testGetDocumentMessage());
+        out.put(DocumentProtocol.REPLY_DOCUMENTIGNORED, new testDocumentIgnoredReply());
+    }
+
+    @Override
+    protected Version version() {
+        return new Version(5, 1);
+    }
+
+    @Override
+    protected boolean shouldTestCoverage() {
+        return true;
+    }
+
+    ////////////////////////////////////////////////////////////////////////////////
+    //
+    // Tests
+    //
+    ////////////////////////////////////////////////////////////////////////////////
+
+    private static int BASE_MESSAGE_LENGTH = 5;
+
+    public class testCreateVisitorMessage implements RunnableTest {
+
+        @Override
+        public void run() {
+            CreateVisitorMessage msg = new CreateVisitorMessage("SomeLibrary", "myvisitor", "newyork", "london");
+            msg.setDocumentSelection("true and false or true");
+            msg.getParameters().put("myvar", Utf8.toBytes("somevalue"));
+            msg.getParameters().put("anothervar", Utf8.toBytes("34"));
+            msg.getBuckets().add(new BucketId(16, 1234));
+            msg.setVisitRemoves(true);
+            msg.setFieldSet("foo bar");
+            msg.setVisitorOrdering(OrderingSpecification.DESCENDING);
+            msg.setMaxBucketsPerVisitor(2);
+            assertEquals(BASE_MESSAGE_LENGTH + 178, serialize("CreateVisitorMessage", msg));
+
+            for (Language lang : LANGUAGES) {
+                msg = (CreateVisitorMessage)deserialize("CreateVisitorMessage", DocumentProtocol.MESSAGE_CREATEVISITOR, lang);
+                assertEquals("SomeLibrary", msg.getLibraryName());
+                assertEquals("myvisitor", msg.getInstanceId());
+                assertEquals("newyork", msg.getControlDestination());
+                assertEquals("london", msg.getDataDestination());
+                assertEquals("true and false or true", msg.getDocumentSelection());
+                assertEquals(8, msg.getMaxPendingReplyCount());
+                assertEquals(true, msg.getVisitRemoves());
+                assertEquals("foo bar", msg.getFieldSet());
+                assertEquals(false, msg.getVisitInconsistentBuckets());
+                assertEquals(1, msg.getBuckets().size());
+                assertEquals(new BucketId(16, 1234), msg.getBuckets().iterator().next());
+                assertEquals("somevalue", Utf8.toString(msg.getParameters().get("myvar")));
+                assertEquals("34", Utf8.toString(msg.getParameters().get("anothervar")));
+                assertEquals(OrderingSpecification.DESCENDING, msg.getVisitorOrdering());
+                assertEquals(2, msg.getMaxBucketsPerVisitor());
+            }
+        }
+    }
+
+    public class testGetDocumentMessage implements RunnableTest {
+
+        @Override
+        public void run() {
+            GetDocumentMessage msg = new GetDocumentMessage(new DocumentId("doc:scheme:"), "foo bar");
+            assertEquals(BASE_MESSAGE_LENGTH + 27, serialize("GetDocumentMessage", msg));
+
+            for (Language lang : LANGUAGES) {
+                msg = (GetDocumentMessage)deserialize("GetDocumentMessage", DocumentProtocol.MESSAGE_GETDOCUMENT, lang);
+                assertEquals("doc:scheme:", msg.getDocumentId().toString());
+                assertEquals("foo bar", msg.getFieldSet());
+            }
+        }
+    }
+
+    public class testDocumentIgnoredReply implements RunnableTest {
+
+        @Override
+        public void run() {
+            DocumentIgnoredReply reply = new DocumentIgnoredReply();
+            assertEquals(BASE_MESSAGE_LENGTH, serialize("DocumentIgnoredReply", reply));
+
+            for (Language lang : LANGUAGES) {
+                reply = (DocumentIgnoredReply)deserialize("DocumentIgnoredReply", DocumentProtocol.REPLY_DOCUMENTIGNORED, lang);
+            }
+        }
+    }
+}
diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages52TestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages52TestCase.java
index 8198044a25e..e98bdd35254 100644
--- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages52TestCase.java
+++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages52TestCase.java
@@ -1,108 +1,108 @@
-// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.

-package com.yahoo.documentapi.messagebus.protocol.test;

-

-import com.google.common.annotations.Beta;

-import com.yahoo.component.Version;

-import com.yahoo.document.*;

-import com.yahoo.document.fieldpathupdate.RemoveFieldPathUpdate;

-import com.yahoo.documentapi.messagebus.protocol.*;

-

-import java.util.Map;

-

-import static org.junit.Assert.assertEquals;

-

-/**

- * @author Vegard Sjonfjell

- */

-

-@Beta

-public class Messages52TestCase extends Messages51TestCase {

-

-    @Override

-    protected Version version() {

-        return new Version(5, 115, 0);

-    }

-

-    @Override

-    protected boolean shouldTestCoverage() {

-        return true;

-    }

-

-    @Override

-    protected void registerTests(Map<Integer, RunnableTest> out) {

-        super.registerTests(out);

-

-        // This list MUST mirror the list of routable factories from the DocumentProtocol constructor that support

-        // version 5.2. When adding tests to this list, please KEEP THEM ORDERED alphabetically like they are now.

-

-        out.put(DocumentProtocol.MESSAGE_PUTDOCUMENT, new testPutDocumentMessage());

-        out.put(DocumentProtocol.MESSAGE_UPDATEDOCUMENT, new testUpdateDocumentMessage());

-        out.put(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, new testRemoveDocumentMessage());

-    }

-

-    private static int BASE_MESSAGE_LENGTH = 5;

-    private static String CONDITION_STRING = "There's just one condition";

-

-    public class testPutDocumentMessage implements RunnableTest {

-        @Override

-        public void run() {

-            PutDocumentMessage msg = new PutDocumentMessage(new DocumentPut(new Document(protocol.getDocumentTypeManager().getDocumentType("testdoc"), "doc:scheme:")));

-

-            msg.setTimestamp(666);

-            msg.setCondition(new TestAndSetCondition(CONDITION_STRING));

-

-            assertEquals(BASE_MESSAGE_LENGTH + 41 + serializedLength(msg.getCondition().getSelection()), serialize("PutDocumentMessage", msg));

-

-            for (Language lang : LANGUAGES) {

-                final PutDocumentMessage deserializedMsg = (PutDocumentMessage)deserialize("PutDocumentMessage", DocumentProtocol.MESSAGE_PUTDOCUMENT, lang);

-                assertEquals(msg.getDocumentPut().getDocument().getDataType().getName(), deserializedMsg.getDocumentPut().getDocument().getDataType().getName());

-                assertEquals(msg.getDocumentPut().getDocument().getId().toString(), deserializedMsg.getDocumentPut().getDocument().getId().toString());

-                assertEquals(msg.getTimestamp(), deserializedMsg.getTimestamp());

-                assertEquals(msg.getCondition().getSelection(), deserializedMsg.getCondition().getSelection());

-            }

-        }

-    }

-

-    public class testUpdateDocumentMessage implements RunnableTest {

-        @Override

-        public void run() {

-            DocumentType docType = protocol.getDocumentTypeManager().getDocumentType("testdoc");

-            DocumentUpdate update = new DocumentUpdate(docType, new DocumentId("doc:scheme:"));

-            update.addFieldPathUpdate(new RemoveFieldPathUpdate(docType, "intfield", "testdoc.intfield > 0"));

-

-            final UpdateDocumentMessage msg = new UpdateDocumentMessage(update);

-            msg.setNewTimestamp(777);

-            msg.setOldTimestamp(666);

-            msg.setCondition(new TestAndSetCondition(CONDITION_STRING));

-

-            assertEquals(BASE_MESSAGE_LENGTH + 89 + serializedLength(msg.getCondition().getSelection()), serialize("UpdateDocumentMessage", msg));

-

-            for (Language lang : LANGUAGES) {

-                final UpdateDocumentMessage deserializedMsg = (UpdateDocumentMessage) deserialize("UpdateDocumentMessage", DocumentProtocol.MESSAGE_UPDATEDOCUMENT, lang);

-                assertEquals(msg.getDocumentUpdate(), deserializedMsg.getDocumentUpdate());

-                assertEquals(msg.getNewTimestamp(), deserializedMsg.getNewTimestamp());

-                assertEquals(msg.getOldTimestamp(), deserializedMsg.getOldTimestamp());

-                assertEquals(msg.getCondition().getSelection(), deserializedMsg.getCondition().getSelection());

-            }

-        }

-    }

-

-    public class testRemoveDocumentMessage implements RunnableTest {

-        @Override

-        public void run() {

-            final RemoveDocumentMessage msg = new RemoveDocumentMessage(new DocumentId("doc:scheme:"));

-            msg.setCondition(new TestAndSetCondition(CONDITION_STRING));

-

-            assertEquals(BASE_MESSAGE_LENGTH + 16 + serializedLength(msg.getCondition().getSelection()), serialize("RemoveDocumentMessage", msg));

-

-            for (Language lang : LANGUAGES) {

-                final RemoveDocumentMessage deserializedMsg = (RemoveDocumentMessage)deserialize("RemoveDocumentMessage", DocumentProtocol.MESSAGE_REMOVEDOCUMENT, lang);

-                assertEquals(deserializedMsg.getDocumentId().toString(), msg.getDocumentId().toString());

-            }

-        }

-    }

-

-    static int serializedLength(String str) {

-        return 4 + str.length();

-    }

-}

+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.documentapi.messagebus.protocol.test;
+
+import com.google.common.annotations.Beta;
+import com.yahoo.component.Version;
+import com.yahoo.document.*;
+import com.yahoo.document.fieldpathupdate.RemoveFieldPathUpdate;
+import com.yahoo.documentapi.messagebus.protocol.*;
+
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author Vegard Sjonfjell
+ */
+
+@Beta
+public class Messages52TestCase extends Messages51TestCase {
+
+    @Override
+    protected Version version() {
+        return new Version(5, 115, 0);
+    }
+
+    @Override
+    protected boolean shouldTestCoverage() {
+        return true;
+    }
+
+    @Override
+    protected void registerTests(Map<Integer, RunnableTest> out) {
+        super.registerTests(out);
+
+        // This list MUST mirror the list of routable factories from the DocumentProtocol constructor that support
+        // version 5.2. When adding tests to this list, please KEEP THEM ORDERED alphabetically like they are now.
+
+        out.put(DocumentProtocol.MESSAGE_PUTDOCUMENT, new testPutDocumentMessage());
+        out.put(DocumentProtocol.MESSAGE_UPDATEDOCUMENT, new testUpdateDocumentMessage());
+        out.put(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, new testRemoveDocumentMessage());
+    }
+
+    private static int BASE_MESSAGE_LENGTH = 5;
+    private static String CONDITION_STRING = "There's just one condition";
+
+    public class testPutDocumentMessage implements RunnableTest {
+        @Override
+        public void run() {
+            PutDocumentMessage msg = new PutDocumentMessage(new DocumentPut(new Document(protocol.getDocumentTypeManager().getDocumentType("testdoc"), "doc:scheme:")));
+
+            msg.setTimestamp(666);
+            msg.setCondition(new TestAndSetCondition(CONDITION_STRING));
+
+            assertEquals(BASE_MESSAGE_LENGTH + 41 + serializedLength(msg.getCondition().getSelection()), serialize("PutDocumentMessage", msg));
+
+            for (Language lang : LANGUAGES) {
+                final PutDocumentMessage deserializedMsg = (PutDocumentMessage)deserialize("PutDocumentMessage", DocumentProtocol.MESSAGE_PUTDOCUMENT, lang);
+                assertEquals(msg.getDocumentPut().getDocument().getDataType().getName(), deserializedMsg.getDocumentPut().getDocument().getDataType().getName());
+                assertEquals(msg.getDocumentPut().getDocument().getId().toString(), deserializedMsg.getDocumentPut().getDocument().getId().toString());
+                assertEquals(msg.getTimestamp(), deserializedMsg.getTimestamp());
+                assertEquals(msg.getCondition().getSelection(), deserializedMsg.getCondition().getSelection());
+            }
+        }
+    }
+
+    public class testUpdateDocumentMessage implements RunnableTest {
+        @Override
+        public void run() {
+            DocumentType docType = protocol.getDocumentTypeManager().getDocumentType("testdoc");
+            DocumentUpdate update = new DocumentUpdate(docType, new DocumentId("doc:scheme:"));
+            update.addFieldPathUpdate(new RemoveFieldPathUpdate(docType, "intfield", "testdoc.intfield > 0"));
+
+            final UpdateDocumentMessage msg = new UpdateDocumentMessage(update);
+            msg.setNewTimestamp(777);
+            msg.setOldTimestamp(666);
+            msg.setCondition(new TestAndSetCondition(CONDITION_STRING));
+
+            assertEquals(BASE_MESSAGE_LENGTH + 89 + serializedLength(msg.getCondition().getSelection()), serialize("UpdateDocumentMessage", msg));
+
+            for (Language lang : LANGUAGES) {
+                final UpdateDocumentMessage deserializedMsg = (UpdateDocumentMessage) deserialize("UpdateDocumentMessage", DocumentProtocol.MESSAGE_UPDATEDOCUMENT, lang);
+                assertEquals(msg.getDocumentUpdate(), deserializedMsg.getDocumentUpdate());
+                assertEquals(msg.getNewTimestamp(), deserializedMsg.getNewTimestamp());
+                assertEquals(msg.getOldTimestamp(), deserializedMsg.getOldTimestamp());
+                assertEquals(msg.getCondition().getSelection(), deserializedMsg.getCondition().getSelection());
+            }
+        }
+    }
+
+    public class testRemoveDocumentMessage implements RunnableTest {
+        @Override
+        public void run() {
+            final RemoveDocumentMessage msg = new RemoveDocumentMessage(new DocumentId("doc:scheme:"));
+            msg.setCondition(new TestAndSetCondition(CONDITION_STRING));
+
+            assertEquals(BASE_MESSAGE_LENGTH + 16 + serializedLength(msg.getCondition().getSelection()), serialize("RemoveDocumentMessage", msg));
+
+            for (Language lang : LANGUAGES) {
+                final RemoveDocumentMessage deserializedMsg = (RemoveDocumentMessage)deserialize("RemoveDocumentMessage", DocumentProtocol.MESSAGE_REMOVEDOCUMENT, lang);
+                assertEquals(deserializedMsg.getDocumentId().toString(), msg.getDocumentId().toString());
+            }
+        }
+    }
+
+    static int serializedLength(String str) {
+        return 4 + str.length();
+    }
+}
diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/MessagesTestBase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/MessagesTestBase.java
index f69454a99f4..7b7ec353284 100755
--- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/MessagesTestBase.java
+++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/MessagesTestBase.java
@@ -1,161 +1,161 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.documentapi.messagebus.protocol.test;

-

-import com.yahoo.component.Version;

-import com.yahoo.document.DocumentTypeManager;

-import com.yahoo.document.DocumentTypeManagerConfigurer;

-import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet;

-import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol;

-import com.yahoo.messagebus.Routable;

-import org.junit.Test;

-

-import java.io.IOException;

-import java.util.*;

-

-import static org.junit.Assert.*;

-

-/**

- * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>

- */

-public abstract class MessagesTestBase {

-

-    protected enum Language {

-        JAVA,

-        CPP

-    }

-    protected static final Set<Language> LANGUAGES = EnumSet.allOf(Language.class);

-

-    protected final DocumentTypeManager docMan = new DocumentTypeManager();

-    protected final LoadTypeSet loadTypes = new LoadTypeSet();

-    protected final DocumentProtocol protocol = new DocumentProtocol(docMan, null, loadTypes);

-

-    public MessagesTestBase() {

-        DocumentTypeManagerConfigurer.configure(docMan, "file:./test/cfg/testdoc.cfg");

-        loadTypes.addLoadType(34, "foo", DocumentProtocol.Priority.NORMAL_2);

-    }

-

-    @Test

-    public void requireThatTestsPass() throws Exception {

-        Map<Integer, RunnableTest> tests = new TreeMap<>();

-        registerTests(tests);

-        for (Map.Entry<Integer, RunnableTest> entry : tests.entrySet()) {

-            entry.getValue().run();

-        }

-        if (shouldTestCoverage()) {

-            assertCoverage(protocol.getRoutableTypes(version()), new ArrayList<>(tests.keySet()));

-        }

-    }

-

-    /**

-     * Returns the version to use for serialization.

-     *

-     * @return The version.

-     */

-    protected abstract Version version();

-

-    /**

-     * Registers the tests to run.

-     */

-    protected abstract void registerTests(Map<Integer, RunnableTest> out);

-

-    /**

-     * Returns whether or not to test message test coverage.

-     */

-    protected abstract boolean shouldTestCoverage();

-

-    /**

-     * Encodes the given routable using the current version of the test case.

-     *

-     * @param routable The routable to encode.

-     * @return The encoded data.

-     */

-    public byte[] encode(Routable routable) {

-        return protocol.encode(version(), routable);

-    }

-

-    /**

-     * Decodes the given byte array using the current version of the test case.

-     *

-     * @param data The data to decode.

-     * @return The decoded routable.

-     */

-    public Routable decode(byte[] data) {

-        return protocol.decode(version(), data);

-    }

-

-    public String getPath(String filename) {

-        return TestFileUtil.getPath(filename);

-    }

-

-    /**

-     * Writes the content of the given routable to the given file.

-     *

-     * @param filename The name of the file to write to.

-     * @param routable The routable to serialize.

-     * @return The size of the written file.

-     */

-    public int serialize(String filename, Routable routable) {

-        Version version = version();

-        String path = getPath(version + "-java-" + filename + ".dat");

-        System.out.println("Serializing to '" + path + "'..");

-        byte[] data = protocol.encode(version, routable);

-        assertNotNull(data);

-        assertTrue(data.length > 0);

-        try {

-            TestFileUtil.writeToFile(path, data);

-        } catch (IOException e) {

-            throw new AssertionError(e);

-        }

-        assertEquals(routable.getType(), protocol.decode(version, data).getType());

-        return data.length;

-    }

-

-    /**

-     * Reads the content of the given file and creates a corresponding routable.

-     *

-     * @param filename The name of the file to read from.

-     * @param classId  The type that the routable must decode as.

-     * @param lang     The language constant that dictates what file format to read from.

-     * @return The decoded routable.

-     */

-    public Routable deserialize(String filename, int classId, Language lang) {

-        Version version = version();

-        String path = getPath(version + "-" + (lang == Language.JAVA ? "java" : "cpp") + "-" + filename + ".dat");

-        System.out.println("Deserializing from '" + path + "'..");

-        byte[] data;

-        try {

-            data = TestFileUtil.readFile(path);

-        } catch (IOException e) {

-            throw new AssertionError(e);

-        }

-        Routable ret = protocol.decode(version, data);

-        assertNotNull(ret);

-        assertEquals(classId, ret.getType());

-        return ret;

-    }

-

-    private static void assertCoverage(List<Integer> registered, List<Integer> tested) {

-        boolean ok = true;

-        List<Integer> lst = new ArrayList<>(tested);

-        for (Integer type : registered) {

-            if (!lst.contains(type)) {

-                System.err.println("Routable type " + type + " is registered in DocumentProtocol but not tested.");

-                ok = false;

-            } else {

-                lst.remove(type);

-            }

-        }

-        if (!lst.isEmpty()) {

-            for (Integer type : lst) {

-                System.err.println("Routable type " + type + " is tested but not registered in DocumentProtocol.");

-            }

-            ok = false;

-        }

-        assertTrue(ok);

-    }

-

-    protected static interface RunnableTest {

-

-        public void run() throws Exception;

-    }

-}

+package com.yahoo.documentapi.messagebus.protocol.test;
+
+import com.yahoo.component.Version;
+import com.yahoo.document.DocumentTypeManager;
+import com.yahoo.document.DocumentTypeManagerConfigurer;
+import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet;
+import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol;
+import com.yahoo.messagebus.Routable;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.*;
+
+import static org.junit.Assert.*;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public abstract class MessagesTestBase {
+
+    protected enum Language {
+        JAVA,
+        CPP
+    }
+    protected static final Set<Language> LANGUAGES = EnumSet.allOf(Language.class);
+
+    protected final DocumentTypeManager docMan = new DocumentTypeManager();
+    protected final LoadTypeSet loadTypes = new LoadTypeSet();
+    protected final DocumentProtocol protocol = new DocumentProtocol(docMan, null, loadTypes);
+
+    public MessagesTestBase() {
+        DocumentTypeManagerConfigurer.configure(docMan, "file:./test/cfg/testdoc.cfg");
+        loadTypes.addLoadType(34, "foo", DocumentProtocol.Priority.NORMAL_2);
+    }
+
+    @Test
+    public void requireThatTestsPass() throws Exception {
+        Map<Integer, RunnableTest> tests = new TreeMap<>();
+        registerTests(tests);
+        for (Map.Entry<Integer, RunnableTest> entry : tests.entrySet()) {
+            entry.getValue().run();
+        }
+        if (shouldTestCoverage()) {
+            assertCoverage(protocol.getRoutableTypes(version()), new ArrayList<>(tests.keySet()));
+        }
+    }
+
+    /**
+     * Returns the version to use for serialization.
+     *
+     * @return The version.
+     */
+    protected abstract Version version();
+
+    /**
+     * Registers the tests to run.
+     */
+    protected abstract void registerTests(Map<Integer, RunnableTest> out);
+
+    /**
+     * Returns whether or not to test message test coverage.
+     */
+    protected abstract boolean shouldTestCoverage();
+
+    /**
+     * Encodes the given routable using the current version of the test case.
+     *
+     * @param routable The routable to encode.
+     * @return The encoded data.
+     */
+    public byte[] encode(Routable routable) {
+        return protocol.encode(version(), routable);
+    }
+
+    /**
+     * Decodes the given byte array using the current version of the test case.
+     *
+     * @param data The data to decode.
+     * @return The decoded routable.
+     */
+    public Routable decode(byte[] data) {
+        return protocol.decode(version(), data);
+    }
+
+    public String getPath(String filename) {
+        return TestFileUtil.getPath(filename);
+    }
+
+    /**
+     * Writes the content of the given routable to the given file.
+     *
+     * @param filename The name of the file to write to.
+     * @param routable The routable to serialize.
+     * @return The size of the written file.
+     */
+    public int serialize(String filename, Routable routable) {
+        Version version = version();
+        String path = getPath(version + "-java-" + filename + ".dat");
+        System.out.println("Serializing to '" + path + "'..");
+        byte[] data = protocol.encode(version, routable);
+        assertNotNull(data);
+        assertTrue(data.length > 0);
+        try {
+            TestFileUtil.writeToFile(path, data);
+        } catch (IOException e) {
+            throw new AssertionError(e);
+        }
+        assertEquals(routable.getType(), protocol.decode(version, data).getType());
+        return data.length;
+    }
+
+    /**
+     * Reads the content of the given file and creates a corresponding routable.
+     *
+     * @param filename The name of the file to read from.
+     * @param classId  The type that the routable must decode as.
+     * @param lang     The language constant that dictates what file format to read from.
+     * @return The decoded routable.
+     */
+    public Routable deserialize(String filename, int classId, Language lang) {
+        Version version = version();
+        String path = getPath(version + "-" + (lang == Language.JAVA ? "java" : "cpp") + "-" + filename + ".dat");
+        System.out.println("Deserializing from '" + path + "'..");
+        byte[] data;
+        try {
+            data = TestFileUtil.readFile(path);
+        } catch (IOException e) {
+            throw new AssertionError(e);
+        }
+        Routable ret = protocol.decode(version, data);
+        assertNotNull(ret);
+        assertEquals(classId, ret.getType());
+        return ret;
+    }
+
+    private static void assertCoverage(List<Integer> registered, List<Integer> tested) {
+        boolean ok = true;
+        List<Integer> lst = new ArrayList<>(tested);
+        for (Integer type : registered) {
+            if (!lst.contains(type)) {
+                System.err.println("Routable type " + type + " is registered in DocumentProtocol but not tested.");
+                ok = false;
+            } else {
+                lst.remove(type);
+            }
+        }
+        if (!lst.isEmpty()) {
+            for (Integer type : lst) {
+                System.err.println("Routable type " + type + " is tested but not registered in DocumentProtocol.");
+            }
+            ok = false;
+        }
+        assertTrue(ok);
+    }
+
+    protected static interface RunnableTest {
+
+        public void run() throws Exception;
+    }
+}
diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyFactoryTestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyFactoryTestCase.java
index 3bbba1c0984..a2f5293bb98 100755
--- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyFactoryTestCase.java
+++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyFactoryTestCase.java
@@ -1,124 +1,124 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.documentapi.messagebus.protocol.test;

-

-import com.yahoo.document.DocumentId;

-import com.yahoo.document.DocumentTypeManager;

-import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol;

-import com.yahoo.documentapi.messagebus.protocol.DocumentProtocolRoutingPolicy;

-import com.yahoo.documentapi.messagebus.protocol.RemoveDocumentMessage;

-import com.yahoo.documentapi.messagebus.protocol.RoutingPolicyFactory;

-import com.yahoo.jrt.ListenFailedException;

-import com.yahoo.jrt.slobrok.server.Slobrok;

-import com.yahoo.messagebus.*;

-import com.yahoo.messagebus.metrics.MetricSet;

-import com.yahoo.messagebus.network.rpc.RPCNetworkParams;

-import com.yahoo.messagebus.network.rpc.test.TestServer;

-import com.yahoo.messagebus.routing.Route;

-import com.yahoo.messagebus.routing.RoutingContext;

-import com.yahoo.messagebus.test.Receptor;

-import com.yahoo.text.Utf8Array;

-import junit.framework.TestCase;

-

-/**

- * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>

- */

-public class PolicyFactoryTestCase extends TestCase {

-

-    ////////////////////////////////////////////////////////////////////////////////

-    //

-    // Setup

-    //

-    ////////////////////////////////////////////////////////////////////////////////

-

-    private Slobrok slobrok;

-    private TestServer srv;

-    private SourceSession src;

-

-    @Override

-    public void setUp() throws ListenFailedException {

-        slobrok = new Slobrok();

-        srv = new TestServer(new MessageBusParams().addProtocol(new DocumentProtocol(new DocumentTypeManager())),

-                             new RPCNetworkParams().setSlobrokConfigId(TestServer.getSlobrokConfig(slobrok)));

-        src = srv.mb.createSourceSession(new SourceSessionParams().setReplyHandler(new Receptor()));

-    }

-

-    @Override

-    public void tearDown() {

-        slobrok.stop();

-        src.destroy();

-        srv.destroy();

-    }

-

-    ////////////////////////////////////////////////////////////////////////////////

-    //

-    // Tests

-    //

-    ////////////////////////////////////////////////////////////////////////////////

-

-    public void testFactory() {

-        Route route = Route.parse("[MyPolicy]");

-        assertTrue(src.send(createMessage(), route).isAccepted());

-        Reply reply = ((Receptor)src.getReplyHandler()).getReply(60);

-        assertNotNull(reply);

-        System.out.println(reply.getTrace());

-        assertEquals(1, reply.getNumErrors());

-        assertEquals(ErrorCode.UNKNOWN_POLICY, reply.getError(0).getCode());

-

-        Protocol obj = srv.mb.getProtocol(DocumentProtocol.NAME);

-        assertTrue(obj instanceof DocumentProtocol);

-        DocumentProtocol protocol = (DocumentProtocol)obj;

-        protocol.putRoutingPolicyFactory("MyPolicy", new MyFactory());

-

-        assertTrue(src.send(createMessage(), route).isAccepted());

-        assertNotNull(reply = ((Receptor)src.getReplyHandler()).getReply(60));

-        System.out.println(reply.getTrace());

-        assertEquals(1, reply.getNumErrors());

-        assertEquals(DocumentProtocol.ERROR_POLICY_FAILURE, reply.getError(0).getCode());

-    }

-

-    ////////////////////////////////////////////////////////////////////////////////

-    //

-    // Utilities

-    //

-    ////////////////////////////////////////////////////////////////////////////////

-

-    private static Message createMessage() {

-        Message msg = new RemoveDocumentMessage(new DocumentId("doc:scheme:"));

-        msg.getTrace().setLevel(9);

-        return msg;

-    }

-

-    private static class MyFactory implements RoutingPolicyFactory {

-

-        public DocumentProtocolRoutingPolicy createPolicy(String param) {

-            return new MyPolicy(param);

-        }

-

-        public void destroy() {

-        }

-    }

-

-    private static class MyPolicy implements DocumentProtocolRoutingPolicy {

-

-        private final String param;

-

-        private MyPolicy(String param) {

-            this.param = param;

-        }

-

-        public void select(RoutingContext ctx) {

-            ctx.setError(DocumentProtocol.ERROR_POLICY_FAILURE, param);

-        }

-

-        public void merge(RoutingContext ctx) {

-            throw new AssertionError("Routing passed terminated select.");

-        }

-

-        public void destroy() {

-        }

-

-        public MetricSet getMetrics() {

-            return null;

-        }

-    }

-}

+package com.yahoo.documentapi.messagebus.protocol.test;
+
+import com.yahoo.document.DocumentId;
+import com.yahoo.document.DocumentTypeManager;
+import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol;
+import com.yahoo.documentapi.messagebus.protocol.DocumentProtocolRoutingPolicy;
+import com.yahoo.documentapi.messagebus.protocol.RemoveDocumentMessage;
+import com.yahoo.documentapi.messagebus.protocol.RoutingPolicyFactory;
+import com.yahoo.jrt.ListenFailedException;
+import com.yahoo.jrt.slobrok.server.Slobrok;
+import com.yahoo.messagebus.*;
+import com.yahoo.messagebus.metrics.MetricSet;
+import com.yahoo.messagebus.network.rpc.RPCNetworkParams;
+import com.yahoo.messagebus.network.rpc.test.TestServer;
+import com.yahoo.messagebus.routing.Route;
+import com.yahoo.messagebus.routing.RoutingContext;
+import com.yahoo.messagebus.test.Receptor;
+import com.yahoo.text.Utf8Array;
+import junit.framework.TestCase;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class PolicyFactoryTestCase extends TestCase {
+
+    ////////////////////////////////////////////////////////////////////////////////
+    //
+    // Setup
+    //
+    ////////////////////////////////////////////////////////////////////////////////
+
+    private Slobrok slobrok;
+    private TestServer srv;
+    private SourceSession src;
+
+    @Override
+    public void setUp() throws ListenFailedException {
+        slobrok = new Slobrok();
+        srv = new TestServer(new MessageBusParams().addProtocol(new DocumentProtocol(new DocumentTypeManager())),
+                             new RPCNetworkParams().setSlobrokConfigId(TestServer.getSlobrokConfig(slobrok)));
+        src = srv.mb.createSourceSession(new SourceSessionParams().setReplyHandler(new Receptor()));
+    }
+
+    @Override
+    public void tearDown() {
+        slobrok.stop();
+        src.destroy();
+        srv.destroy();
+    }
+
+    ////////////////////////////////////////////////////////////////////////////////
+    //
+    // Tests
+    //
+    ////////////////////////////////////////////////////////////////////////////////
+
+    public void testFactory() {
+        Route route = Route.parse("[MyPolicy]");
+        assertTrue(src.send(createMessage(), route).isAccepted());
+        Reply reply = ((Receptor)src.getReplyHandler()).getReply(60);
+        assertNotNull(reply);
+        System.out.println(reply.getTrace());
+        assertEquals(1, reply.getNumErrors());
+        assertEquals(ErrorCode.UNKNOWN_POLICY, reply.getError(0).getCode());
+
+        Protocol obj = srv.mb.getProtocol(DocumentProtocol.NAME);
+        assertTrue(obj instanceof DocumentProtocol);
+        DocumentProtocol protocol = (DocumentProtocol)obj;
+        protocol.putRoutingPolicyFactory("MyPolicy", new MyFactory());
+
+        assertTrue(src.send(createMessage(), route).isAccepted());
+        assertNotNull(reply = ((Receptor)src.getReplyHandler()).getReply(60));
+        System.out.println(reply.getTrace());
+        assertEquals(1, reply.getNumErrors());
+        assertEquals(DocumentProtocol.ERROR_POLICY_FAILURE, reply.getError(0).getCode());
+    }
+
+    ////////////////////////////////////////////////////////////////////////////////
+    //
+    // Utilities
+    //
+    ////////////////////////////////////////////////////////////////////////////////
+
+    private static Message createMessage() {
+        Message msg = new RemoveDocumentMessage(new DocumentId("doc:scheme:"));
+        msg.getTrace().setLevel(9);
+        return msg;
+    }
+
+    private static class MyFactory implements RoutingPolicyFactory {
+
+        public DocumentProtocolRoutingPolicy createPolicy(String param) {
+            return new MyPolicy(param);
+        }
+
+        public void destroy() {
+        }
+    }
+
+    private static class MyPolicy implements DocumentProtocolRoutingPolicy {
+
+        private final String param;
+
+        private MyPolicy(String param) {
+            this.param = param;
+        }
+
+        public void select(RoutingContext ctx) {
+            ctx.setError(DocumentProtocol.ERROR_POLICY_FAILURE, param);
+        }
+
+        public void merge(RoutingContext ctx) {
+            throw new AssertionError("Routing passed terminated select.");
+        }
+
+        public void destroy() {
+        }
+
+        public MetricSet getMetrics() {
+            return null;
+        }
+    }
+}
diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyTestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyTestCase.java
index ef1cdd2818e..82573600d4c 100755
--- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyTestCase.java
+++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyTestCase.java
@@ -1,902 +1,902 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.documentapi.messagebus.protocol.test;

-

-import com.yahoo.document.*;

-import com.yahoo.documentapi.messagebus.protocol.*;

-import com.yahoo.jrt.ListenFailedException;

-import com.yahoo.jrt.slobrok.api.IMirror;

-import com.yahoo.jrt.slobrok.api.Mirror;

-import com.yahoo.jrt.slobrok.server.Slobrok;

-import com.yahoo.messagebus.*;

-import com.yahoo.messagebus.Error;

-import com.yahoo.messagebus.network.rpc.test.TestServer;

-import com.yahoo.messagebus.routing.*;

-import com.yahoo.messagebus.test.Receptor;

-import com.yahoo.vdslib.DocumentList;

-import com.yahoo.vdslib.Entry;

-import org.junit.Before;

-import org.junit.Test;

-

-import java.util.*;

-import java.util.concurrent.TimeUnit;

-import java.util.concurrent.TimeoutException;

-

-import static org.junit.Assert.*;

-import static org.junit.Assert.assertEquals;

-import static org.junit.Assert.assertNull;

-

-/**

- * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>

- */

-@SuppressWarnings("deprecation")

-public class PolicyTestCase {

-

-    private static final int TIMEOUT = 300;

-    private static final TimeUnit TIMEOUT_UNIT = TimeUnit.SECONDS;

-    private static final long TIMEOUT_MILLIS = TIMEOUT_UNIT.toMillis(TIMEOUT);

-    private final DocumentTypeManager manager = new DocumentTypeManager();

-

-    @Before

-    public void setUp() {

-        DocumentTypeManagerConfigurer.configure(manager, "file:./test/cfg/testdoc.cfg");

-    }

-

-    @Test

-    public void testProtocol() {

-        DocumentProtocol protocol = new DocumentProtocol(manager);

-

-        RoutingPolicy policy = protocol.createPolicy("AND", null);

-        assertTrue(policy instanceof ANDPolicy);

-

-        policy = new DocumentProtocol(manager).createPolicy("DocumentRouteSelector", "raw:route[0]\n");

-        assertTrue(policy instanceof DocumentRouteSelectorPolicy);

-

-        policy = new DocumentProtocol(manager).createPolicy("Extern", "foo;bar/baz");

-        assertTrue(policy instanceof ExternPolicy);

-

-        policy = new DocumentProtocol(manager).createPolicy("LocalService", null);

-        assertTrue(policy instanceof LocalServicePolicy);

-

-        policy = new DocumentProtocol(manager).createPolicy("RoundRobin", null);

-        assertTrue(policy instanceof RoundRobinPolicy);

-

-        policy = new DocumentProtocol(manager).createPolicy("SearchRow", null);

-        assertTrue(policy instanceof SearchRowPolicy);

-

-        policy = new DocumentProtocol(manager).createPolicy("SearchColumn", null);

-        assertTrue(policy instanceof SearchColumnPolicy);

-

-        policy = new DocumentProtocol(manager).createPolicy("SubsetService", null);

-        assertTrue(policy instanceof SubsetServicePolicy);

-

-        policy = new DocumentProtocol(manager).createPolicy("LoadBalancer", null);

-        assertTrue(policy instanceof LoadBalancerPolicy);

-    }

-

-    @Test

-    public void testAND() {

-        PolicyTestFrame frame = new PolicyTestFrame(manager);

-        frame.setMessage(new PutDocumentMessage(new DocumentPut(new Document(manager.getDocumentType("testdoc"),

-                                                             new DocumentId("doc:scheme:")))));

-        frame.setHop(new HopSpec("test", "[AND]")

-                     .addRecipient("foo")

-                     .addRecipient("bar"));

-        frame.assertSelect(Arrays.asList("foo", "bar"));

-

-        frame.setHop(new HopSpec("test", "[AND:baz]")

-                     .addRecipient("foo")

-                     .addRecipient("bar"));

-        frame.assertSelect(Arrays.asList("baz")); // param precedes recipients

-

-        frame.setHop(new HopSpec("test", "[AND:foo]"));

-        frame.assertMergeOneReply("foo");

-

-        frame.setHop(new HopSpec("test", "[AND:foo bar]"));

-        frame.assertMergeTwoReplies("foo", "bar");

-        frame.destroy();

-    }

-

-    @Test

-    public void requireThatExternPolicyWithIllegalParamIsAnErrorPolicy() throws ListenFailedException {

-        Slobrok slobrok = new Slobrok();

-        String spec = "tcp/localhost:" + slobrok.port();

-        assertTrue(new DocumentProtocol(manager).createPolicy("Extern", null) instanceof ErrorPolicy);

-        assertTrue(new DocumentProtocol(manager).createPolicy("Extern", "") instanceof ErrorPolicy);

-        assertTrue(new DocumentProtocol(manager).createPolicy("Extern", spec) instanceof ErrorPolicy);

-        assertTrue(new DocumentProtocol(manager).createPolicy("Extern", spec + ";") instanceof ErrorPolicy);

-        assertTrue(new DocumentProtocol(manager).createPolicy("Extern", spec + ";bar") instanceof ErrorPolicy);

-    }

-

-    @Test

-    public void requireThatExternPolicyWithUnknownPatternSelectsNone() throws Exception {

-        PolicyTestFrame frame = newPutDocumentFrame("doc:scheme:");

-        setupExternPolicy(frame, new Slobrok(), "foo/bar");

-        frame.assertSelect(null);

-    }

-

-    @Test

-    public void requireThatExternPolicySelectsFromExternSlobrok() throws Exception {

-        PolicyTestFrame frame = newPutDocumentFrame("doc:scheme:");

-        Slobrok slobrok = new Slobrok();

-        List<TestServer> servers = new ArrayList<>();

-        for (int i = 0; i < 10; ++i) {

-            TestServer server = new TestServer("docproc/cluster.default/" + i, null, slobrok, null,

-                                               new DocumentProtocol(manager));

-            server.net.registerSession("chain.default");

-            servers.add(server);

-        }

-        setupExternPolicy(frame, slobrok, "docproc/cluster.default/*/chain.default", 10);

-        Set<String> lst = new HashSet<>();

-        for (int i = 0; i < 10; ++i) {

-            RoutingNode leaf = frame.select(1).get(0);

-            String recipient = leaf.getRoute().toString();

-            lst.add(recipient);

-

-            leaf.handleReply(new EmptyReply());

-            assertNotNull(frame.getReceptor().getReply(TIMEOUT));

-        }

-        assertEquals(10, lst.size());

-        for (TestServer server : servers) {

-            server.destroy();

-        }

-        frame.destroy();

-    }

-    

-    @Test

-    public void requireThatExternPolicyMergesOneReplyAsProtocol() throws Exception {

-        PolicyTestFrame frame = newPutDocumentFrame("doc:scheme:");

-        Slobrok slobrok = new Slobrok();

-        TestServer server = new TestServer("docproc/cluster.default/0", null, slobrok, null,

-                                           new DocumentProtocol(manager));

-        server.net.registerSession("chain.default");

-        setupExternPolicy(frame, slobrok, "docproc/cluster.default/*/chain.default", 1);

-        frame.assertMergeOneReply(server.net.getConnectionSpec() + "/chain.default");

-        server.destroy();

-        frame.destroy();

-    }

-

-    @Test

-    public void testExternSend() throws Exception {

-        // Setup local source node.

-        Slobrok local = new Slobrok();

-        TestServer src = new TestServer("src", null, local, null, new DocumentProtocol(manager));

-        SourceSession ss = src.mb.createSourceSession(new Receptor(), new SourceSessionParams().setTimeout(TIMEOUT));

-

-        // Setup remote cluster with routing config.

-        Slobrok slobrok = new Slobrok();

-        TestServer itr = new TestServer("itr",

-                                        new RoutingTableSpec(DocumentProtocol.NAME)

-                                                .addRoute(new RouteSpec("default").addHop("dst"))

-                                                .addHop(new HopSpec("dst", "dst/session")),

-                                        slobrok, null, new DocumentProtocol(manager));

-        IntermediateSession is = itr.mb.createIntermediateSession("session", true, new Receptor(), new Receptor());

-        TestServer dst = new TestServer("dst", null, slobrok, null, new DocumentProtocol(manager));

-        DestinationSession ds = dst.mb.createDestinationSession("session", true, new Receptor());

-

-        // Send message from local node to remote cluster and resolve route there.

-        Message msg = new RemoveDocumentMessage(new DocumentId("doc:scheme:"));

-        msg.getTrace().setLevel(9);

-        msg.setRoute(Route.parse("[Extern:tcp/localhost:" + slobrok.port() + ";itr/session] default"));

-

-        assertTrue(ss.send(msg).isAccepted());

-        assertNotNull(msg = ((Receptor)is.getMessageHandler()).getMessage(TIMEOUT));

-        is.forward(msg);

-        assertNotNull(msg = ((Receptor)ds.getMessageHandler()).getMessage(TIMEOUT));

-        ds.acknowledge(msg);

-        Reply reply = ((Receptor)is.getReplyHandler()).getReply(TIMEOUT);

-        assertNotNull(reply);

-        is.forward(reply);

-        assertNotNull(reply = ((Receptor)ss.getReplyHandler()).getReply(TIMEOUT));

-

-        System.out.println(reply.getTrace().toString());

-

-        // Perform necessary cleanup.

-        src.destroy();

-        itr.destroy();

-        dst.destroy();

-        slobrok.stop();

-        local.stop();

-    }

-

-    @Test

-    public void testExternMultipleSlobroks() throws ListenFailedException {

-        Slobrok local = new Slobrok();

-        TestServer srcServer = new TestServer("src", null, local, null, new DocumentProtocol(manager));

-        SourceSession srcSession =

-                srcServer.mb.createSourceSession(new Receptor(), new SourceSessionParams().setTimeout(TIMEOUT));

-

-        Slobrok extern = new Slobrok();

-        String spec = "tcp/localhost:" + extern.port();

-

-        TestServer dstServer = new TestServer("dst", null, extern, null, new DocumentProtocol(manager));

-        Receptor dstHandler = new Receptor();

-        DestinationSession dstSession = dstServer.mb.createDestinationSession("session", true, dstHandler);

-

-        Message msg = new RemoveDocumentMessage(new DocumentId("doc:scheme:"));

-        msg.setRoute(Route.parse("[Extern:" + spec + ";dst/session]"));

-        assertTrue(srcSession.send(msg).isAccepted());

-        assertNotNull(msg = dstHandler.getMessage(TIMEOUT));

-        dstSession.acknowledge(msg);

-        Reply reply = ((Receptor)srcSession.getReplyHandler()).getReply(TIMEOUT);

-        assertNotNull(reply);

-

-        extern.stop();

-        dstSession.destroy();

-        dstServer.destroy();

-        dstHandler.reset();

-        assertNull(dstHandler.getMessage(0));

-

-        extern = new Slobrok();

-        spec += ",tcp/localhost:" + extern.port();

-

-        dstServer = new TestServer("dst", null, extern, null, new DocumentProtocol(manager));

-        dstHandler = new Receptor();

-        dstSession = dstServer.mb.createDestinationSession("session", true, dstHandler);

-

-        msg = new RemoveDocumentMessage(new DocumentId("doc:scheme:"));

-        msg.setRoute(Route.parse("[Extern:" + spec + ";dst/session]"));

-        assertTrue(srcSession.send(msg).isAccepted());

-        assertNotNull(msg = dstHandler.getMessage(TIMEOUT));

-        dstSession.acknowledge(msg);

-        reply = ((Receptor)srcSession.getReplyHandler()).getReply(TIMEOUT);

-        assertNotNull(reply);

-

-        extern.stop();

-        dstSession.destroy();

-        dstServer.destroy();

-

-        local.stop();

-        srcSession.destroy();

-        srcServer.destroy();

-    }

-

-    @Test

-    public void testLocalService() {

-        // Test select with proper address.

-        PolicyTestFrame frame = new PolicyTestFrame("docproc/cluster.default", manager);

-        frame.setMessage(new PutDocumentMessage(new DocumentPut(new Document(manager.getDocumentType("testdoc"),

-                                                             new DocumentId("doc:scheme:0")))));

-        for (int i = 0; i < 10; ++i) {

-            frame.getNetwork().registerSession(i + "/chain.default");

-        }

-        assertTrue(frame.waitSlobrok("docproc/cluster.default/*/chain.default", 10));

-        frame.setHop(new HopSpec("test", "docproc/cluster.default/[LocalService]/chain.default"));

-

-        Set<String> lst = new HashSet<>();

-        for (int i = 0; i < 10; ++i) {

-            RoutingNode leaf = frame.select(1).get(0);

-            String recipient = leaf.getRoute().toString();

-            lst.add(recipient);

-

-            leaf.handleReply(new EmptyReply());

-            assertNotNull(frame.getReceptor().getReply(TIMEOUT));

-        }

-        assertEquals(10, lst.size());

-

-        // Test select with broken address.

-        lst.clear();

-        frame.setHop(new HopSpec("test", "docproc/cluster.default/[LocalService:broken]/chain.default"));

-        for (int i = 0; i < 10; ++i) {

-            RoutingNode leaf = frame.select(1).get(0);

-            String recipient = leaf.getRoute().toString();

-            assertTrue(recipient.equals("docproc/cluster.default/*/chain.default"));

-            lst.add(recipient);

-

-            leaf.handleReply(new EmptyReply());

-            assertNotNull(frame.getReceptor().getReply(TIMEOUT));

-        }

-        assertEquals(1, lst.size());

-

-        // Test merge behavior.

-        frame.setMessage(new PutDocumentMessage(new DocumentPut(new Document(manager.getDocumentType("testdoc"),

-                                                             new DocumentId("doc:scheme:")))));

-        frame.setHop(new HopSpec("test", "[LocalService]"));

-        frame.assertMergeOneReply("*");

-

-        frame.destroy();

-    }

-

-    @Test

-    public void testLocalServiceCache() {

-        PolicyTestFrame fooFrame = new PolicyTestFrame("docproc/cluster.default", manager);

-        HopSpec fooHop = new HopSpec("foo", "docproc/cluster.default/[LocalService]/chain.foo");

-        fooFrame.setMessage(new RemoveDocumentMessage(new DocumentId("doc:scheme:foo")));

-        fooFrame.setHop(fooHop);

-

-        PolicyTestFrame barFrame = new PolicyTestFrame(fooFrame);

-        HopSpec barHop = new HopSpec("bar", "docproc/cluster.default/[LocalService]/chain.bar");

-        barFrame.setMessage(new RemoveDocumentMessage(new DocumentId("doc:scheme:bar")));

-        barFrame.setHop(barHop);

-

-        fooFrame.getMessageBus().setupRouting(

-                new RoutingSpec().addTable(new RoutingTableSpec(DocumentProtocol.NAME)

-                                                   .addHop(fooHop)

-                                                   .addHop(barHop)));

-

-        fooFrame.getNetwork().registerSession("0/chain.foo");

-        fooFrame.getNetwork().registerSession("0/chain.bar");

-        assertTrue(fooFrame.waitSlobrok("docproc/cluster.default/0/*", 2));

-

-        RoutingNode fooChild = fooFrame.select(1).get(0);

-        assertEquals("docproc/cluster.default/0/chain.foo", fooChild.getRoute().getHop(0).toString());

-        RoutingNode barChild = barFrame.select(1).get(0);

-        assertEquals("docproc/cluster.default/0/chain.bar", barChild.getRoute().getHop(0).toString());

-

-        barChild.handleReply(new EmptyReply());

-        fooChild.handleReply(new EmptyReply());

-

-        assertNotNull(barFrame.getReceptor().getReply(TIMEOUT));

-        assertNotNull(fooFrame.getReceptor().getReply(TIMEOUT));

-    }

-

-    @Test

-    public void testSearchRow() {

-        PolicyTestFrame frame = new PolicyTestFrame(manager);

-        frame.setMessage(new PutDocumentMessage(new DocumentPut(new Document(manager.getDocumentType("testdoc"),

-                                                             new DocumentId("doc:scheme:")))));

-        frame.setHop(new HopSpec("test", "[SearchRow]").addRecipient("foo"));

-        frame.assertMergeOneReply("foo");

-        frame.setHop(new HopSpec("test", "[SearchRow]").addRecipient("foo").addRecipient("bar"));

-        frame.assertMergeTwoReplies("foo", "bar");

-

-        frame.setHop(new HopSpec("test", "[SearchRow:1]").addRecipient("foo"));

-        Map<String, Integer> replies = new HashMap<>();

-        replies.put("foo", ErrorCode.SERVICE_OOS);

-        frame.assertMergeError(replies, Arrays.asList(ErrorCode.SERVICE_OOS));

-

-        frame.setHop(new HopSpec("test", "[SearchRow:1]").addRecipient("foo").addRecipient("bar"));

-        replies.put("foo", ErrorCode.SERVICE_OOS);

-        replies.put("bar", ErrorCode.NONE);

-        frame.assertMergeOk(replies, Arrays.asList("bar"));

-

-        replies.put("foo", ErrorCode.SERVICE_OOS);

-        replies.put("bar", ErrorCode.SERVICE_OOS);

-        frame.assertMergeError(replies, Arrays.asList(ErrorCode.SERVICE_OOS, ErrorCode.SERVICE_OOS));

-

-        frame.setHop(new HopSpec("test", "[SearchRow:1]").addRecipient("foo").addRecipient("bar").addRecipient("baz"));

-        replies.put("foo", ErrorCode.SERVICE_OOS);

-        replies.put("bar", ErrorCode.NONE);

-        replies.put("baz", ErrorCode.NONE);

-        frame.assertMergeOk(replies, Arrays.asList("bar", "baz"));

-

-        replies.put("foo", ErrorCode.SERVICE_OOS);

-        replies.put("bar", ErrorCode.SERVICE_OOS);

-        replies.put("baz", ErrorCode.NONE);

-        frame.assertMergeOk(replies, Arrays.asList("baz"));

-

-        replies.put("foo", ErrorCode.SERVICE_OOS);

-        replies.put("bar", ErrorCode.SERVICE_OOS);

-        replies.put("baz", ErrorCode.SERVICE_OOS);

-        frame.assertMergeError(replies,

-                               Arrays.asList(ErrorCode.SERVICE_OOS, ErrorCode.SERVICE_OOS, ErrorCode.SERVICE_OOS));

-

-        frame.setHop(new HopSpec("test", "[SearchRow:2]").addRecipient("foo").addRecipient("bar").addRecipient("baz"));

-        replies.put("foo", ErrorCode.SERVICE_OOS);

-        replies.put("bar", ErrorCode.NONE);

-        replies.put("baz", ErrorCode.NONE);

-        frame.assertMergeOk(replies, Arrays.asList("bar", "baz"));

-

-        replies.put("foo", ErrorCode.SERVICE_OOS);

-        replies.put("bar", ErrorCode.SERVICE_OOS);

-        replies.put("baz", ErrorCode.NONE);

-        frame.assertMergeError(replies, Arrays.asList(ErrorCode.SERVICE_OOS, ErrorCode.SERVICE_OOS));

-

-        replies.put("foo", ErrorCode.SERVICE_OOS);

-        replies.put("bar", ErrorCode.SERVICE_OOS);

-        replies.put("baz", ErrorCode.SERVICE_OOS);

-        frame.assertMergeError(replies,

-                               Arrays.asList(ErrorCode.SERVICE_OOS, ErrorCode.SERVICE_OOS, ErrorCode.SERVICE_OOS));

-

-        frame.destroy();

-    }

-

-    @Test

-    public void testSearchRowMerge() {

-        PolicyTestFrame frame = new PolicyTestFrame(manager);

-        frame.setHop(new HopSpec("test", "[SearchRow]").addRecipient("foo"));

-        tryWasFound(frame, 1, 0x0, false);

-        tryWasFound(frame, 1, 0x1, true);

-

-        frame.setHop(new HopSpec("test", "[SearchRow]").addRecipient("foo").addRecipient("bar"));

-        tryWasFound(frame, 2, 0x0, false);

-        tryWasFound(frame, 2, 0x1, true);

-        tryWasFound(frame, 2, 0x2, true);

-        tryWasFound(frame, 2, 0x3, true);

-

-        frame.setHop(new HopSpec("test", "[SearchRow]").addRecipient("foo").addRecipient("bar").addRecipient("baz"));

-        tryWasFound(frame, 3, 0x0, false);

-        tryWasFound(frame, 3, 0x1, true);

-        tryWasFound(frame, 3, 0x2, true);

-        tryWasFound(frame, 3, 0x3, true);

-        tryWasFound(frame, 3, 0x4, true);

-        tryWasFound(frame, 3, 0x5, true);

-        tryWasFound(frame, 3, 0x6, true);

-        tryWasFound(frame, 3, 0x7, true);

-        frame.destroy();

-    }

-    

-    private void tryWasFound(PolicyTestFrame frame, int expectedRecipients, 

-                             int foundMask, boolean expectedFound) 

-    {

-        {

-            frame.setMessage(new RemoveDocumentMessage(new DocumentId("doc:scheme:69")));

-            List<RoutingNode> selected = frame.select(expectedRecipients);

-            for (int i = 0, len = selected.size(); i < len; ++i) {

-                RemoveDocumentReply reply = new RemoveDocumentReply();

-                reply.setWasFound(((1 << i) & foundMask) != 0);

-                selected.get(i).handleReply(reply);

-            }

-            Reply reply = frame.getReceptor().getReply(TIMEOUT);

-            assertNotNull(reply);

-            assertEquals(DocumentProtocol.REPLY_REMOVEDOCUMENT, reply.getType());

-            assertEquals(expectedFound, ((RemoveDocumentReply)reply).wasFound());

-        }

-        {

-            DocumentUpdate  upd = new DocumentUpdate(manager.getDocumentType("testdoc"),

-                                                     new DocumentId("doc:scheme:"));

-            frame.setMessage(new UpdateDocumentMessage(upd));

-            List<RoutingNode> selected = frame.select(expectedRecipients);

-            for (int i = 0, len = selected.size(); i < len; ++i) {

-                UpdateDocumentReply reply = new UpdateDocumentReply();

-                reply.setWasFound(((1 << i) & foundMask) != 0);

-                selected.get(i).handleReply(reply);

-            }

-            Reply reply = frame.getReceptor().getReply(TIMEOUT);

-            assertNotNull(reply);

-            assertEquals(DocumentProtocol.REPLY_UPDATEDOCUMENT, reply.getType());

-            assertEquals(expectedFound, ((UpdateDocumentReply)reply).wasFound());

-        }

-    }

-

-    @Test

-    public void multipleGetRepliesAreMergedToFoundDocument() {

-        PolicyTestFrame frame = new PolicyTestFrame(manager);

-        frame.setHop(new HopSpec("test", getDocumentRouteSelectorRawConfig())

-                .addRecipient("foo").addRecipient("bar"));

-        frame.setMessage(new GetDocumentMessage(new DocumentId("doc:scheme:yarn"), "[all]"));

-        List<RoutingNode> selected = frame.select(2);

-        for (int i = 0, len = selected.size(); i < len; ++i) {

-            Document doc = null;

-            if (i == 0) {

-                doc = new Document(manager.getDocumentType("testdoc"),

-                                   new DocumentId("doc:scheme:yarn"));

-                doc.setLastModified(123456L);

-            }

-            GetDocumentReply reply = new GetDocumentReply(null);

-            reply.setDocument(doc);

-            selected.get(i).handleReply(reply);

-        }

-        Reply reply = frame.getReceptor().getReply(TIMEOUT);

-        assertNotNull(reply);

-        assertEquals(DocumentProtocol.REPLY_GETDOCUMENT, reply.getType());

-        assertEquals(123456L, ((GetDocumentReply)reply).getLastModified());

-    }

-

-    private String getDocumentRouteSelectorRawConfig() {

-        return "[DocumentRouteSelector:raw:" +

-                "route[2]\n" +

-                "route[0].name \"foo\"\n" +

-                "route[0].selector \"testdoc\"\n" +

-                "route[0].feed \"myfeed\"\n" +

-                "route[1].name \"bar\"\n" +

-                "route[1].selector \"other\"\n" +

-                "route[1].feed \"myfeed\"\n]";

-    }

-

-    @Test

-    public void testSearchColumn() {

-        PolicyTestFrame frame = new PolicyTestFrame(manager);

-        frame.setHop(new HopSpec("test", "[SearchColumn]")

-                .addRecipient("c0").addRecipient("c1")

-                .addRecipient("c2").addRecipient("c3"));

-

-        // Test hash distribution.

-        assertDistribution(frame, "doc:ns:3", "c0");

-        assertDistribution(frame, "doc:ns:18", "c1");

-        assertDistribution(frame, "doc:ns:0", "c2");

-        assertDistribution(frame, "doc:ns:4", "c3");

-

-        assertDistribution(frame, "userdoc:ns:49152:0", "c0");

-        assertDistribution(frame, "userdoc:ns:49152:1", "c0");

-        assertDistribution(frame, "userdoc:ns:16384:2", "c1");

-        assertDistribution(frame, "userdoc:ns:16384:3", "c1");

-        assertDistribution(frame, "userdoc:ns:5461:4", "c2");

-        assertDistribution(frame, "userdoc:ns:5461:5", "c2");

-        assertDistribution(frame, "userdoc:ns:0:6", "c3");

-        assertDistribution(frame, "userdoc:ns:0:7", "c3");

-

-        assertDistribution(frame, "groupdoc:ns:0:0", "c0");

-        assertDistribution(frame, "groupdoc:ns:0:1", "c0");

-        assertDistribution(frame, "groupdoc:ns:4:2", "c1");

-        assertDistribution(frame, "groupdoc:ns:4:3", "c1");

-        assertDistribution(frame, "groupdoc:ns:2:4", "c2");

-        assertDistribution(frame, "groupdoc:ns:2:5", "c2");

-        assertDistribution(frame, "groupdoc:ns:7:6", "c3");

-        assertDistribution(frame, "groupdoc:ns:7:7", "c3");

-

-        // Test routing based on message type.

-        Message put = new PutDocumentMessage(new DocumentPut(new Document(manager.getDocumentType("testdoc"),

-                                                          new DocumentId("doc:scheme:"))));

-        frame.setHop(new HopSpec("test", "[SearchColumn]").addRecipient("c0").addRecipient("c1"));

-        frame.setMessage(put);

-        frame.assertMergeOneReply("c0");

-

-        // Test allowed bad parts.

-        frame.setHop(new HopSpec("test", "[SearchColumn:1]").addRecipient("c0"));

-        frame.setMessage(put);

-        Map<String, Integer> replies = new HashMap<>();

-        replies.put("c0", ErrorCode.SERVICE_OOS);

-        frame.assertMergeOk(replies, null);

-

-        replies.put("c0", ErrorCode.SERVICE_OOS);

-        frame.assertMergeOk(replies, null);

-

-        frame.setHop(new HopSpec("test", "[SearchColumn:1]").addRecipient("c0").addRecipient("c1"));

-        frame.setMessage(put);

-        replies.put("c0", ErrorCode.SERVICE_OOS);

-        frame.assertMergeOk(replies, null);

-

-        frame.setHop(new HopSpec("test", "[SearchColumn:1]").addRecipient("c0").addRecipient("c1").addRecipient("c2"));

-        frame.setMessage(put);

-        replies.clear();

-        replies.put("c0", ErrorCode.SERVICE_OOS);

-        frame.assertMergeOk(replies, null);

-

-        frame.setHop(new HopSpec("test", "[SearchColumn:2]").addRecipient("c0").addRecipient("c1").addRecipient("c2"));

-        frame.setMessage(put);

-        replies.clear();

-        replies.put("c0", ErrorCode.SERVICE_OOS);

-        frame.assertMergeOk(replies, null);

-

-        frame.destroy();

-    }

-

-    private void assertDistribution(PolicyTestFrame frame, String id, String expected) {

-        frame.setMessage(new PutDocumentMessage(new DocumentPut(new Document(manager.getDocumentType("testdoc"),

-                                                             new DocumentId(id)))));

-        frame.assertSelect(Arrays.asList(expected));

-    }

-

-    @Test

-    public void testSubsetService() {

-        PolicyTestFrame frame = new PolicyTestFrame("docproc/cluster.default", manager);

-        frame.setMessage(new PutDocumentMessage(new DocumentPut(new DocumentPut(new Document(manager.getDocumentType("testdoc"),

-                                                             new DocumentId("doc:scheme:"))))));

-

-        // Test requerying for adding nodes.

-        frame.setHop(new HopSpec("test", "docproc/cluster.default/[SubsetService:2]/chain.default"));

-        Set<String> lst = new HashSet<>();

-        for (int i = 1; i <= 10; ++i) {

-            frame.getNetwork().registerSession(i + "/chain.default");

-            assertTrue(frame.waitSlobrok("docproc/cluster.default/*/chain.default", i));

-

-            RoutingNode leaf = frame.select(1).get(0);

-            lst.add(leaf.getRoute().toString());

-            leaf.handleReply(new EmptyReply());

-            assertNotNull(frame.getReceptor().getReply(TIMEOUT));

-        }

-        assertTrue(lst.size() > 1); // must have requeried

-

-        // Test load balancing.

-        String prev = null;

-        for (int i = 1; i <= 10; ++i) {

-            RoutingNode leaf = frame.select(1).get(0);

-            String next = leaf.getRoute().toString();

-            if (prev == null) {

-                assertNotNull(next);

-            } else {

-                assertFalse(prev.equals(next));

-            }

-            prev = next;

-            leaf.handleReply(new EmptyReply());

-            assertNotNull(frame.getReceptor().getReply(TIMEOUT));

-        }

-

-        // Test requerying for dropping nodes.

-        lst.clear();

-        for (int i = 1; i <= 10; ++i) {

-            RoutingNode leaf = frame.select(1).get(0);

-            String route = leaf.getRoute().toString();

-            lst.add(route);

-

-            frame.getNetwork().unregisterSession(route.substring(frame.getIdentity().length() + 1));

-            assertTrue(frame.waitSlobrok("docproc/cluster.default/*/chain.default", 10 - i));

-

-            Reply reply = new EmptyReply();

-            reply.addError(new Error(ErrorCode.NO_ADDRESS_FOR_SERVICE, route));

-            leaf.handleReply(reply);

-            assertNotNull(frame.getReceptor().getReply(TIMEOUT));

-        }

-        assertEquals(10, lst.size());

-

-        // Test merge behavior.

-        frame.setHop(new HopSpec("test", "[SubsetService]"));

-        frame.assertMergeOneReply("*");

-

-        frame.destroy();

-    }

-

-    @Test

-    public void testSubsetServiceCache() {

-        PolicyTestFrame fooFrame = new PolicyTestFrame("docproc/cluster.default", manager);

-        HopSpec fooHop = new HopSpec("foo", "docproc/cluster.default/[SubsetService:2]/chain.foo");

-        fooFrame.setMessage(new RemoveDocumentMessage(new DocumentId("doc:scheme:foo")));

-        fooFrame.setHop(fooHop);

-

-        PolicyTestFrame barFrame = new PolicyTestFrame(fooFrame);

-        HopSpec barHop = new HopSpec("bar", "docproc/cluster.default/[SubsetService:2]/chain.bar");

-        barFrame.setMessage(new RemoveDocumentMessage(new DocumentId("doc:scheme:bar")));

-        barFrame.setHop(barHop);

-

-        fooFrame.getMessageBus().setupRouting(

-                new RoutingSpec().addTable(new RoutingTableSpec(DocumentProtocol.NAME)

-                                                   .addHop(fooHop)

-                                                   .addHop(barHop)));

-

-        fooFrame.getNetwork().registerSession("0/chain.foo");

-        fooFrame.getNetwork().registerSession("0/chain.bar");

-        assertTrue(fooFrame.waitSlobrok("docproc/cluster.default/0/*", 2));

-

-        RoutingNode fooChild = fooFrame.select(1).get(0);

-        assertEquals("docproc/cluster.default/0/chain.foo", fooChild.getRoute().getHop(0).toString());

-        RoutingNode barChild = barFrame.select(1).get(0);

-        assertEquals("docproc/cluster.default/0/chain.bar", barChild.getRoute().getHop(0).toString());

-

-        barChild.handleReply(new EmptyReply());

-        fooChild.handleReply(new EmptyReply());

-

-        assertNotNull(barFrame.getReceptor().getReply(TIMEOUT));

-        assertNotNull(fooFrame.getReceptor().getReply(TIMEOUT));

-    }

-

-    @Test

-    public void testDocumentRouteSelector() {

-        // Test policy usage safeguard.

-        String okConfig = "raw:route[0]\n";

-        String errConfig = "raw:" +

-                           "route[1]\n" +

-                           "route[0].name \"foo\"\n" +

-                           "route[0].selector \"foo bar\"\n" +

-                           "route[0].feed \"baz\"\n";

-

-        DocumentProtocol protocol = new DocumentProtocol(manager, okConfig);

-        assertTrue(protocol.createPolicy("DocumentRouteSelector", null) instanceof DocumentRouteSelectorPolicy);

-        assertTrue(protocol.createPolicy("DocumentRouteSelector", "") instanceof DocumentRouteSelectorPolicy);

-        assertTrue(protocol.createPolicy("DocumentRouteSelector", errConfig) instanceof ErrorPolicy);

-

-        protocol = new DocumentProtocol(manager, errConfig);

-        assertTrue(protocol.createPolicy("DocumentRouteSelector", null) instanceof ErrorPolicy);

-        assertTrue(protocol.createPolicy("DocumentRouteSelector", "") instanceof ErrorPolicy);

-        assertTrue(protocol.createPolicy("DocumentRouteSelector", okConfig) instanceof DocumentRouteSelectorPolicy);

-

-        // Test policy with proper config.

-        PolicyTestFrame frame = new PolicyTestFrame(manager);

-        frame.setHop(new HopSpec("test", "[DocumentRouteSelector:raw:" +

-                                         "route[2]\n" +

-                                         "route[0].name \"foo\"\n" +

-                                         "route[0].selector \"testdoc\"\n" +

-                                         "route[0].feed \"myfeed\"\n" +

-                                         "route[1].name \"bar\"\n" +

-                                         "route[1].selector \"other\"\n" +

-                                         "route[1].feed \"myfeed\"\n]").addRecipient("foo").addRecipient("bar"));

-

-        frame.setMessage(new GetDocumentMessage(new DocumentId("doc:scheme:"), "fieldSet"));

-        frame.assertSelect(Arrays.asList("bar", "foo"));

-

-        Message put = new PutDocumentMessage(new DocumentPut(new Document(manager.getDocumentType("testdoc"),

-                                                          new DocumentId("doc:scheme:"))));

-        frame.setMessage(put);

-        frame.assertSelect(Arrays.asList("foo"));

-

-        frame.setMessage(new RemoveDocumentMessage(new DocumentId("doc:scheme:")));

-        frame.assertSelect(Arrays.asList("bar", "foo"));

-

-        frame.setMessage(new UpdateDocumentMessage(new DocumentUpdate(manager.getDocumentType("testdoc"),

-                                                                      new DocumentId("doc:scheme:"))));

-        frame.assertSelect(Arrays.asList("foo"));

-

-        frame.setMessage(put);

-        frame.assertMergeOneReply("foo");

-

-        frame.destroy();

-    }

-

-

-    @Test

-    public void testDocumentRouteSelectorIgnore() {

-        PolicyTestFrame frame = new PolicyTestFrame(manager);

-        frame.setHop(new HopSpec("test", "[DocumentRouteSelector:raw:" +

-                                         "route[1]\n" +

-                                         "route[0].name \"docproc/cluster.foo\"\n" +

-                                         "route[0].selector \"testdoc and testdoc.stringfield == 'foo'\"\n" +

-                                         "route[0].feed \"myfeed\"\n]").addRecipient("docproc/cluster.foo"));

-

-        frame.setMessage(new PutDocumentMessage(

-                new DocumentPut(new Document(manager.getDocumentType("testdoc"),

-                             new DocumentId("id:yarn:testdoc:n=1234:fluff")))));

-        frame.select(0);

-        Reply reply = frame.getReceptor().getReply(TIMEOUT);

-        assertNotNull(reply);

-        assertEquals(DocumentProtocol.REPLY_DOCUMENTIGNORED, reply.getType());

-        assertEquals(0, reply.getNumErrors());

-

-        frame.setMessage(new UpdateDocumentMessage(new DocumentUpdate(manager.getDocumentType("testdoc"),

-                                                                      new DocumentId("doc:scheme:"))));

-        frame.assertSelect(Arrays.asList("docproc/cluster.foo"));

-

-        frame.destroy();

-    }

-

-    @Test

-    public void testLoadBalancer() {

-        PolicyTestFrame frame = new PolicyTestFrame("docproc/cluster.default", manager);

-        frame.setMessage(new PutDocumentMessage(new DocumentPut(new Document(manager.getDocumentType("testdoc"),

-                                                             new DocumentId("doc:scheme:")))));

-        frame.getNetwork().registerSession("0/chain.default");

-        assertTrue(frame.waitSlobrok("docproc/cluster.default/*/chain.default", 1));

-        frame.setHop(new HopSpec("test", "[LoadBalancer:cluster=docproc/cluster.default;session=chain.default]"));

-

-        assertSelect(frame, 1, Arrays.asList(frame.getNetwork().getConnectionSpec() + "/chain.default"));

-    }

-

-    @Test

-    public void testRoundRobin() {

-        // Test select with proper address.

-        PolicyTestFrame frame = new PolicyTestFrame("docproc/cluster.default", manager);

-        frame.setMessage(new PutDocumentMessage(new DocumentPut(new Document(manager.getDocumentType("testdoc"),

-                                                             new DocumentId("doc:scheme:")))));

-        for (int i = 0; i < 10; ++i) {

-            frame.getNetwork().registerSession(i + "/chain.default");

-        }

-        assertTrue(frame.waitSlobrok("docproc/cluster.default/*/chain.default", 10));

-        frame.setHop(new HopSpec("test", "[RoundRobin]")

-                .addRecipient("docproc/cluster.default/3/chain.default")

-                .addRecipient("docproc/cluster.default/6/chain.default")

-                .addRecipient("docproc/cluster.default/9/chain.default"));

-        assertSelect(frame, 32, Arrays.asList("docproc/cluster.default/3/chain.default",

-                                              "docproc/cluster.default/6/chain.default",

-                                              "docproc/cluster.default/9/chain.default"));

-        frame.getNetwork().unregisterSession("6/chain.default");

-        assertTrue(frame.waitSlobrok("docproc/cluster.default/*/chain.default", 9));

-        assertSelect(frame, 32, Arrays.asList("docproc/cluster.default/3/chain.default",

-                                              "docproc/cluster.default/9/chain.default"));

-        frame.getNetwork().unregisterSession("3/chain.default");

-        assertTrue(frame.waitSlobrok("docproc/cluster.default/*/chain.default", 8));

-        assertSelect(frame, 32, Arrays.asList("docproc/cluster.default/9/chain.default"));

-        frame.getNetwork().unregisterSession("9/chain.default");

-        assertTrue(frame.waitSlobrok("docproc/cluster.default/*/chain.default", 7));

-        assertSelect(frame, 32, new ArrayList<String>());

-

-        // Test merge behavior.

-        frame.setHop(new HopSpec("test", "[RoundRobin]").addRecipient("docproc/cluster.default/0/chain.default"));

-        frame.assertMergeOneReply("docproc/cluster.default/0/chain.default");

-

-        frame.destroy();

-    }

-

-    @Test

-    public void testRoundRobinCache() {

-        PolicyTestFrame fooFrame = new PolicyTestFrame("docproc/cluster.default", manager);

-        HopSpec fooHop = new HopSpec("foo", "[RoundRobin]").addRecipient("docproc/cluster.default/0/chain.foo");

-        fooFrame.setMessage(new RemoveDocumentMessage(new DocumentId("doc:scheme:foo")));

-        fooFrame.setHop(fooHop);

-

-        PolicyTestFrame barFrame = new PolicyTestFrame(fooFrame);

-        HopSpec barHop = new HopSpec("bar", "[RoundRobin]").addRecipient("docproc/cluster.default/0/chain.bar");

-        barFrame.setMessage(new RemoveDocumentMessage(new DocumentId("doc:scheme:bar")));

-        barFrame.setHop(barHop);

-

-        fooFrame.getMessageBus().setupRouting(

-                new RoutingSpec().addTable(new RoutingTableSpec(DocumentProtocol.NAME)

-                                                   .addHop(fooHop)

-                                                   .addHop(barHop)));

-

-        fooFrame.getNetwork().registerSession("0/chain.foo");

-        fooFrame.getNetwork().registerSession("0/chain.bar");

-        assertTrue(fooFrame.waitSlobrok("docproc/cluster.default/0/*", 2));

-

-        RoutingNode fooChild = fooFrame.select(1).get(0);

-        assertEquals("docproc/cluster.default/0/chain.foo", fooChild.getRoute().toString());

-        RoutingNode barChild = barFrame.select(1).get(0);

-        assertEquals("docproc/cluster.default/0/chain.bar", barChild.getRoute().toString());

-

-        barChild.handleReply(new EmptyReply());

-        fooChild.handleReply(new EmptyReply());

-

-        assertNotNull(barFrame.getReceptor().getReply(TIMEOUT));

-        assertNotNull(fooFrame.getReceptor().getReply(TIMEOUT));

-    }

-

-    /**

-     * Ensures that the given number of select passes on the given frame produces an expected list of recipients.

-     *

-     * @param frame      The frame to select on.

-     * @param numSelects The number of selects to perform.

-     * @param expected   The list of expected recipients.

-     */

-    private static void assertSelect(PolicyTestFrame frame, int numSelects, List<String> expected) {

-        Set<String> lst = new TreeSet<>();

-

-        for (int i = 0; i < numSelects; ++i) {

-            if (!expected.isEmpty()) {

-                RoutingNode leaf = frame.select(1).get(0);

-                String recipient = leaf.getRoute().toString();

-                lst.add(recipient);

-                leaf.handleReply(new EmptyReply());

-            } else {

-                frame.select(0);

-            }

-            assertNotNull(frame.getReceptor().getReply(TIMEOUT));

-        }

-

-        assertEquals(expected.size(), lst.size());

-        Iterator<String> it = lst.iterator();

-        for (String recipient : expected) {

-            assertEquals(recipient, it.next());

-        }

-    }

-

-    private static void assertMirrorReady(Mirror slobrok)

-            throws InterruptedException, TimeoutException

-    {

-        for (int i = 0; i < TIMEOUT_MILLIS / 10; ++i) {

-            if (slobrok.ready()) {

-                return;

-            }

-            Thread.sleep(10);

-        }

-        throw new TimeoutException();

-    }

-

-    private static void assertMirrorContains(IMirror slobrok, String pattern, int numEntries) 

-            throws InterruptedException, TimeoutException 

-    {

-        for (int i = 0; i < TIMEOUT_MILLIS / 10; ++i) {

-            if (slobrok.lookup(pattern).length == numEntries) {

-                return;

-            }

-            Thread.sleep(10);

-        }

-        throw new TimeoutException();

-    }

-

-    private void setupExternPolicy(PolicyTestFrame frame, Slobrok slobrok, String pattern)

-            throws InterruptedException, TimeoutException

-    {

-        setupExternPolicy(frame, slobrok, pattern, -1);

-    }

-

-    private void setupExternPolicy(PolicyTestFrame frame, Slobrok slobrok, String pattern, int numEntries)

-            throws InterruptedException, TimeoutException

-    {

-        String param = "tcp/localhost:" + slobrok.port() + ";" + pattern;

-        frame.setHop(new HopSpec("test", "[Extern:" + param + "]"));

-        MessageBus mbus = frame.getMessageBus();

-        HopBlueprint hop = mbus.getRoutingTable(DocumentProtocol.NAME).getHop("test");

-        PolicyDirective dir = (PolicyDirective)hop.getDirective(0);

-        ExternPolicy policy = (ExternPolicy)mbus.getRoutingPolicy(DocumentProtocol.NAME, dir.getName(), dir.getParam());

-        assertMirrorReady(policy.getMirror());

-        if (numEntries >= 0) {

-            assertMirrorContains(policy.getMirror(), pattern, numEntries);

-        }

-    }

-

-    private PolicyTestFrame newFrame() {

-        return new PolicyTestFrame(manager);

-    }

-

-    private PolicyTestFrame newFrame(Message msg) {

-        PolicyTestFrame frame = newFrame();

-        frame.setMessage(msg);

-        return frame;

-    }

-

-    private PutDocumentMessage newPutDocument(String documentId) {

-        return new PutDocumentMessage(new DocumentPut(new Document(manager.getDocumentType("testdoc"),

-                                                                   new DocumentId(documentId))));

-    }

-

-    private PolicyTestFrame newPutDocumentFrame(String documentId) {

-        return newFrame(newPutDocument(documentId));

-    }

-}

+package com.yahoo.documentapi.messagebus.protocol.test;
+
+import com.yahoo.document.*;
+import com.yahoo.documentapi.messagebus.protocol.*;
+import com.yahoo.jrt.ListenFailedException;
+import com.yahoo.jrt.slobrok.api.IMirror;
+import com.yahoo.jrt.slobrok.api.Mirror;
+import com.yahoo.jrt.slobrok.server.Slobrok;
+import com.yahoo.messagebus.*;
+import com.yahoo.messagebus.Error;
+import com.yahoo.messagebus.network.rpc.test.TestServer;
+import com.yahoo.messagebus.routing.*;
+import com.yahoo.messagebus.test.Receptor;
+import com.yahoo.vdslib.DocumentList;
+import com.yahoo.vdslib.Entry;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.*;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+@SuppressWarnings("deprecation")
+public class PolicyTestCase {
+
+    private static final int TIMEOUT = 300;
+    private static final TimeUnit TIMEOUT_UNIT = TimeUnit.SECONDS;
+    private static final long TIMEOUT_MILLIS = TIMEOUT_UNIT.toMillis(TIMEOUT);
+    private final DocumentTypeManager manager = new DocumentTypeManager();
+
+    @Before
+    public void setUp() {
+        DocumentTypeManagerConfigurer.configure(manager, "file:./test/cfg/testdoc.cfg");
+    }
+
+    @Test
+    public void testProtocol() {
+        DocumentProtocol protocol = new DocumentProtocol(manager);
+
+        RoutingPolicy policy = protocol.createPolicy("AND", null);
+        assertTrue(policy instanceof ANDPolicy);
+
+        policy = new DocumentProtocol(manager).createPolicy("DocumentRouteSelector", "raw:route[0]\n");
+        assertTrue(policy instanceof DocumentRouteSelectorPolicy);
+
+        policy = new DocumentProtocol(manager).createPolicy("Extern", "foo;bar/baz");
+        assertTrue(policy instanceof ExternPolicy);
+
+        policy = new DocumentProtocol(manager).createPolicy("LocalService", null);
+        assertTrue(policy instanceof LocalServicePolicy);
+
+        policy = new DocumentProtocol(manager).createPolicy("RoundRobin", null);
+        assertTrue(policy instanceof RoundRobinPolicy);
+
+        policy = new DocumentProtocol(manager).createPolicy("SearchRow", null);
+        assertTrue(policy instanceof SearchRowPolicy);
+
+        policy = new DocumentProtocol(manager).createPolicy("SearchColumn", null);
+        assertTrue(policy instanceof SearchColumnPolicy);
+
+        policy = new DocumentProtocol(manager).createPolicy("SubsetService", null);
+        assertTrue(policy instanceof SubsetServicePolicy);
+
+        policy = new DocumentProtocol(manager).createPolicy("LoadBalancer", null);
+        assertTrue(policy instanceof LoadBalancerPolicy);
+    }
+
+    @Test
+    public void testAND() {
+        PolicyTestFrame frame = new PolicyTestFrame(manager);
+        frame.setMessage(new PutDocumentMessage(new DocumentPut(new Document(manager.getDocumentType("testdoc"),
+                                                             new DocumentId("doc:scheme:")))));
+        frame.setHop(new HopSpec("test", "[AND]")
+                     .addRecipient("foo")
+                     .addRecipient("bar"));
+        frame.assertSelect(Arrays.asList("foo", "bar"));
+
+        frame.setHop(new HopSpec("test", "[AND:baz]")
+                     .addRecipient("foo")
+                     .addRecipient("bar"));
+        frame.assertSelect(Arrays.asList("baz")); // param precedes recipients
+
+        frame.setHop(new HopSpec("test", "[AND:foo]"));
+        frame.assertMergeOneReply("foo");
+
+        frame.setHop(new HopSpec("test", "[AND:foo bar]"));
+        frame.assertMergeTwoReplies("foo", "bar");
+        frame.destroy();
+    }
+
+    @Test
+    public void requireThatExternPolicyWithIllegalParamIsAnErrorPolicy() throws ListenFailedException {
+        Slobrok slobrok = new Slobrok();
+        String spec = "tcp/localhost:" + slobrok.port();
+        assertTrue(new DocumentProtocol(manager).createPolicy("Extern", null) instanceof ErrorPolicy);
+        assertTrue(new DocumentProtocol(manager).createPolicy("Extern", "") instanceof ErrorPolicy);
+        assertTrue(new DocumentProtocol(manager).createPolicy("Extern", spec) instanceof ErrorPolicy);
+        assertTrue(new DocumentProtocol(manager).createPolicy("Extern", spec + ";") instanceof ErrorPolicy);
+        assertTrue(new DocumentProtocol(manager).createPolicy("Extern", spec + ";bar") instanceof ErrorPolicy);
+    }
+
+    @Test
+    public void requireThatExternPolicyWithUnknownPatternSelectsNone() throws Exception {
+        PolicyTestFrame frame = newPutDocumentFrame("doc:scheme:");
+        setupExternPolicy(frame, new Slobrok(), "foo/bar");
+        frame.assertSelect(null);
+    }
+
+    @Test
+    public void requireThatExternPolicySelectsFromExternSlobrok() throws Exception {
+        PolicyTestFrame frame = newPutDocumentFrame("doc:scheme:");
+        Slobrok slobrok = new Slobrok();
+        List<TestServer> servers = new ArrayList<>();
+        for (int i = 0; i < 10; ++i) {
+            TestServer server = new TestServer("docproc/cluster.default/" + i, null, slobrok, null,
+                                               new DocumentProtocol(manager));
+            server.net.registerSession("chain.default");
+            servers.add(server);
+        }
+        setupExternPolicy(frame, slobrok, "docproc/cluster.default/*/chain.default", 10);
+        Set<String> lst = new HashSet<>();
+        for (int i = 0; i < 10; ++i) {
+            RoutingNode leaf = frame.select(1).get(0);
+            String recipient = leaf.getRoute().toString();
+            lst.add(recipient);
+
+            leaf.handleReply(new EmptyReply());
+            assertNotNull(frame.getReceptor().getReply(TIMEOUT));
+        }
+        assertEquals(10, lst.size());
+        for (TestServer server : servers) {
+            server.destroy();
+        }
+        frame.destroy();
+    }
+
+    @Test
+    public void requireThatExternPolicyMergesOneReplyAsProtocol() throws Exception {
+        PolicyTestFrame frame = newPutDocumentFrame("doc:scheme:");
+        Slobrok slobrok = new Slobrok();
+        TestServer server = new TestServer("docproc/cluster.default/0", null, slobrok, null,
+                                           new DocumentProtocol(manager));
+        server.net.registerSession("chain.default");
+        setupExternPolicy(frame, slobrok, "docproc/cluster.default/*/chain.default", 1);
+        frame.assertMergeOneReply(server.net.getConnectionSpec() + "/chain.default");
+        server.destroy();
+        frame.destroy();
+    }
+
+    @Test
+    public void testExternSend() throws Exception {
+        // Setup local source node.
+        Slobrok local = new Slobrok();
+        TestServer src = new TestServer("src", null, local, null, new DocumentProtocol(manager));
+        SourceSession ss = src.mb.createSourceSession(new Receptor(), new SourceSessionParams().setTimeout(TIMEOUT));
+
+        // Setup remote cluster with routing config.
+        Slobrok slobrok = new Slobrok();
+        TestServer itr = new TestServer("itr",
+                                        new RoutingTableSpec(DocumentProtocol.NAME)
+                                                .addRoute(new RouteSpec("default").addHop("dst"))
+                                                .addHop(new HopSpec("dst", "dst/session")),
+                                        slobrok, null, new DocumentProtocol(manager));
+        IntermediateSession is = itr.mb.createIntermediateSession("session", true, new Receptor(), new Receptor());
+        TestServer dst = new TestServer("dst", null, slobrok, null, new DocumentProtocol(manager));
+        DestinationSession ds = dst.mb.createDestinationSession("session", true, new Receptor());
+
+        // Send message from local node to remote cluster and resolve route there.
+        Message msg = new RemoveDocumentMessage(new DocumentId("doc:scheme:"));
+        msg.getTrace().setLevel(9);
+        msg.setRoute(Route.parse("[Extern:tcp/localhost:" + slobrok.port() + ";itr/session] default"));
+
+        assertTrue(ss.send(msg).isAccepted());
+        assertNotNull(msg = ((Receptor)is.getMessageHandler()).getMessage(TIMEOUT));
+        is.forward(msg);
+        assertNotNull(msg = ((Receptor)ds.getMessageHandler()).getMessage(TIMEOUT));
+        ds.acknowledge(msg);
+        Reply reply = ((Receptor)is.getReplyHandler()).getReply(TIMEOUT);
+        assertNotNull(reply);
+        is.forward(reply);
+        assertNotNull(reply = ((Receptor)ss.getReplyHandler()).getReply(TIMEOUT));
+
+        System.out.println(reply.getTrace().toString());
+
+        // Perform necessary cleanup.
+        src.destroy();
+        itr.destroy();
+        dst.destroy();
+        slobrok.stop();
+        local.stop();
+    }
+
+    @Test
+    public void testExternMultipleSlobroks() throws ListenFailedException {
+        Slobrok local = new Slobrok();
+        TestServer srcServer = new TestServer("src", null, local, null, new DocumentProtocol(manager));
+        SourceSession srcSession =
+                srcServer.mb.createSourceSession(new Receptor(), new SourceSessionParams().setTimeout(TIMEOUT));
+
+        Slobrok extern = new Slobrok();
+        String spec = "tcp/localhost:" + extern.port();
+
+        TestServer dstServer = new TestServer("dst", null, extern, null, new DocumentProtocol(manager));
+        Receptor dstHandler = new Receptor();
+        DestinationSession dstSession = dstServer.mb.createDestinationSession("session", true, dstHandler);
+
+        Message msg = new RemoveDocumentMessage(new DocumentId("doc:scheme:"));
+        msg.setRoute(Route.parse("[Extern:" + spec + ";dst/session]"));
+        assertTrue(srcSession.send(msg).isAccepted());
+        assertNotNull(msg = dstHandler.getMessage(TIMEOUT));
+        dstSession.acknowledge(msg);
+        Reply reply = ((Receptor)srcSession.getReplyHandler()).getReply(TIMEOUT);
+        assertNotNull(reply);
+
+        extern.stop();
+        dstSession.destroy();
+        dstServer.destroy();
+        dstHandler.reset();
+        assertNull(dstHandler.getMessage(0));
+
+        extern = new Slobrok();
+        spec += ",tcp/localhost:" + extern.port();
+
+        dstServer = new TestServer("dst", null, extern, null, new DocumentProtocol(manager));
+        dstHandler = new Receptor();
+        dstSession = dstServer.mb.createDestinationSession("session", true, dstHandler);
+
+        msg = new RemoveDocumentMessage(new DocumentId("doc:scheme:"));
+        msg.setRoute(Route.parse("[Extern:" + spec + ";dst/session]"));
+        assertTrue(srcSession.send(msg).isAccepted());
+        assertNotNull(msg = dstHandler.getMessage(TIMEOUT));
+        dstSession.acknowledge(msg);
+        reply = ((Receptor)srcSession.getReplyHandler()).getReply(TIMEOUT);
+        assertNotNull(reply);
+
+        extern.stop();
+        dstSession.destroy();
+        dstServer.destroy();
+
+        local.stop();
+        srcSession.destroy();
+        srcServer.destroy();
+    }
+
+    @Test
+    public void testLocalService() {
+        // Test select with proper address.
+        PolicyTestFrame frame = new PolicyTestFrame("docproc/cluster.default", manager);
+        frame.setMessage(new PutDocumentMessage(new DocumentPut(new Document(manager.getDocumentType("testdoc"),
+                                                             new DocumentId("doc:scheme:0")))));
+        for (int i = 0; i < 10; ++i) {
+            frame.getNetwork().registerSession(i + "/chain.default");
+        }
+        assertTrue(frame.waitSlobrok("docproc/cluster.default/*/chain.default", 10));
+        frame.setHop(new HopSpec("test", "docproc/cluster.default/[LocalService]/chain.default"));
+
+        Set<String> lst = new HashSet<>();
+        for (int i = 0; i < 10; ++i) {
+            RoutingNode leaf = frame.select(1).get(0);
+            String recipient = leaf.getRoute().toString();
+            lst.add(recipient);
+
+            leaf.handleReply(new EmptyReply());
+            assertNotNull(frame.getReceptor().getReply(TIMEOUT));
+        }
+        assertEquals(10, lst.size());
+
+        // Test select with broken address.
+        lst.clear();
+        frame.setHop(new HopSpec("test", "docproc/cluster.default/[LocalService:broken]/chain.default"));
+        for (int i = 0; i < 10; ++i) {
+            RoutingNode leaf = frame.select(1).get(0);
+            String recipient = leaf.getRoute().toString();
+            assertTrue(recipient.equals("docproc/cluster.default/*/chain.default"));
+            lst.add(recipient);
+
+            leaf.handleReply(new EmptyReply());
+            assertNotNull(frame.getReceptor().getReply(TIMEOUT));
+        }
+        assertEquals(1, lst.size());
+
+        // Test merge behavior.
+        frame.setMessage(new PutDocumentMessage(new DocumentPut(new Document(manager.getDocumentType("testdoc"),
+                                                             new DocumentId("doc:scheme:")))));
+        frame.setHop(new HopSpec("test", "[LocalService]"));
+        frame.assertMergeOneReply("*");
+
+        frame.destroy();
+    }
+
+    @Test
+    public void testLocalServiceCache() {
+        PolicyTestFrame fooFrame = new PolicyTestFrame("docproc/cluster.default", manager);
+        HopSpec fooHop = new HopSpec("foo", "docproc/cluster.default/[LocalService]/chain.foo");
+        fooFrame.setMessage(new RemoveDocumentMessage(new DocumentId("doc:scheme:foo")));
+        fooFrame.setHop(fooHop);
+
+        PolicyTestFrame barFrame = new PolicyTestFrame(fooFrame);
+        HopSpec barHop = new HopSpec("bar", "docproc/cluster.default/[LocalService]/chain.bar");
+        barFrame.setMessage(new RemoveDocumentMessage(new DocumentId("doc:scheme:bar")));
+        barFrame.setHop(barHop);
+
+        fooFrame.getMessageBus().setupRouting(
+                new RoutingSpec().addTable(new RoutingTableSpec(DocumentProtocol.NAME)
+                                                   .addHop(fooHop)
+                                                   .addHop(barHop)));
+
+        fooFrame.getNetwork().registerSession("0/chain.foo");
+        fooFrame.getNetwork().registerSession("0/chain.bar");
+        assertTrue(fooFrame.waitSlobrok("docproc/cluster.default/0/*", 2));
+
+        RoutingNode fooChild = fooFrame.select(1).get(0);
+        assertEquals("docproc/cluster.default/0/chain.foo", fooChild.getRoute().getHop(0).toString());
+        RoutingNode barChild = barFrame.select(1).get(0);
+        assertEquals("docproc/cluster.default/0/chain.bar", barChild.getRoute().getHop(0).toString());
+
+        barChild.handleReply(new EmptyReply());
+        fooChild.handleReply(new EmptyReply());
+
+        assertNotNull(barFrame.getReceptor().getReply(TIMEOUT));
+        assertNotNull(fooFrame.getReceptor().getReply(TIMEOUT));
+    }
+
+    @Test
+    public void testSearchRow() {
+        PolicyTestFrame frame = new PolicyTestFrame(manager);
+        frame.setMessage(new PutDocumentMessage(new DocumentPut(new Document(manager.getDocumentType("testdoc"),
+                                                             new DocumentId("doc:scheme:")))));
+        frame.setHop(new HopSpec("test", "[SearchRow]").addRecipient("foo"));
+        frame.assertMergeOneReply("foo");
+        frame.setHop(new HopSpec("test", "[SearchRow]").addRecipient("foo").addRecipient("bar"));
+        frame.assertMergeTwoReplies("foo", "bar");
+
+        frame.setHop(new HopSpec("test", "[SearchRow:1]").addRecipient("foo"));
+        Map<String, Integer> replies = new HashMap<>();
+        replies.put("foo", ErrorCode.SERVICE_OOS);
+        frame.assertMergeError(replies, Arrays.asList(ErrorCode.SERVICE_OOS));
+
+        frame.setHop(new HopSpec("test", "[SearchRow:1]").addRecipient("foo").addRecipient("bar"));
+        replies.put("foo", ErrorCode.SERVICE_OOS);
+        replies.put("bar", ErrorCode.NONE);
+        frame.assertMergeOk(replies, Arrays.asList("bar"));
+
+        replies.put("foo", ErrorCode.SERVICE_OOS);
+        replies.put("bar", ErrorCode.SERVICE_OOS);
+        frame.assertMergeError(replies, Arrays.asList(ErrorCode.SERVICE_OOS, ErrorCode.SERVICE_OOS));
+
+        frame.setHop(new HopSpec("test", "[SearchRow:1]").addRecipient("foo").addRecipient("bar").addRecipient("baz"));
+        replies.put("foo", ErrorCode.SERVICE_OOS);
+        replies.put("bar", ErrorCode.NONE);
+        replies.put("baz", ErrorCode.NONE);
+        frame.assertMergeOk(replies, Arrays.asList("bar", "baz"));
+
+        replies.put("foo", ErrorCode.SERVICE_OOS);
+        replies.put("bar", ErrorCode.SERVICE_OOS);
+        replies.put("baz", ErrorCode.NONE);
+        frame.assertMergeOk(replies, Arrays.asList("baz"));
+
+        replies.put("foo", ErrorCode.SERVICE_OOS);
+        replies.put("bar", ErrorCode.SERVICE_OOS);
+        replies.put("baz", ErrorCode.SERVICE_OOS);
+        frame.assertMergeError(replies,
+                               Arrays.asList(ErrorCode.SERVICE_OOS, ErrorCode.SERVICE_OOS, ErrorCode.SERVICE_OOS));
+
+        frame.setHop(new HopSpec("test", "[SearchRow:2]").addRecipient("foo").addRecipient("bar").addRecipient("baz"));
+        replies.put("foo", ErrorCode.SERVICE_OOS);
+        replies.put("bar", ErrorCode.NONE);
+        replies.put("baz", ErrorCode.NONE);
+        frame.assertMergeOk(replies, Arrays.asList("bar", "baz"));
+
+        replies.put("foo", ErrorCode.SERVICE_OOS);
+        replies.put("bar", ErrorCode.SERVICE_OOS);
+        replies.put("baz", ErrorCode.NONE);
+        frame.assertMergeError(replies, Arrays.asList(ErrorCode.SERVICE_OOS, ErrorCode.SERVICE_OOS));
+
+        replies.put("foo", ErrorCode.SERVICE_OOS);
+        replies.put("bar", ErrorCode.SERVICE_OOS);
+        replies.put("baz", ErrorCode.SERVICE_OOS);
+        frame.assertMergeError(replies,
+                               Arrays.asList(ErrorCode.SERVICE_OOS, ErrorCode.SERVICE_OOS, ErrorCode.SERVICE_OOS));
+
+        frame.destroy();
+    }
+
+    @Test
+    public void testSearchRowMerge() {
+        PolicyTestFrame frame = new PolicyTestFrame(manager);
+        frame.setHop(new HopSpec("test", "[SearchRow]").addRecipient("foo"));
+        tryWasFound(frame, 1, 0x0, false);
+        tryWasFound(frame, 1, 0x1, true);
+
+        frame.setHop(new HopSpec("test", "[SearchRow]").addRecipient("foo").addRecipient("bar"));
+        tryWasFound(frame, 2, 0x0, false);
+        tryWasFound(frame, 2, 0x1, true);
+        tryWasFound(frame, 2, 0x2, true);
+        tryWasFound(frame, 2, 0x3, true);
+
+        frame.setHop(new HopSpec("test", "[SearchRow]").addRecipient("foo").addRecipient("bar").addRecipient("baz"));
+        tryWasFound(frame, 3, 0x0, false);
+        tryWasFound(frame, 3, 0x1, true);
+        tryWasFound(frame, 3, 0x2, true);
+        tryWasFound(frame, 3, 0x3, true);
+        tryWasFound(frame, 3, 0x4, true);
+        tryWasFound(frame, 3, 0x5, true);
+        tryWasFound(frame, 3, 0x6, true);
+        tryWasFound(frame, 3, 0x7, true);
+        frame.destroy();
+    }
+
+    private void tryWasFound(PolicyTestFrame frame, int expectedRecipients,
+                             int foundMask, boolean expectedFound)
+    {
+        {
+            frame.setMessage(new RemoveDocumentMessage(new DocumentId("doc:scheme:69")));
+            List<RoutingNode> selected = frame.select(expectedRecipients);
+            for (int i = 0, len = selected.size(); i < len; ++i) {
+                RemoveDocumentReply reply = new RemoveDocumentReply();
+                reply.setWasFound(((1 << i) & foundMask) != 0);
+                selected.get(i).handleReply(reply);
+            }
+            Reply reply = frame.getReceptor().getReply(TIMEOUT);
+            assertNotNull(reply);
+            assertEquals(DocumentProtocol.REPLY_REMOVEDOCUMENT, reply.getType());
+            assertEquals(expectedFound, ((RemoveDocumentReply)reply).wasFound());
+        }
+        {
+            DocumentUpdate  upd = new DocumentUpdate(manager.getDocumentType("testdoc"),
+                                                     new DocumentId("doc:scheme:"));
+            frame.setMessage(new UpdateDocumentMessage(upd));
+            List<RoutingNode> selected = frame.select(expectedRecipients);
+            for (int i = 0, len = selected.size(); i < len; ++i) {
+                UpdateDocumentReply reply = new UpdateDocumentReply();
+                reply.setWasFound(((1 << i) & foundMask) != 0);
+                selected.get(i).handleReply(reply);
+            }
+            Reply reply = frame.getReceptor().getReply(TIMEOUT);
+            assertNotNull(reply);
+            assertEquals(DocumentProtocol.REPLY_UPDATEDOCUMENT, reply.getType());
+            assertEquals(expectedFound, ((UpdateDocumentReply)reply).wasFound());
+        }
+    }
+
+    @Test
+    public void multipleGetRepliesAreMergedToFoundDocument() {
+        PolicyTestFrame frame = new PolicyTestFrame(manager);
+        frame.setHop(new HopSpec("test", getDocumentRouteSelectorRawConfig())
+                .addRecipient("foo").addRecipient("bar"));
+        frame.setMessage(new GetDocumentMessage(new DocumentId("doc:scheme:yarn"), "[all]"));
+        List<RoutingNode> selected = frame.select(2);
+        for (int i = 0, len = selected.size(); i < len; ++i) {
+            Document doc = null;
+            if (i == 0) {
+                doc = new Document(manager.getDocumentType("testdoc"),
+                                   new DocumentId("doc:scheme:yarn"));
+                doc.setLastModified(123456L);
+            }
+            GetDocumentReply reply = new GetDocumentReply(null);
+            reply.setDocument(doc);
+            selected.get(i).handleReply(reply);
+        }
+        Reply reply = frame.getReceptor().getReply(TIMEOUT);
+        assertNotNull(reply);
+        assertEquals(DocumentProtocol.REPLY_GETDOCUMENT, reply.getType());
+        assertEquals(123456L, ((GetDocumentReply)reply).getLastModified());
+    }
+
+    private String getDocumentRouteSelectorRawConfig() {
+        return "[DocumentRouteSelector:raw:" +
+                "route[2]\n" +
+                "route[0].name \"foo\"\n" +
+                "route[0].selector \"testdoc\"\n" +
+                "route[0].feed \"myfeed\"\n" +
+                "route[1].name \"bar\"\n" +
+                "route[1].selector \"other\"\n" +
+                "route[1].feed \"myfeed\"\n]";
+    }
+
+    @Test
+    public void testSearchColumn() {
+        PolicyTestFrame frame = new PolicyTestFrame(manager);
+        frame.setHop(new HopSpec("test", "[SearchColumn]")
+                .addRecipient("c0").addRecipient("c1")
+                .addRecipient("c2").addRecipient("c3"));
+
+        // Test hash distribution.
+        assertDistribution(frame, "doc:ns:3", "c0");
+        assertDistribution(frame, "doc:ns:18", "c1");
+        assertDistribution(frame, "doc:ns:0", "c2");
+        assertDistribution(frame, "doc:ns:4", "c3");
+
+        assertDistribution(frame, "userdoc:ns:49152:0", "c0");
+        assertDistribution(frame, "userdoc:ns:49152:1", "c0");
+        assertDistribution(frame, "userdoc:ns:16384:2", "c1");
+        assertDistribution(frame, "userdoc:ns:16384:3", "c1");
+        assertDistribution(frame, "userdoc:ns:5461:4", "c2");
+        assertDistribution(frame, "userdoc:ns:5461:5", "c2");
+        assertDistribution(frame, "userdoc:ns:0:6", "c3");
+        assertDistribution(frame, "userdoc:ns:0:7", "c3");
+
+        assertDistribution(frame, "groupdoc:ns:0:0", "c0");
+        assertDistribution(frame, "groupdoc:ns:0:1", "c0");
+        assertDistribution(frame, "groupdoc:ns:4:2", "c1");
+        assertDistribution(frame, "groupdoc:ns:4:3", "c1");
+        assertDistribution(frame, "groupdoc:ns:2:4", "c2");
+        assertDistribution(frame, "groupdoc:ns:2:5", "c2");
+        assertDistribution(frame, "groupdoc:ns:7:6", "c3");
+        assertDistribution(frame, "groupdoc:ns:7:7", "c3");
+
+        // Test routing based on message type.
+        Message put = new PutDocumentMessage(new DocumentPut(new Document(manager.getDocumentType("testdoc"),
+                                                          new DocumentId("doc:scheme:"))));
+        frame.setHop(new HopSpec("test", "[SearchColumn]").addRecipient("c0").addRecipient("c1"));
+        frame.setMessage(put);
+        frame.assertMergeOneReply("c0");
+
+        // Test allowed bad parts.
+        frame.setHop(new HopSpec("test", "[SearchColumn:1]").addRecipient("c0"));
+        frame.setMessage(put);
+        Map<String, Integer> replies = new HashMap<>();
+        replies.put("c0", ErrorCode.SERVICE_OOS);
+        frame.assertMergeOk(replies, null);
+
+        replies.put("c0", ErrorCode.SERVICE_OOS);
+        frame.assertMergeOk(replies, null);
+
+        frame.setHop(new HopSpec("test", "[SearchColumn:1]").addRecipient("c0").addRecipient("c1"));
+        frame.setMessage(put);
+        replies.put("c0", ErrorCode.SERVICE_OOS);
+        frame.assertMergeOk(replies, null);
+
+        frame.setHop(new HopSpec("test", "[SearchColumn:1]").addRecipient("c0").addRecipient("c1").addRecipient("c2"));
+        frame.setMessage(put);
+        replies.clear();
+        replies.put("c0", ErrorCode.SERVICE_OOS);
+        frame.assertMergeOk(replies, null);
+
+        frame.setHop(new HopSpec("test", "[SearchColumn:2]").addRecipient("c0").addRecipient("c1").addRecipient("c2"));
+        frame.setMessage(put);
+        replies.clear();
+        replies.put("c0", ErrorCode.SERVICE_OOS);
+        frame.assertMergeOk(replies, null);
+
+        frame.destroy();
+    }
+
+    private void assertDistribution(PolicyTestFrame frame, String id, String expected) {
+        frame.setMessage(new PutDocumentMessage(new DocumentPut(new Document(manager.getDocumentType("testdoc"),
+                                                             new DocumentId(id)))));
+        frame.assertSelect(Arrays.asList(expected));
+    }
+
+    @Test
+    public void testSubsetService() {
+        PolicyTestFrame frame = new PolicyTestFrame("docproc/cluster.default", manager);
+        frame.setMessage(new PutDocumentMessage(new DocumentPut(new DocumentPut(new Document(manager.getDocumentType("testdoc"),
+                                                             new DocumentId("doc:scheme:"))))));
+
+        // Test requerying for adding nodes.
+        frame.setHop(new HopSpec("test", "docproc/cluster.default/[SubsetService:2]/chain.default"));
+        Set<String> lst = new HashSet<>();
+        for (int i = 1; i <= 10; ++i) {
+            frame.getNetwork().registerSession(i + "/chain.default");
+            assertTrue(frame.waitSlobrok("docproc/cluster.default/*/chain.default", i));
+
+            RoutingNode leaf = frame.select(1).get(0);
+            lst.add(leaf.getRoute().toString());
+            leaf.handleReply(new EmptyReply());
+            assertNotNull(frame.getReceptor().getReply(TIMEOUT));
+        }
+        assertTrue(lst.size() > 1); // must have requeried
+
+        // Test load balancing.
+        String prev = null;
+        for (int i = 1; i <= 10; ++i) {
+            RoutingNode leaf = frame.select(1).get(0);
+            String next = leaf.getRoute().toString();
+            if (prev == null) {
+                assertNotNull(next);
+            } else {
+                assertFalse(prev.equals(next));
+            }
+            prev = next;
+            leaf.handleReply(new EmptyReply());
+            assertNotNull(frame.getReceptor().getReply(TIMEOUT));
+        }
+
+        // Test requerying for dropping nodes.
+        lst.clear();
+        for (int i = 1; i <= 10; ++i) {
+            RoutingNode leaf = frame.select(1).get(0);
+            String route = leaf.getRoute().toString();
+            lst.add(route);
+
+            frame.getNetwork().unregisterSession(route.substring(frame.getIdentity().length() + 1));
+            assertTrue(frame.waitSlobrok("docproc/cluster.default/*/chain.default", 10 - i));
+
+            Reply reply = new EmptyReply();
+            reply.addError(new Error(ErrorCode.NO_ADDRESS_FOR_SERVICE, route));
+            leaf.handleReply(reply);
+            assertNotNull(frame.getReceptor().getReply(TIMEOUT));
+        }
+        assertEquals(10, lst.size());
+
+        // Test merge behavior.
+        frame.setHop(new HopSpec("test", "[SubsetService]"));
+        frame.assertMergeOneReply("*");
+
+        frame.destroy();
+    }
+
+    @Test
+    public void testSubsetServiceCache() {
+        PolicyTestFrame fooFrame = new PolicyTestFrame("docproc/cluster.default", manager);
+        HopSpec fooHop = new HopSpec("foo", "docproc/cluster.default/[SubsetService:2]/chain.foo");
+        fooFrame.setMessage(new RemoveDocumentMessage(new DocumentId("doc:scheme:foo")));
+        fooFrame.setHop(fooHop);
+
+        PolicyTestFrame barFrame = new PolicyTestFrame(fooFrame);
+        HopSpec barHop = new HopSpec("bar", "docproc/cluster.default/[SubsetService:2]/chain.bar");
+        barFrame.setMessage(new RemoveDocumentMessage(new DocumentId("doc:scheme:bar")));
+        barFrame.setHop(barHop);
+
+        fooFrame.getMessageBus().setupRouting(
+                new RoutingSpec().addTable(new RoutingTableSpec(DocumentProtocol.NAME)
+                                                   .addHop(fooHop)
+                                                   .addHop(barHop)));
+
+        fooFrame.getNetwork().registerSession("0/chain.foo");
+        fooFrame.getNetwork().registerSession("0/chain.bar");
+        assertTrue(fooFrame.waitSlobrok("docproc/cluster.default/0/*", 2));
+
+        RoutingNode fooChild = fooFrame.select(1).get(0);
+        assertEquals("docproc/cluster.default/0/chain.foo", fooChild.getRoute().getHop(0).toString());
+        RoutingNode barChild = barFrame.select(1).get(0);
+        assertEquals("docproc/cluster.default/0/chain.bar", barChild.getRoute().getHop(0).toString());
+
+        barChild.handleReply(new EmptyReply());
+        fooChild.handleReply(new EmptyReply());
+
+        assertNotNull(barFrame.getReceptor().getReply(TIMEOUT));
+        assertNotNull(fooFrame.getReceptor().getReply(TIMEOUT));
+    }
+
+    @Test
+    public void testDocumentRouteSelector() {
+        // Test policy usage safeguard.
+        String okConfig = "raw:route[0]\n";
+        String errConfig = "raw:" +
+                           "route[1]\n" +
+                           "route[0].name \"foo\"\n" +
+                           "route[0].selector \"foo bar\"\n" +
+                           "route[0].feed \"baz\"\n";
+
+        DocumentProtocol protocol = new DocumentProtocol(manager, okConfig);
+        assertTrue(protocol.createPolicy("DocumentRouteSelector", null) instanceof DocumentRouteSelectorPolicy);
+        assertTrue(protocol.createPolicy("DocumentRouteSelector", "") instanceof DocumentRouteSelectorPolicy);
+        assertTrue(protocol.createPolicy("DocumentRouteSelector", errConfig) instanceof ErrorPolicy);
+
+        protocol = new DocumentProtocol(manager, errConfig);
+        assertTrue(protocol.createPolicy("DocumentRouteSelector", null) instanceof ErrorPolicy);
+        assertTrue(protocol.createPolicy("DocumentRouteSelector", "") instanceof ErrorPolicy);
+        assertTrue(protocol.createPolicy("DocumentRouteSelector", okConfig) instanceof DocumentRouteSelectorPolicy);
+
+        // Test policy with proper config.
+        PolicyTestFrame frame = new PolicyTestFrame(manager);
+        frame.setHop(new HopSpec("test", "[DocumentRouteSelector:raw:" +
+                                         "route[2]\n" +
+                                         "route[0].name \"foo\"\n" +
+                                         "route[0].selector \"testdoc\"\n" +
+                                         "route[0].feed \"myfeed\"\n" +
+                                         "route[1].name \"bar\"\n" +
+                                         "route[1].selector \"other\"\n" +
+                                         "route[1].feed \"myfeed\"\n]").addRecipient("foo").addRecipient("bar"));
+
+        frame.setMessage(new GetDocumentMessage(new DocumentId("doc:scheme:"), "fieldSet"));
+        frame.assertSelect(Arrays.asList("bar", "foo"));
+
+        Message put = new PutDocumentMessage(new DocumentPut(new Document(manager.getDocumentType("testdoc"),
+                                                          new DocumentId("doc:scheme:"))));
+        frame.setMessage(put);
+        frame.assertSelect(Arrays.asList("foo"));
+
+        frame.setMessage(new RemoveDocumentMessage(new DocumentId("doc:scheme:")));
+        frame.assertSelect(Arrays.asList("bar", "foo"));
+
+        frame.setMessage(new UpdateDocumentMessage(new DocumentUpdate(manager.getDocumentType("testdoc"),
+                                                                      new DocumentId("doc:scheme:"))));
+        frame.assertSelect(Arrays.asList("foo"));
+
+        frame.setMessage(put);
+        frame.assertMergeOneReply("foo");
+
+        frame.destroy();
+    }
+
+
+    @Test
+    public void testDocumentRouteSelectorIgnore() {
+        PolicyTestFrame frame = new PolicyTestFrame(manager);
+        frame.setHop(new HopSpec("test", "[DocumentRouteSelector:raw:" +
+                                         "route[1]\n" +
+                                         "route[0].name \"docproc/cluster.foo\"\n" +
+                                         "route[0].selector \"testdoc and testdoc.stringfield == 'foo'\"\n" +
+                                         "route[0].feed \"myfeed\"\n]").addRecipient("docproc/cluster.foo"));
+
+        frame.setMessage(new PutDocumentMessage(
+                new DocumentPut(new Document(manager.getDocumentType("testdoc"),
+                             new DocumentId("id:yarn:testdoc:n=1234:fluff")))));
+        frame.select(0);
+        Reply reply = frame.getReceptor().getReply(TIMEOUT);
+        assertNotNull(reply);
+        assertEquals(DocumentProtocol.REPLY_DOCUMENTIGNORED, reply.getType());
+        assertEquals(0, reply.getNumErrors());
+
+        frame.setMessage(new UpdateDocumentMessage(new DocumentUpdate(manager.getDocumentType("testdoc"),
+                                                                      new DocumentId("doc:scheme:"))));
+        frame.assertSelect(Arrays.asList("docproc/cluster.foo"));
+
+        frame.destroy();
+    }
+
+    @Test
+    public void testLoadBalancer() {
+        PolicyTestFrame frame = new PolicyTestFrame("docproc/cluster.default", manager);
+        frame.setMessage(new PutDocumentMessage(new DocumentPut(new Document(manager.getDocumentType("testdoc"),
+                                                             new DocumentId("doc:scheme:")))));
+        frame.getNetwork().registerSession("0/chain.default");
+        assertTrue(frame.waitSlobrok("docproc/cluster.default/*/chain.default", 1));
+        frame.setHop(new HopSpec("test", "[LoadBalancer:cluster=docproc/cluster.default;session=chain.default]"));
+
+        assertSelect(frame, 1, Arrays.asList(frame.getNetwork().getConnectionSpec() + "/chain.default"));
+    }
+
+    @Test
+    public void testRoundRobin() {
+        // Test select with proper address.
+        PolicyTestFrame frame = new PolicyTestFrame("docproc/cluster.default", manager);
+        frame.setMessage(new PutDocumentMessage(new DocumentPut(new Document(manager.getDocumentType("testdoc"),
+                                                             new DocumentId("doc:scheme:")))));
+        for (int i = 0; i < 10; ++i) {
+            frame.getNetwork().registerSession(i + "/chain.default");
+        }
+        assertTrue(frame.waitSlobrok("docproc/cluster.default/*/chain.default", 10));
+        frame.setHop(new HopSpec("test", "[RoundRobin]")
+                .addRecipient("docproc/cluster.default/3/chain.default")
+                .addRecipient("docproc/cluster.default/6/chain.default")
+                .addRecipient("docproc/cluster.default/9/chain.default"));
+        assertSelect(frame, 32, Arrays.asList("docproc/cluster.default/3/chain.default",
+                                              "docproc/cluster.default/6/chain.default",
+                                              "docproc/cluster.default/9/chain.default"));
+        frame.getNetwork().unregisterSession("6/chain.default");
+        assertTrue(frame.waitSlobrok("docproc/cluster.default/*/chain.default", 9));
+        assertSelect(frame, 32, Arrays.asList("docproc/cluster.default/3/chain.default",
+                                              "docproc/cluster.default/9/chain.default"));
+        frame.getNetwork().unregisterSession("3/chain.default");
+        assertTrue(frame.waitSlobrok("docproc/cluster.default/*/chain.default", 8));
+        assertSelect(frame, 32, Arrays.asList("docproc/cluster.default/9/chain.default"));
+        frame.getNetwork().unregisterSession("9/chain.default");
+        assertTrue(frame.waitSlobrok("docproc/cluster.default/*/chain.default", 7));
+        assertSelect(frame, 32, new ArrayList<String>());
+
+        // Test merge behavior.
+        frame.setHop(new HopSpec("test", "[RoundRobin]").addRecipient("docproc/cluster.default/0/chain.default"));
+        frame.assertMergeOneReply("docproc/cluster.default/0/chain.default");
+
+        frame.destroy();
+    }
+
+    @Test
+    public void testRoundRobinCache() {
+        PolicyTestFrame fooFrame = new PolicyTestFrame("docproc/cluster.default", manager);
+        HopSpec fooHop = new HopSpec("foo", "[RoundRobin]").addRecipient("docproc/cluster.default/0/chain.foo");
+        fooFrame.setMessage(new RemoveDocumentMessage(new DocumentId("doc:scheme:foo")));
+        fooFrame.setHop(fooHop);
+
+        PolicyTestFrame barFrame = new PolicyTestFrame(fooFrame);
+        HopSpec barHop = new HopSpec("bar", "[RoundRobin]").addRecipient("docproc/cluster.default/0/chain.bar");
+        barFrame.setMessage(new RemoveDocumentMessage(new DocumentId("doc:scheme:bar")));
+        barFrame.setHop(barHop);
+
+        fooFrame.getMessageBus().setupRouting(
+                new RoutingSpec().addTable(new RoutingTableSpec(DocumentProtocol.NAME)
+                                                   .addHop(fooHop)
+                                                   .addHop(barHop)));
+
+        fooFrame.getNetwork().registerSession("0/chain.foo");
+        fooFrame.getNetwork().registerSession("0/chain.bar");
+        assertTrue(fooFrame.waitSlobrok("docproc/cluster.default/0/*", 2));
+
+        RoutingNode fooChild = fooFrame.select(1).get(0);
+        assertEquals("docproc/cluster.default/0/chain.foo", fooChild.getRoute().toString());
+        RoutingNode barChild = barFrame.select(1).get(0);
+        assertEquals("docproc/cluster.default/0/chain.bar", barChild.getRoute().toString());
+
+        barChild.handleReply(new EmptyReply());
+        fooChild.handleReply(new EmptyReply());
+
+        assertNotNull(barFrame.getReceptor().getReply(TIMEOUT));
+        assertNotNull(fooFrame.getReceptor().getReply(TIMEOUT));
+    }
+
+    /**
+     * Ensures that the given number of select passes on the given frame produces an expected list of recipients.
+     *
+     * @param frame      The frame to select on.
+     * @param numSelects The number of selects to perform.
+     * @param expected   The list of expected recipients.
+     */
+    private static void assertSelect(PolicyTestFrame frame, int numSelects, List<String> expected) {
+        Set<String> lst = new TreeSet<>();
+
+        for (int i = 0; i < numSelects; ++i) {
+            if (!expected.isEmpty()) {
+                RoutingNode leaf = frame.select(1).get(0);
+                String recipient = leaf.getRoute().toString();
+                lst.add(recipient);
+                leaf.handleReply(new EmptyReply());
+            } else {
+                frame.select(0);
+            }
+            assertNotNull(frame.getReceptor().getReply(TIMEOUT));
+        }
+
+        assertEquals(expected.size(), lst.size());
+        Iterator<String> it = lst.iterator();
+        for (String recipient : expected) {
+            assertEquals(recipient, it.next());
+        }
+    }
+
+    private static void assertMirrorReady(Mirror slobrok)
+            throws InterruptedException, TimeoutException
+    {
+        for (int i = 0; i < TIMEOUT_MILLIS / 10; ++i) {
+            if (slobrok.ready()) {
+                return;
+            }
+            Thread.sleep(10);
+        }
+        throw new TimeoutException();
+    }
+
+    private static void assertMirrorContains(IMirror slobrok, String pattern, int numEntries)
+            throws InterruptedException, TimeoutException
+    {
+        for (int i = 0; i < TIMEOUT_MILLIS / 10; ++i) {
+            if (slobrok.lookup(pattern).length == numEntries) {
+                return;
+            }
+            Thread.sleep(10);
+        }
+        throw new TimeoutException();
+    }
+
+    private void setupExternPolicy(PolicyTestFrame frame, Slobrok slobrok, String pattern)
+            throws InterruptedException, TimeoutException
+    {
+        setupExternPolicy(frame, slobrok, pattern, -1);
+    }
+
+    private void setupExternPolicy(PolicyTestFrame frame, Slobrok slobrok, String pattern, int numEntries)
+            throws InterruptedException, TimeoutException
+    {
+        String param = "tcp/localhost:" + slobrok.port() + ";" + pattern;
+        frame.setHop(new HopSpec("test", "[Extern:" + param + "]"));
+        MessageBus mbus = frame.getMessageBus();
+        HopBlueprint hop = mbus.getRoutingTable(DocumentProtocol.NAME).getHop("test");
+        PolicyDirective dir = (PolicyDirective)hop.getDirective(0);
+        ExternPolicy policy = (ExternPolicy)mbus.getRoutingPolicy(DocumentProtocol.NAME, dir.getName(), dir.getParam());
+        assertMirrorReady(policy.getMirror());
+        if (numEntries >= 0) {
+            assertMirrorContains(policy.getMirror(), pattern, numEntries);
+        }
+    }
+
+    private PolicyTestFrame newFrame() {
+        return new PolicyTestFrame(manager);
+    }
+
+    private PolicyTestFrame newFrame(Message msg) {
+        PolicyTestFrame frame = newFrame();
+        frame.setMessage(msg);
+        return frame;
+    }
+
+    private PutDocumentMessage newPutDocument(String documentId) {
+        return new PutDocumentMessage(new DocumentPut(new Document(manager.getDocumentType("testdoc"),
+                                                                   new DocumentId(documentId))));
+    }
+
+    private PolicyTestFrame newPutDocumentFrame(String documentId) {
+        return newFrame(newPutDocument(documentId));
+    }
+}
diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyTestFrame.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyTestFrame.java
index 37b6c1a9068..f3721ff0173 100755
--- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyTestFrame.java
+++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyTestFrame.java
@@ -1,385 +1,385 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.documentapi.messagebus.protocol.test;

-

-import com.yahoo.document.DocumentTypeManager;

-import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol;

-import com.yahoo.jrt.ListenFailedException;

-import com.yahoo.jrt.slobrok.api.Mirror;

-import com.yahoo.jrt.slobrok.server.Slobrok;

-import com.yahoo.messagebus.*;

-import com.yahoo.messagebus.network.Identity;

-import com.yahoo.messagebus.network.Network;

-import com.yahoo.messagebus.network.ServiceAddress;

-import com.yahoo.messagebus.network.rpc.RPCNetwork;

-import com.yahoo.messagebus.network.rpc.RPCNetworkParams;

-import com.yahoo.messagebus.network.rpc.test.TestServer;

-import com.yahoo.messagebus.routing.*;

-import com.yahoo.messagebus.test.Receptor;

-import com.yahoo.messagebus.test.SimpleProtocol;

-import com.yahoo.messagebus.test.SimpleReply;

-

-import java.util.*;

-import java.util.concurrent.atomic.AtomicBoolean;

-

-/**

- * This is a utility class to allow easier policy test cases. The most important reason to use this is to make sure that

- * each test uses a "clean" mbus and slobrok instance.

- *

- * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>

- */

-public class PolicyTestFrame extends junit.framework.Assert {

-

-    private final AtomicBoolean destroyed = new AtomicBoolean(false);

-    private String identity;

-    private Slobrok slobrok;

-    private MessageBus mbus;

-    private MyNetwork net;

-    private Message msg = null;

-    private HopSpec hop = null;

-    private Receptor handler = new Receptor();

-

-    /**

-     * Create an anonymous test frame.

-     *

-     * @param documentMgr The document manager to use.

-     */

-    public PolicyTestFrame(DocumentTypeManager documentMgr) {

-        this("anonymous", documentMgr);

-    }

-

-    /**

-     * Create a named test frame.

-     *

-     * @param identity    The identity to use for the server.

-     * @param documentMgr The document manager to use.

-     */

-    public PolicyTestFrame(String identity, DocumentTypeManager documentMgr) {

-        this.identity = identity;

-        try {

-            slobrok = new Slobrok();

-        } catch (ListenFailedException e) {

-            e.printStackTrace();

-            fail(e.getMessage());

-        }

-        net = new MyNetwork(new RPCNetworkParams()

-                .setIdentity(new Identity(identity))

-                .setSlobrokConfigId(TestServer.getSlobrokConfig(slobrok)));

-        mbus = new MessageBus(net, new MessageBusParams()

-                .addProtocol(new DocumentProtocol(documentMgr)));

-    }

-

-    /**

-     * Create a test frame running on the same slobrok and mbus as another.

-     *

-     * @param frame The frame whose internals to share.

-     */

-    public PolicyTestFrame(PolicyTestFrame frame) {

-        identity = frame.identity;

-        slobrok = frame.slobrok;

-        net = frame.net;

-        mbus = frame.mbus;

-    }

-

-    // Inherit doc from Object.

-    @Override

-    public void finalize() throws Throwable {

-        destroy();

-        super.finalize();

-    }

-

-    /**

-     * Sets the destroyed flag to true. The very first time this method is called, it cleans up all its dependencies.

-     * Even if you retain a reference to this object, all of its content is allowed to be garbage collected.

-     */

-    public void destroy() {

-        if (!destroyed.getAndSet(true)) {

-            mbus.destroy();

-            mbus = null;

-            net = null;

-

-            slobrok.stop();

-            slobrok = null;

-        }

-    }

-

-    /**

-     * Routes the contained message based on the current setup, and returns the leaf send contexts.

-     *

-     * @param numExpected The expected number of leaf nodes.

-     * @return The list of selected send contexts.

-     */

-    public List<RoutingNode> select(int numExpected) {

-        msg.setRoute(Route.parse(hop.getName()));

-        new RoutingNode(mbus, net, null, handler, msg).send();

-        List<RoutingNode> ret = net.removeNodes();

-        assertEquals(numExpected, ret.size());

-        return ret;

-    }

-

-    /**

-     * Ensures that the current setup selects a given set of routes.

-     *

-     * @param expected A list of expected route leaf nodes.

-     */

-    public void assertSelect(List<String> expected) {

-        if (expected == null) {

-            assertEquals(0, select(0).size());

-        } else {

-            List<RoutingNode> selected = select(expected.size());

-            for (RoutingNode node : selected) {

-                assertTrue("Route '" + node.getRoute() + "' not selected.", 

-                           expected.contains(node.getRoute().toString()));

-                node.handleReply(new EmptyReply());

-            }

-        }

-        assertNotNull(handler.getReply(60));

-    }

-

-    /**

-     * This is a convenience method for invoking {@link #assertMerge(Map,List,List)} with no expected value.

-     *

-     * @param replies        The errors to set in the leaf node replies.

-     * @param expectedErrors The list of expected errors in the merged reply.

-     */

-    public void assertMergeError(Map<String, Integer> replies, List<Integer> expectedErrors) {

-        assertMerge(replies, expectedErrors, null);

-    }

-

-    /**

-     * This is a convenience method for invoking {@link this#assertMerge(Map,List,List)} with no expected errors.

-     *

-     * @param replies       The errors to set in the leaf node replies.

-     * @param allowedValues The list of allowed values in the final reply.

-     */

-    public void assertMergeOk(Map<String, Integer> replies, List<String> allowedValues) {

-        assertMerge(replies, null, allowedValues);

-    }

-

-    /**

-     * Ensures that the current setup generates as many leaf nodes as there are members of the errors argument. Each

-     * error is then given one of these errors, and the method then ensures that the single returned reply contains the

-     * given list of expected errors. Finally, if the expected value argument is non-null, this method ensures that the

-     * reply is a SimpleReply whose string value exists in the allowed list.

-     *

-     * @param replies        The errors to set in the leaf node replies.

-     * @param expectedErrors The list of expected errors in the merged reply.

-     * @param allowedValues  The list of allowed values in the final reply.

-     */

-    public void assertMerge(Map<String, Integer> replies, List<Integer> expectedErrors, List<String> allowedValues) {

-        List<RoutingNode> selected = select(replies.size());

-

-        for (RoutingNode node : selected) {

-            String route = node.getRoute().toString();

-            assertTrue(replies.containsKey(route));

-            Reply ret = new SimpleReply(route);

-            if (replies.get(route) != ErrorCode.NONE) {

-                ret.addError(new com.yahoo.messagebus.Error(replies.get(route), route));

-            }

-            node.handleReply(ret);

-        }

-

-        Reply reply = handler.getReply(60);

-        assertNotNull(reply);

-        if (expectedErrors != null) {

-            assertEquals(expectedErrors.size(), reply.getNumErrors());

-            for (int i = 0; i < expectedErrors.size(); ++i) {

-                assertTrue(expectedErrors.contains(reply.getError(i).getCode()));

-            }

-        } else if (reply.hasErrors()) {

-            StringBuilder err = new StringBuilder("Got unexpected error(s):\n");

-            for (int i = 0; i < reply.getNumErrors(); ++i) {

-                err.append("\t").append(reply.getError(i).toString());

-                if (i < reply.getNumErrors() - 1) {

-                    err.append("\n");

-                }

-            }

-            fail(err.toString());

-        }

-        if (allowedValues != null) {

-            assertEquals(SimpleProtocol.REPLY, reply.getType());

-            assertTrue(allowedValues.contains(((SimpleReply)reply).getValue()));

-        } else {

-            assertEquals(0, reply.getType());

-        }

-    }

-

-    /**

-     * Ensures that the current setup chooses a single recipient, and that it merges similarly to how the

-     * {@link DocumentProtocol} would merge these.

-     *

-     * @param recipient The expected recipient.

-     */

-    public void assertMergeOneReply(String recipient) {

-        assertSelect(Arrays.asList(recipient));

-

-        Map<String, Integer> replies = new HashMap<>();

-        replies.put(recipient, ErrorCode.NONE);

-        assertMergeOk(replies, Arrays.asList(recipient));

-

-        replies.put(recipient, ErrorCode.TRANSIENT_ERROR);

-        assertMergeError(replies, Arrays.asList(ErrorCode.TRANSIENT_ERROR));

-    }

-

-    /**

-     * Ensures that the current setup will choose the two given recipients, and that it merges similarly to how the

-     * {@link DocumentProtocol} would merge these.

-     *

-     * @param recipientOne The first expected recipient.

-     * @param recipientTwo The second expected recipient.

-     */

-    public void assertMergeTwoReplies(String recipientOne, String recipientTwo) {

-        assertSelect(Arrays.asList(recipientOne, recipientTwo));

-

-        Map<String, Integer> replies = new HashMap<>();

-        replies.put(recipientOne, ErrorCode.NONE);

-        replies.put(recipientTwo, ErrorCode.NONE);

-        assertMergeOk(replies, Arrays.asList(recipientOne, recipientTwo));

-

-        replies.put(recipientOne, ErrorCode.TRANSIENT_ERROR);

-        replies.put(recipientTwo, ErrorCode.NONE);

-        assertMergeError(replies, Arrays.asList(ErrorCode.TRANSIENT_ERROR));

-

-        replies.put(recipientOne, ErrorCode.TRANSIENT_ERROR);

-        replies.put(recipientTwo, ErrorCode.TRANSIENT_ERROR);

-        assertMergeError(replies, Arrays.asList(ErrorCode.TRANSIENT_ERROR, ErrorCode.TRANSIENT_ERROR));

-

-        replies.put(recipientOne, ErrorCode.NONE);

-        replies.put(recipientTwo, DocumentProtocol.ERROR_MESSAGE_IGNORED);

-        assertMergeOk(replies, Arrays.asList(recipientOne));

-

-        replies.put(recipientOne, DocumentProtocol.ERROR_MESSAGE_IGNORED);

-        replies.put(recipientTwo, ErrorCode.NONE);

-        assertMergeOk(replies, Arrays.asList(recipientTwo));

-

-        replies.put(recipientOne, DocumentProtocol.ERROR_MESSAGE_IGNORED);

-        replies.put(recipientTwo, DocumentProtocol.ERROR_MESSAGE_IGNORED);

-        assertMergeError(replies, Arrays.asList(DocumentProtocol.ERROR_MESSAGE_IGNORED,

-                                                DocumentProtocol.ERROR_MESSAGE_IGNORED));

-    }

-

-    /**

-     * Waits for a given service pattern to resolve to the given number of hits in the local slobrok.

-     *

-     * @param pattern The pattern to lookup.

-     * @param cnt     The number of entries to wait for.

-     * @return True if the expected number of entries was found.

-     */

-    public boolean waitSlobrok(String pattern, int cnt) {

-        for (int i = 0; i < 1000 && !Thread.currentThread().isInterrupted(); ++i) {

-            Mirror.Entry[] res = net.getMirror().lookup(pattern);

-            if (res.length == cnt) {

-                return true;

-            }

-            try { Thread.sleep(10); } catch (InterruptedException e) { /* ignore */ }

-        }

-        return false;

-    }

-

-    /**

-     * Returns the identity of this frame.

-     *

-     * @return The ident string.

-     */

-    public String getIdentity() {

-        return identity;

-    }

-

-    /**

-     * Returns the private slobrok server.

-     *

-     * @return The slobrok.

-     */

-    public Slobrok getSlobrok() {

-        return slobrok;

-    }

-

-    /**

-     * Returns the private message bus.

-     *

-     * @return The bus.

-     */

-    public MessageBus getMessageBus() {

-        return mbus;

-    }

-

-    /**

-     * Returns the private network layer.

-     *

-     * @return The network.

-     */

-    public Network getNetwork() {

-        return net;

-    }

-

-    /**

-     * Returns the message being tested.

-     *

-     * @return The message.

-     */

-    public Message getMessage() {

-        return msg;

-    }

-

-    /**

-     * Sets the message being tested.

-     *

-     * @param msg The message to set.

-     */

-    public void setMessage(Message msg) {

-        this.msg = msg;

-    }

-

-    /**

-     * Sets the spec of the hop to test with.

-     *

-     * @param hop The spec to set.

-     */

-    public void setHop(HopSpec hop) {

-        this.hop = hop;

-        mbus.setupRouting(new RoutingSpec().addTable(new RoutingTableSpec(DocumentProtocol.NAME).addHop(hop)));

-    }

-

-    /**

-     * Returns the reply receptor used by this frame. All messages tested are tagged with this receptor, so after a

-     * successful select, the receptor should contain a non-null reply.

-     *

-     * @return The reply receptor.

-     */

-    public Receptor getReceptor() {

-        return handler;

-    }

-

-    /**

-     * Implements a dummy network.

-     */

-    private class MyNetwork extends RPCNetwork {

-

-        private List<RoutingNode> nodes = new ArrayList<>();

-

-        public MyNetwork(RPCNetworkParams params) {

-            super(params);

-        }

-

-        @Override

-        public boolean allocServiceAddress(RoutingNode recipient) {

-            recipient.setServiceAddress(new ServiceAddress() { });

-            return true;

-        }

-

-        @Override

-        public void freeServiceAddress(RoutingNode recipient) {

-            recipient.setServiceAddress(null);

-        }

-

-        @Override

-        public void send(Message msg, List<RoutingNode> recipients) {

-            this.nodes.addAll(recipients);

-        }

-

-        public List<RoutingNode> removeNodes() {

-            List<RoutingNode> ret = nodes;

-            nodes = new ArrayList<>();

-            return ret;

-        }

-    }

-}

+package com.yahoo.documentapi.messagebus.protocol.test;
+
+import com.yahoo.document.DocumentTypeManager;
+import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol;
+import com.yahoo.jrt.ListenFailedException;
+import com.yahoo.jrt.slobrok.api.Mirror;
+import com.yahoo.jrt.slobrok.server.Slobrok;
+import com.yahoo.messagebus.*;
+import com.yahoo.messagebus.network.Identity;
+import com.yahoo.messagebus.network.Network;
+import com.yahoo.messagebus.network.ServiceAddress;
+import com.yahoo.messagebus.network.rpc.RPCNetwork;
+import com.yahoo.messagebus.network.rpc.RPCNetworkParams;
+import com.yahoo.messagebus.network.rpc.test.TestServer;
+import com.yahoo.messagebus.routing.*;
+import com.yahoo.messagebus.test.Receptor;
+import com.yahoo.messagebus.test.SimpleProtocol;
+import com.yahoo.messagebus.test.SimpleReply;
+
+import java.util.*;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * This is a utility class to allow easier policy test cases. The most important reason to use this is to make sure that
+ * each test uses a "clean" mbus and slobrok instance.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class PolicyTestFrame extends junit.framework.Assert {
+
+    private final AtomicBoolean destroyed = new AtomicBoolean(false);
+    private String identity;
+    private Slobrok slobrok;
+    private MessageBus mbus;
+    private MyNetwork net;
+    private Message msg = null;
+    private HopSpec hop = null;
+    private Receptor handler = new Receptor();
+
+    /**
+     * Create an anonymous test frame.
+     *
+     * @param documentMgr The document manager to use.
+     */
+    public PolicyTestFrame(DocumentTypeManager documentMgr) {
+        this("anonymous", documentMgr);
+    }
+
+    /**
+     * Create a named test frame.
+     *
+     * @param identity    The identity to use for the server.
+     * @param documentMgr The document manager to use.
+     */
+    public PolicyTestFrame(String identity, DocumentTypeManager documentMgr) {
+        this.identity = identity;
+        try {
+            slobrok = new Slobrok();
+        } catch (ListenFailedException e) {
+            e.printStackTrace();
+            fail(e.getMessage());
+        }
+        net = new MyNetwork(new RPCNetworkParams()
+                .setIdentity(new Identity(identity))
+                .setSlobrokConfigId(TestServer.getSlobrokConfig(slobrok)));
+        mbus = new MessageBus(net, new MessageBusParams()
+                .addProtocol(new DocumentProtocol(documentMgr)));
+    }
+
+    /**
+     * Create a test frame running on the same slobrok and mbus as another.
+     *
+     * @param frame The frame whose internals to share.
+     */
+    public PolicyTestFrame(PolicyTestFrame frame) {
+        identity = frame.identity;
+        slobrok = frame.slobrok;
+        net = frame.net;
+        mbus = frame.mbus;
+    }
+
+    // Inherit doc from Object.
+    @Override
+    public void finalize() throws Throwable {
+        destroy();
+        super.finalize();
+    }
+
+    /**
+     * Sets the destroyed flag to true. The very first time this method is called, it cleans up all its dependencies.
+     * Even if you retain a reference to this object, all of its content is allowed to be garbage collected.
+     */
+    public void destroy() {
+        if (!destroyed.getAndSet(true)) {
+            mbus.destroy();
+            mbus = null;
+            net = null;
+
+            slobrok.stop();
+            slobrok = null;
+        }
+    }
+
+    /**
+     * Routes the contained message based on the current setup, and returns the leaf send contexts.
+     *
+     * @param numExpected The expected number of leaf nodes.
+     * @return The list of selected send contexts.
+     */
+    public List<RoutingNode> select(int numExpected) {
+        msg.setRoute(Route.parse(hop.getName()));
+        new RoutingNode(mbus, net, null, handler, msg).send();
+        List<RoutingNode> ret = net.removeNodes();
+        assertEquals(numExpected, ret.size());
+        return ret;
+    }
+
+    /**
+     * Ensures that the current setup selects a given set of routes.
+     *
+     * @param expected A list of expected route leaf nodes.
+     */
+    public void assertSelect(List<String> expected) {
+        if (expected == null) {
+            assertEquals(0, select(0).size());
+        } else {
+            List<RoutingNode> selected = select(expected.size());
+            for (RoutingNode node : selected) {
+                assertTrue("Route '" + node.getRoute() + "' not selected.",
+                           expected.contains(node.getRoute().toString()));
+                node.handleReply(new EmptyReply());
+            }
+        }
+        assertNotNull(handler.getReply(60));
+    }
+
+    /**
+     * This is a convenience method for invoking {@link #assertMerge(Map,List,List)} with no expected value.
+     *
+     * @param replies        The errors to set in the leaf node replies.
+     * @param expectedErrors The list of expected errors in the merged reply.
+     */
+    public void assertMergeError(Map<String, Integer> replies, List<Integer> expectedErrors) {
+        assertMerge(replies, expectedErrors, null);
+    }
+
+    /**
+     * This is a convenience method for invoking {@link this#assertMerge(Map,List,List)} with no expected errors.
+     *
+     * @param replies       The errors to set in the leaf node replies.
+     * @param allowedValues The list of allowed values in the final reply.
+     */
+    public void assertMergeOk(Map<String, Integer> replies, List<String> allowedValues) {
+        assertMerge(replies, null, allowedValues);
+    }
+
+    /**
+     * Ensures that the current setup generates as many leaf nodes as there are members of the errors argument. Each
+     * error is then given one of these errors, and the method then ensures that the single returned reply contains the
+     * given list of expected errors. Finally, if the expected value argument is non-null, this method ensures that the
+     * reply is a SimpleReply whose string value exists in the allowed list.
+     *
+     * @param replies        The errors to set in the leaf node replies.
+     * @param expectedErrors The list of expected errors in the merged reply.
+     * @param allowedValues  The list of allowed values in the final reply.
+     */
+    public void assertMerge(Map<String, Integer> replies, List<Integer> expectedErrors, List<String> allowedValues) {
+        List<RoutingNode> selected = select(replies.size());
+
+        for (RoutingNode node : selected) {
+            String route = node.getRoute().toString();
+            assertTrue(replies.containsKey(route));
+            Reply ret = new SimpleReply(route);
+            if (replies.get(route) != ErrorCode.NONE) {
+                ret.addError(new com.yahoo.messagebus.Error(replies.get(route), route));
+            }
+            node.handleReply(ret);
+        }
+
+        Reply reply = handler.getReply(60);
+        assertNotNull(reply);
+        if (expectedErrors != null) {
+            assertEquals(expectedErrors.size(), reply.getNumErrors());
+            for (int i = 0; i < expectedErrors.size(); ++i) {
+                assertTrue(expectedErrors.contains(reply.getError(i).getCode()));
+            }
+        } else if (reply.hasErrors()) {
+            StringBuilder err = new StringBuilder("Got unexpected error(s):\n");
+            for (int i = 0; i < reply.getNumErrors(); ++i) {
+                err.append("\t").append(reply.getError(i).toString());
+                if (i < reply.getNumErrors() - 1) {
+                    err.append("\n");
+                }
+            }
+            fail(err.toString());
+        }
+        if (allowedValues != null) {
+            assertEquals(SimpleProtocol.REPLY, reply.getType());
+            assertTrue(allowedValues.contains(((SimpleReply)reply).getValue()));
+        } else {
+            assertEquals(0, reply.getType());
+        }
+    }
+
+    /**
+     * Ensures that the current setup chooses a single recipient, and that it merges similarly to how the
+     * {@link DocumentProtocol} would merge these.
+     *
+     * @param recipient The expected recipient.
+     */
+    public void assertMergeOneReply(String recipient) {
+        assertSelect(Arrays.asList(recipient));
+
+        Map<String, Integer> replies = new HashMap<>();
+        replies.put(recipient, ErrorCode.NONE);
+        assertMergeOk(replies, Arrays.asList(recipient));
+
+        replies.put(recipient, ErrorCode.TRANSIENT_ERROR);
+        assertMergeError(replies, Arrays.asList(ErrorCode.TRANSIENT_ERROR));
+    }
+
+    /**
+     * Ensures that the current setup will choose the two given recipients, and that it merges similarly to how the
+     * {@link DocumentProtocol} would merge these.
+     *
+     * @param recipientOne The first expected recipient.
+     * @param recipientTwo The second expected recipient.
+     */
+    public void assertMergeTwoReplies(String recipientOne, String recipientTwo) {
+        assertSelect(Arrays.asList(recipientOne, recipientTwo));
+
+        Map<String, Integer> replies = new HashMap<>();
+        replies.put(recipientOne, ErrorCode.NONE);
+        replies.put(recipientTwo, ErrorCode.NONE);
+        assertMergeOk(replies, Arrays.asList(recipientOne, recipientTwo));
+
+        replies.put(recipientOne, ErrorCode.TRANSIENT_ERROR);
+        replies.put(recipientTwo, ErrorCode.NONE);
+        assertMergeError(replies, Arrays.asList(ErrorCode.TRANSIENT_ERROR));
+
+        replies.put(recipientOne, ErrorCode.TRANSIENT_ERROR);
+        replies.put(recipientTwo, ErrorCode.TRANSIENT_ERROR);
+        assertMergeError(replies, Arrays.asList(ErrorCode.TRANSIENT_ERROR, ErrorCode.TRANSIENT_ERROR));
+
+        replies.put(recipientOne, ErrorCode.NONE);
+        replies.put(recipientTwo, DocumentProtocol.ERROR_MESSAGE_IGNORED);
+        assertMergeOk(replies, Arrays.asList(recipientOne));
+
+        replies.put(recipientOne, DocumentProtocol.ERROR_MESSAGE_IGNORED);
+        replies.put(recipientTwo, ErrorCode.NONE);
+        assertMergeOk(replies, Arrays.asList(recipientTwo));
+
+        replies.put(recipientOne, DocumentProtocol.ERROR_MESSAGE_IGNORED);
+        replies.put(recipientTwo, DocumentProtocol.ERROR_MESSAGE_IGNORED);
+        assertMergeError(replies, Arrays.asList(DocumentProtocol.ERROR_MESSAGE_IGNORED,
+                                                DocumentProtocol.ERROR_MESSAGE_IGNORED));
+    }
+
+    /**
+     * Waits for a given service pattern to resolve to the given number of hits in the local slobrok.
+     *
+     * @param pattern The pattern to lookup.
+     * @param cnt     The number of entries to wait for.
+     * @return True if the expected number of entries was found.
+     */
+    public boolean waitSlobrok(String pattern, int cnt) {
+        for (int i = 0; i < 1000 && !Thread.currentThread().isInterrupted(); ++i) {
+            Mirror.Entry[] res = net.getMirror().lookup(pattern);
+            if (res.length == cnt) {
+                return true;
+            }
+            try { Thread.sleep(10); } catch (InterruptedException e) { /* ignore */ }
+        }
+        return false;
+    }
+
+    /**
+     * Returns the identity of this frame.
+     *
+     * @return The ident string.
+     */
+    public String getIdentity() {
+        return identity;
+    }
+
+    /**
+     * Returns the private slobrok server.
+     *
+     * @return The slobrok.
+     */
+    public Slobrok getSlobrok() {
+        return slobrok;
+    }
+
+    /**
+     * Returns the private message bus.
+     *
+     * @return The bus.
+     */
+    public MessageBus getMessageBus() {
+        return mbus;
+    }
+
+    /**
+     * Returns the private network layer.
+     *
+     * @return The network.
+     */
+    public Network getNetwork() {
+        return net;
+    }
+
+    /**
+     * Returns the message being tested.
+     *
+     * @return The message.
+     */
+    public Message getMessage() {
+        return msg;
+    }
+
+    /**
+     * Sets the message being tested.
+     *
+     * @param msg The message to set.
+     */
+    public void setMessage(Message msg) {
+        this.msg = msg;
+    }
+
+    /**
+     * Sets the spec of the hop to test with.
+     *
+     * @param hop The spec to set.
+     */
+    public void setHop(HopSpec hop) {
+        this.hop = hop;
+        mbus.setupRouting(new RoutingSpec().addTable(new RoutingTableSpec(DocumentProtocol.NAME).addHop(hop)));
+    }
+
+    /**
+     * Returns the reply receptor used by this frame. All messages tested are tagged with this receptor, so after a
+     * successful select, the receptor should contain a non-null reply.
+     *
+     * @return The reply receptor.
+     */
+    public Receptor getReceptor() {
+        return handler;
+    }
+
+    /**
+     * Implements a dummy network.
+     */
+    private class MyNetwork extends RPCNetwork {
+
+        private List<RoutingNode> nodes = new ArrayList<>();
+
+        public MyNetwork(RPCNetworkParams params) {
+            super(params);
+        }
+
+        @Override
+        public boolean allocServiceAddress(RoutingNode recipient) {
+            recipient.setServiceAddress(new ServiceAddress() { });
+            return true;
+        }
+
+        @Override
+        public void freeServiceAddress(RoutingNode recipient) {
+            recipient.setServiceAddress(null);
+        }
+
+        @Override
+        public void send(Message msg, List<RoutingNode> recipients) {
+            this.nodes.addAll(recipients);
+        }
+
+        public List<RoutingNode> removeNodes() {
+            List<RoutingNode> ret = nodes;
+            nodes = new ArrayList<>();
+            return ret;
+        }
+    }
+}
diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/RoutableFactoryTestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/RoutableFactoryTestCase.java
index 8ba217ce15d..c4650dd6099 100755
--- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/RoutableFactoryTestCase.java
+++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/RoutableFactoryTestCase.java
@@ -1,188 +1,188 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.documentapi.messagebus.protocol.test;

-

-import com.yahoo.component.VersionSpecification;

-import com.yahoo.document.DocumentTypeManager;

-import com.yahoo.document.serialization.DocumentDeserializer;

-import com.yahoo.document.serialization.DocumentSerializer;

-import com.yahoo.documentapi.messagebus.protocol.DocumentMessage;

-import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol;

-import com.yahoo.documentapi.messagebus.protocol.DocumentReply;

-import com.yahoo.documentapi.messagebus.protocol.RoutableFactories50;

-import com.yahoo.jrt.ListenFailedException;

-import com.yahoo.jrt.slobrok.server.Slobrok;

-import com.yahoo.messagebus.*;

-import com.yahoo.messagebus.network.Identity;

-import com.yahoo.messagebus.network.rpc.RPCNetworkParams;

-import com.yahoo.messagebus.network.rpc.test.TestServer;

-import com.yahoo.messagebus.routing.Route;

-import com.yahoo.messagebus.test.Receptor;

-import junit.framework.TestCase;

-

-/**

- * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>

- */

-public class RoutableFactoryTestCase extends TestCase {

-

-    ////////////////////////////////////////////////////////////////////////////////

-    //

-    // Setup

-    //

-    ////////////////////////////////////////////////////////////////////////////////

-

-    private Slobrok slobrok;

-    private DocumentProtocol srcProtocol, dstProtocol;

-    private TestServer srcServer, dstServer;

-    private SourceSession srcSession;

-    private DestinationSession dstSession;

-

-    @Override

-    public void setUp() throws ListenFailedException {

-        slobrok = new Slobrok();

-        DocumentTypeManager docMan = new DocumentTypeManager();

-        dstProtocol = new DocumentProtocol(docMan);

-        dstServer = new TestServer(new MessageBusParams().addProtocol(dstProtocol),

-                                   new RPCNetworkParams().setIdentity(new Identity("dst")).setSlobrokConfigId(TestServer.getSlobrokConfig(slobrok)));

-        dstSession = dstServer.mb.createDestinationSession(new DestinationSessionParams().setName("session").setMessageHandler(new Receptor()));

-        srcProtocol = new DocumentProtocol(docMan);

-        srcServer = new TestServer(new MessageBusParams().addProtocol(srcProtocol),

-                                   new RPCNetworkParams().setSlobrokConfigId(TestServer.getSlobrokConfig(slobrok)));

-        srcSession = srcServer.mb.createSourceSession(new SourceSessionParams().setReplyHandler(new Receptor()));

-        assertTrue(srcServer.waitSlobrok("dst/session", 1));

-    }

-

-    @Override

-    public void tearDown() {

-        slobrok.stop();

-        dstSession.destroy();

-        dstServer.destroy();

-        srcSession.destroy();

-        srcServer.destroy();

-    }

-

-    ////////////////////////////////////////////////////////////////////////////////

-    //

-    // Tests

-    //

-    ////////////////////////////////////////////////////////////////////////////////

-

-    public void testFactory() {

-        Route route = Route.parse("dst/session");

-

-        // Source should fail to encode the message.

-        assertTrue(srcSession.send(new MyMessage(), route).isAccepted());

-        Reply reply = ((Receptor)srcSession.getReplyHandler()).getReply(60);

-        assertNotNull(reply);

-        System.out.println(reply.getTrace());

-        assertTrue(reply.hasErrors());

-        assertEquals(ErrorCode.ENCODE_ERROR, reply.getError(0).getCode());

-        assertNull(reply.getError(0).getService());

-

-        // Destination should fail to decode the message.

-        srcProtocol.putRoutableFactory(MyMessage.TYPE, new MyMessageFactory(), new VersionSpecification());

-        assertTrue(srcSession.send(new MyMessage(), route).isAccepted());

-        assertNotNull(reply = ((Receptor)srcSession.getReplyHandler()).getReply(60));

-        System.out.println(reply.getTrace());

-        assertTrue(reply.hasErrors());

-        assertEquals(ErrorCode.DECODE_ERROR, reply.getError(0).getCode());

-        assertEquals("dst/session", reply.getError(0).getService());

-

-        // Destination should fail to encode the reply.

-        dstProtocol.putRoutableFactory(MyMessage.TYPE, new MyMessageFactory(), new VersionSpecification());

-        assertTrue(srcSession.send(new MyMessage(), route).isAccepted());

-        Message msg = ((Receptor)dstSession.getMessageHandler()).getMessage(60);

-        assertNotNull(msg);

-        reply = new MyReply();

-        reply.swapState(msg);

-        dstSession.reply(reply);

-        assertNotNull(reply = ((Receptor)srcSession.getReplyHandler()).getReply(60));

-        System.out.println(reply.getTrace());

-        assertTrue(reply.hasErrors());

-        assertEquals(ErrorCode.ENCODE_ERROR, reply.getError(0).getCode());

-        assertEquals("dst/session", reply.getError(0).getService());

-

-        // Source should fail to decode the reply.

-        dstProtocol.putRoutableFactory(MyReply.TYPE, new MyReplyFactory(), new VersionSpecification());

-        assertTrue(srcSession.send(new MyMessage(), route).isAccepted());

-        assertNotNull(msg = ((Receptor)dstSession.getMessageHandler()).getMessage(60));

-        reply = new MyReply();

-        reply.swapState(msg);

-        dstSession.reply(reply);

-        assertNotNull(reply = ((Receptor)srcSession.getReplyHandler()).getReply(60));

-        System.out.println(reply.getTrace());

-        assertTrue(reply.hasErrors());

-        assertEquals(ErrorCode.DECODE_ERROR, reply.getError(0).getCode());

-        assertNull(reply.getError(0).getService());

-

-        // All should succeed.

-        srcProtocol.putRoutableFactory(MyReply.TYPE, new MyReplyFactory(), new VersionSpecification());

-        assertTrue(srcSession.send(new MyMessage(), route).isAccepted());

-        assertNotNull(msg = ((Receptor)dstSession.getMessageHandler()).getMessage(60));

-        reply = new MyReply();

-        reply.swapState(msg);

-        dstSession.reply(reply);

-        assertNotNull(reply = ((Receptor)srcSession.getReplyHandler()).getReply(60));

-        System.out.println(reply.getTrace());

-        assertFalse(reply.hasErrors());

-    }

-

-    ////////////////////////////////////////////////////////////////////////////////

-    //

-    // Utilities

-    //

-    ////////////////////////////////////////////////////////////////////////////////

-

-    private static class MyMessageFactory extends RoutableFactories50.DocumentMessageFactory {

-

-        @Override

-        protected DocumentMessage doDecode(DocumentDeserializer buf) {

-            return new MyMessage();

-        }

-

-        @Override

-        protected boolean doEncode(DocumentMessage msg, DocumentSerializer buf) {

-            return true;

-        }

-    }

-

-    private static class MyReplyFactory extends RoutableFactories50.DocumentReplyFactory {

-

-        @Override

-        protected DocumentReply doDecode(DocumentDeserializer buf) {

-            return new MyReply();

-        }

-

-        @Override

-        protected boolean doEncode(DocumentReply msg, DocumentSerializer buf) {

-            return true;

-        }

-    }

-

-    private static class MyMessage extends DocumentMessage {

-

-        final static int TYPE = 666;

-

-        MyMessage() {

-            getTrace().setLevel(9);

-        }

-

-        @Override

-        public DocumentReply createReply() {

-            return new MyReply();

-        }

-

-        @Override

-        public int getType() {

-            return TYPE;

-        }

-    }

-

-    private static class MyReply extends DocumentReply {

-

-        final static int TYPE = 777;

-

-        public MyReply() {

-            super(TYPE);

-        }

-    }

-}

+package com.yahoo.documentapi.messagebus.protocol.test;
+
+import com.yahoo.component.VersionSpecification;
+import com.yahoo.document.DocumentTypeManager;
+import com.yahoo.document.serialization.DocumentDeserializer;
+import com.yahoo.document.serialization.DocumentSerializer;
+import com.yahoo.documentapi.messagebus.protocol.DocumentMessage;
+import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol;
+import com.yahoo.documentapi.messagebus.protocol.DocumentReply;
+import com.yahoo.documentapi.messagebus.protocol.RoutableFactories50;
+import com.yahoo.jrt.ListenFailedException;
+import com.yahoo.jrt.slobrok.server.Slobrok;
+import com.yahoo.messagebus.*;
+import com.yahoo.messagebus.network.Identity;
+import com.yahoo.messagebus.network.rpc.RPCNetworkParams;
+import com.yahoo.messagebus.network.rpc.test.TestServer;
+import com.yahoo.messagebus.routing.Route;
+import com.yahoo.messagebus.test.Receptor;
+import junit.framework.TestCase;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class RoutableFactoryTestCase extends TestCase {
+
+    ////////////////////////////////////////////////////////////////////////////////
+    //
+    // Setup
+    //
+    ////////////////////////////////////////////////////////////////////////////////
+
+    private Slobrok slobrok;
+    private DocumentProtocol srcProtocol, dstProtocol;
+    private TestServer srcServer, dstServer;
+    private SourceSession srcSession;
+    private DestinationSession dstSession;
+
+    @Override
+    public void setUp() throws ListenFailedException {
+        slobrok = new Slobrok();
+        DocumentTypeManager docMan = new DocumentTypeManager();
+        dstProtocol = new DocumentProtocol(docMan);
+        dstServer = new TestServer(new MessageBusParams().addProtocol(dstProtocol),
+                                   new RPCNetworkParams().setIdentity(new Identity("dst")).setSlobrokConfigId(TestServer.getSlobrokConfig(slobrok)));
+        dstSession = dstServer.mb.createDestinationSession(new DestinationSessionParams().setName("session").setMessageHandler(new Receptor()));
+        srcProtocol = new DocumentProtocol(docMan);
+        srcServer = new TestServer(new MessageBusParams().addProtocol(srcProtocol),
+                                   new RPCNetworkParams().setSlobrokConfigId(TestServer.getSlobrokConfig(slobrok)));
+        srcSession = srcServer.mb.createSourceSession(new SourceSessionParams().setReplyHandler(new Receptor()));
+        assertTrue(srcServer.waitSlobrok("dst/session", 1));
+    }
+
+    @Override
+    public void tearDown() {
+        slobrok.stop();
+        dstSession.destroy();
+        dstServer.destroy();
+        srcSession.destroy();
+        srcServer.destroy();
+    }
+
+    ////////////////////////////////////////////////////////////////////////////////
+    //
+    // Tests
+    //
+    ////////////////////////////////////////////////////////////////////////////////
+
+    public void testFactory() {
+        Route route = Route.parse("dst/session");
+
+        // Source should fail to encode the message.
+        assertTrue(srcSession.send(new MyMessage(), route).isAccepted());
+        Reply reply = ((Receptor)srcSession.getReplyHandler()).getReply(60);
+        assertNotNull(reply);
+        System.out.println(reply.getTrace());
+        assertTrue(reply.hasErrors());
+        assertEquals(ErrorCode.ENCODE_ERROR, reply.getError(0).getCode());
+        assertNull(reply.getError(0).getService());
+
+        // Destination should fail to decode the message.
+        srcProtocol.putRoutableFactory(MyMessage.TYPE, new MyMessageFactory(), new VersionSpecification());
+        assertTrue(srcSession.send(new MyMessage(), route).isAccepted());
+        assertNotNull(reply = ((Receptor)srcSession.getReplyHandler()).getReply(60));
+        System.out.println(reply.getTrace());
+        assertTrue(reply.hasErrors());
+        assertEquals(ErrorCode.DECODE_ERROR, reply.getError(0).getCode());
+        assertEquals("dst/session", reply.getError(0).getService());
+
+        // Destination should fail to encode the reply.
+        dstProtocol.putRoutableFactory(MyMessage.TYPE, new MyMessageFactory(), new VersionSpecification());
+        assertTrue(srcSession.send(new MyMessage(), route).isAccepted());
+        Message msg = ((Receptor)dstSession.getMessageHandler()).getMessage(60);
+        assertNotNull(msg);
+        reply = new MyReply();
+        reply.swapState(msg);
+        dstSession.reply(reply);
+        assertNotNull(reply = ((Receptor)srcSession.getReplyHandler()).getReply(60));
+        System.out.println(reply.getTrace());
+        assertTrue(reply.hasErrors());
+        assertEquals(ErrorCode.ENCODE_ERROR, reply.getError(0).getCode());
+        assertEquals("dst/session", reply.getError(0).getService());
+
+        // Source should fail to decode the reply.
+        dstProtocol.putRoutableFactory(MyReply.TYPE, new MyReplyFactory(), new VersionSpecification());
+        assertTrue(srcSession.send(new MyMessage(), route).isAccepted());
+        assertNotNull(msg = ((Receptor)dstSession.getMessageHandler()).getMessage(60));
+        reply = new MyReply();
+        reply.swapState(msg);
+        dstSession.reply(reply);
+        assertNotNull(reply = ((Receptor)srcSession.getReplyHandler()).getReply(60));
+        System.out.println(reply.getTrace());
+        assertTrue(reply.hasErrors());
+        assertEquals(ErrorCode.DECODE_ERROR, reply.getError(0).getCode());
+        assertNull(reply.getError(0).getService());
+
+        // All should succeed.
+        srcProtocol.putRoutableFactory(MyReply.TYPE, new MyReplyFactory(), new VersionSpecification());
+        assertTrue(srcSession.send(new MyMessage(), route).isAccepted());
+        assertNotNull(msg = ((Receptor)dstSession.getMessageHandler()).getMessage(60));
+        reply = new MyReply();
+        reply.swapState(msg);
+        dstSession.reply(reply);
+        assertNotNull(reply = ((Receptor)srcSession.getReplyHandler()).getReply(60));
+        System.out.println(reply.getTrace());
+        assertFalse(reply.hasErrors());
+    }
+
+    ////////////////////////////////////////////////////////////////////////////////
+    //
+    // Utilities
+    //
+    ////////////////////////////////////////////////////////////////////////////////
+
+    private static class MyMessageFactory extends RoutableFactories50.DocumentMessageFactory {
+
+        @Override
+        protected DocumentMessage doDecode(DocumentDeserializer buf) {
+            return new MyMessage();
+        }
+
+        @Override
+        protected boolean doEncode(DocumentMessage msg, DocumentSerializer buf) {
+            return true;
+        }
+    }
+
+    private static class MyReplyFactory extends RoutableFactories50.DocumentReplyFactory {
+
+        @Override
+        protected DocumentReply doDecode(DocumentDeserializer buf) {
+            return new MyReply();
+        }
+
+        @Override
+        protected boolean doEncode(DocumentReply msg, DocumentSerializer buf) {
+            return true;
+        }
+    }
+
+    private static class MyMessage extends DocumentMessage {
+
+        final static int TYPE = 666;
+
+        MyMessage() {
+            getTrace().setLevel(9);
+        }
+
+        @Override
+        public DocumentReply createReply() {
+            return new MyReply();
+        }
+
+        @Override
+        public int getType() {
+            return TYPE;
+        }
+    }
+
+    private static class MyReply extends DocumentReply {
+
+        final static int TYPE = 777;
+
+        public MyReply() {
+            super(TYPE);
+        }
+    }
+}