Merge branch 'master' into hmusum/remove-support-for-removed-api

author: Harald Musum <musum@yahoo-inc.com> 2017-06-07 10:38:51 +0200
committer: Harald Musum <musum@yahoo-inc.com> 2017-06-07 10:38:51 +0200
commit: 125a73cd75b8db5caf0a67f7019faf57411e3a45 (patch)
tree: c8033a7954d57393580afe3640acf39af1193f3c
parent: f7ca4ca6e113aba0f7648d6ddcac18ee7757fb79 (diff)
parent: 86e1f3e7b711c2d7787ed8a546f9c809c4ccc787 (diff)
305 files changed, 7868 insertions, 794 deletions
diff --git a/.gitignore b/.gitignore
index 7bc8af51e33..14366c5c31b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -40,3 +40,4 @@ Testing
 /hadoop/dependency-reduced-pom.xml
 /vespa-hadoop/dependency-reduced-pom.xml
 .preprocessed/
+install_manifest.txt
diff --git a/config-model/src/main/perl/deploy b/config-model/src/main/perl/deploy
index 1a594065a33..dc7b2132792 100755
--- a/config-model/src/main/perl/deploy
+++ b/config-model/src/main/perl/deploy
@@ -72,7 +72,7 @@ readConfFile();
 use strict;
 use warnings;
 use feature qw(switch say);
-use vars qw/ $opt_h $opt_n $opt_v $opt_f $opt_t $opt_a $opt_e $opt_E $opt_r $opt_i $opt_p $opt_H $opt_R /;
+use vars qw/ $opt_c $opt_h $opt_n $opt_v $opt_f $opt_t $opt_a $opt_e $opt_E $opt_r $opt_i $opt_p $opt_H $opt_R /;
 use Env qw($HOME);
 use JSON;
 use Getopt::Std;
@@ -99,15 +99,20 @@ my $environment = "prod";
 my $region = "default";
 my $instance = "default";
 my $version = "v2";
+my $configserver = "";
 my $port = "19071";
 
-getopts('fhnt:ve:E:r:a:i:p:HR:');
+getopts('c:fhnt:ve:E:r:a:i:p:HR:');
 
 if ($opt_h) {
   usage();
   exit 0;
 }
 
+if ($opt_c) {
+    $configserver = $opt_c;
+}
+
 if ($opt_e) {
   $tenant = $opt_e;
 }
@@ -297,7 +302,13 @@ sub fetch_directory {
 sub get_configsource_url {
   my ($command) = @_;
 
-  my @configsources = split(' ', `$VESPA_HOME/bin/vespa-print-default configservers_http`);
+  my @configsources;
+  if ($configserver and $configserver ne "") {
+      @configsources = ('http://' . $configserver . ':' . $port . '/');
+  } else {
+      @configsources = split(' ', `$VESPA_HOME/bin/vespa-print-default configservers_http`);
+  }
+
   my $configsource_url = shift(@configsources);
   if (!$configsource_url) {
     die "Could not get url to config server, make sure that VESPA_HOME and services.addr_configserver is set\n";
diff --git a/config/src/tests/configholder/configholder.cpp b/config/src/tests/configholder/configholder.cpp
index 8901ba65ad4..c103f86f6e9 100644
--- a/config/src/tests/configholder/configholder.cpp
+++ b/config/src/tests/configholder/configholder.cpp
@@ -61,7 +61,7 @@ TEST_MT_F("Require that wait is interrupted", 2, ConfigHolder)
         TEST_BARRIER();
     } else {
         TEST_BARRIER();
-        FastOS_Thread::Sleep(500);
+        std::this_thread::sleep_for(std::chrono::milliseconds(500));
         f.interrupt();
         TEST_BARRIER();
     }
diff --git a/config/src/tests/subscription/subscription.cpp b/config/src/tests/subscription/subscription.cpp
index 5974279518a..52b9ec624ac 100644
--- a/config/src/tests/subscription/subscription.cpp
+++ b/config/src/tests/subscription/subscription.cpp
@@ -79,7 +79,7 @@ TEST_MT_F("requireThatNextUpdateReturnsWhenNotified", 2, SubscriptionFixture(Con
         ASSERT_TRUE(f1.sub.nextUpdate(2, 5000));
         ASSERT_TRUE(timer.MilliSecsToNow() > 200.0);
     } else {
-        FastOS_Thread::Sleep(500);
+        std::this_thread::sleep_for(std::chrono::milliseconds(500));
         f1.holder->handle(ConfigUpdate::UP(new ConfigUpdate(ConfigValue(), 1, 1)));
     }
 }
@@ -94,7 +94,7 @@ TEST_MT_F("requireThatNextUpdateReturnsInterrupted", 2, SubscriptionFixture(Conf
         ASSERT_TRUE(f1.sub.nextUpdate(1, 5000));
         ASSERT_TRUE(timer.MilliSecsToNow() > 300.0);
     } else {
-        FastOS_Thread::Sleep(500);
+        std::this_thread::sleep_for(std::chrono::milliseconds(500));
         f1.sub.close();
     }
 }
diff --git a/config/src/vespa/config/common/configcontext.h b/config/src/vespa/config/common/configcontext.h
index c00bceaef47..4f19be9a928 100644
--- a/config/src/vespa/config/common/configcontext.h
+++ b/config/src/vespa/config/common/configcontext.h
@@ -5,8 +5,6 @@
 #include "timingvalues.h"
 #include "configmanager.h"
 #include <vespa/config/subscription/sourcespec.h>
-#include <vespa/vespalib/util/sync.h>
-#include <vespa/vespalib/stllike/hash_map.h>
 
 namespace config {
 
diff --git a/config/src/vespa/config/common/configparser.cpp b/config/src/vespa/config/common/configparser.cpp
index fde55f257dc..e0a0b0138b9 100644
--- a/config/src/vespa/config/common/configparser.cpp
+++ b/config/src/vespa/config/common/configparser.cpp
@@ -1,9 +1,8 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-#include <vespa/config/common/configparser.h>
-#include <vespa/vespalib/stllike/string.h>
-#include <vespa/vespalib/stllike/asciistream.h>
-#include <stdio.h>
+
+#include "configparser.h"
 #include "misc.h"
+#include <vespa/vespalib/stllike/asciistream.h>
 
 namespace config {
 
diff --git a/config/src/vespa/config/common/configparser.h b/config/src/vespa/config/common/configparser.h
index 3b1185a48fa..613dfc33d94 100644
--- a/config/src/vespa/config/common/configparser.h
+++ b/config/src/vespa/config/common/configparser.h
@@ -2,12 +2,12 @@
 #pragma once
 
 #include <vespa/config/common/exceptions.h>
+#include <vespa/vespalib/util/stringfmt.h>
 #include <map>
 #include <set>
 #include <vector>
-#include <errno.h>
-#include <stdint.h>
-#include <vespa/vespalib/stllike/string.h>
+#include <cerrno>
+#include <cstdint>
 
 namespace config {
 
diff --git a/config/src/vespa/config/common/exceptions.cpp b/config/src/vespa/config/common/exceptions.cpp
index d19c27b5999..b5cbe7f0e06 100644
--- a/config/src/vespa/config/common/exceptions.cpp
+++ b/config/src/vespa/config/common/exceptions.cpp
@@ -1,6 +1,7 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "exceptions.h"
+#include <vespa/vespalib/util/stringfmt.h>
 
 namespace config {
 
diff --git a/configgen/src/main/java/com/yahoo/config/codegen/CppClassBuilder.java b/configgen/src/main/java/com/yahoo/config/codegen/CppClassBuilder.java
index 997416e5bfc..49f5275c7fe 100644
--- a/configgen/src/main/java/com/yahoo/config/codegen/CppClassBuilder.java
+++ b/configgen/src/main/java/com/yahoo/config/codegen/CppClassBuilder.java
@@ -8,10 +8,12 @@ import java.io.IOException;
 import java.io.FileReader;
 import java.io.StringWriter;
 import java.io.Writer;
-import java.util.Collections;
-import java.util.HashMap;
 import java.util.Map;
+import java.util.HashMap;
+import java.util.Collections;
+import java.util.Arrays;
 import java.util.StringTokenizer;
+import java.util.stream.Collectors;
 
 
 /**
@@ -189,15 +191,19 @@ public class CppClassBuilder implements ClassBuilder {
     }
 
     void writeNameSpaceBegin(Writer w, String [] namespaceList) throws IOException {
-        for (int i = 0; i < namespaceList.length; i++) {
-            w.write("namespace " + namespaceList[i] + " {\n\n");
-        }
+        w.write("namespace ");
+        w.write(getNestedNameSpace(namespaceList));
+        w.write(" {\n");
+    }
+
+    String getNestedNameSpace(String [] namespaceList) {
+        return Arrays.stream(namespaceList).map(String::toString).collect(Collectors.joining("::"));
     }
 
     void writeNameSpaceEnd(Writer w, String [] namespaceList) throws IOException {
-        for (int i = 0; i < namespaceList.length; i++) {
-            w.write("} // namespace " + namespaceList[i] + "\n\n");
-        }
+        w.write("} // namespace ");
+        w.write(getNestedNameSpace(namespaceList));
+        w.write("\n");
     }
 
     void writeHeaderHeader(Writer w, CNode root) throws IOException {
@@ -229,17 +235,21 @@ public class CppClassBuilder implements ClassBuilder {
                 + "#ifndef CLOUD_CONFIG_" + defineName + "_H\n"
                 + "#define CLOUD_CONFIG_" + defineName + "_H\n"
                 + "\n"
-                + "#include <vespa/config/common/configvalue.h>\n"
-                + "#include <vespa/config/configgen/configpayload.h>\n"
                 + "#include <vespa/config/configgen/configinstance.h>\n"
-                + "#include <vespa/config/print/configdatabuffer.h>\n"
                 + "#include <vespa/vespalib/stllike/string.h>\n"
                 + "#include <vector>\n"
                 + "#include <map>\n"
                 + "\n");
+        w.write("namespace config {\n");
+        w.write("    class ConfigValue;\n");
+        w.write("    class ConfigPayload;\n");
+        w.write("}\n\n");
+        w.write("namespace vespalib::slime {\n");
+        w.write("    class Inspector;\n");
+        w.write("    class Cursor;\n");
+        w.write("}\n\n");
         writeNameSpaceBegin(w, namespaceList);
-        w.write("\n");
-        w.write("namespace internal {\n\n");
+        w.write("\nnamespace internal {\n\n");
         w.write(""
                 + "/**\n"
                 + " * This class contains the config. DO NOT USE THIS CLASS DIRECTLY. Use the typedeffed\n"
@@ -551,17 +561,18 @@ public class CppClassBuilder implements ClassBuilder {
             w.write("#include <" + subdir + "/" + getFileName(root, "h") + ">");
         }
         w.write("\n");
+        w.write("#include <vespa/config/common/configvalue.h>\n");
+        w.write("#include <vespa/config/configgen/configpayload.h>\n");
+        w.write("#include <vespa/config/print/configdatabuffer.h>\n");
         w.write("#include <vespa/config/common/configparser.h>\n");
         w.write("#include <vespa/config/configgen/vector_inserter.h>\n");
         w.write("#include <vespa/config/configgen/map_inserter.h>\n");
         w.write("#include <vespa/vespalib/data/slime/convenience.h>\n");
         w.write("#include <vespa/vespalib/data/slime/slime.h>\n");
         w.write("#include <vespa/vespalib/stllike/asciistream.h>\n");
-        w.write("#include <set>\n");
-        w.write("\n\n");
-        writeNameSpaceBegin(w, generateCppNameSpace(root));
         w.write("\n");
-        w.write("namespace internal {\n\n");
+        writeNameSpaceBegin(w, generateCppNameSpace(root));
+        w.write("\nnamespace internal {\n\n");
         w.write("using ::config::ConfigParser;\n");
         w.write("using ::config::InvalidConfigException;\n");
         w.write("using ::config::ConfigInstance;\n");
diff --git a/dist/post_install.sh b/dist/post_install.sh
index ddaa5f0ff00..6c70863806c 100755
--- a/dist/post_install.sh
+++ b/dist/post_install.sh
@@ -68,3 +68,11 @@ ln -s $PREFIX/lib/jars/node-repository-jar-with-dependencies.jar $INSTALLPATH/co
 ln -s $PREFIX/lib/jars/zkfacade-jar-with-dependencies.jar $INSTALLPATH/conf/configserver-app/components/zkfacade.jar
 ln -s $PREFIX/conf/configserver-app/components $INSTALLPATH/lib/jars/config-models
 ln -s storaged-bin $INSTALLPATH/sbin/distributord-bin
+
+# Temporary when renaming binaries in searchcore
+ln -s vespa-proton-bin $INSTALLPATH/sbin/proton-bin
+ln -s vespa-fdispatch-bin $INSTALLPATH/sbin/fdispatch-bin
+ln -s vespa-verify-ranksetup-bin $INSTALLPATH/bin/verify_ranksetup-bin
+ln -s vespa-proton $INSTALLPATH/sbin/proton
+ln -s vespa-fdispatch $INSTALLPATH/sbin/fdispatch
+ln -s vespa-verify-ranksetup $INSTALLPATH/bin/verify_ranksetup
diff --git a/document/src/tests/arrayfieldvaluetest.cpp b/document/src/tests/arrayfieldvaluetest.cpp
index 7f31e0c3b3a..0535edc495b 100644
--- a/document/src/tests/arrayfieldvaluetest.cpp
+++ b/document/src/tests/arrayfieldvaluetest.cpp
@@ -4,6 +4,7 @@
 #include <vespa/document/serialization/vespadocumentdeserializer.h>
 #include <vespa/vdstestlib/cppunit/macros.h>
 #include <vespa/vespalib/objects/nbostream.h>
+#include <vespa/document/util/bytebuffer.h>
 
 using vespalib::nbostream;
 
diff --git a/document/src/tests/documentcalculatortestcase.cpp b/document/src/tests/documentcalculatortestcase.cpp
index a292302ad61..bc112c80a8c 100644
--- a/document/src/tests/documentcalculatortestcase.cpp
+++ b/document/src/tests/documentcalculatortestcase.cpp
@@ -9,6 +9,7 @@
 #include <vespa/document/fieldvalue/intfieldvalue.h>
 #include <vespa/document/fieldvalue/longfieldvalue.h>
 #include <vespa/document/fieldvalue/floatfieldvalue.h>
+#include <vespa/document/select/variablemap.h>
 #include <vespa/vespalib/util/exceptions.h>
 
 namespace document {
@@ -52,46 +53,42 @@ CPPUNIT_TEST_SUITE_REGISTRATION(DocumentCalculatorTest);
 
 void
 DocumentCalculatorTest::testConstant() {
-    DocumentCalculator::VariableMap variables;
+    auto variables = std::make_unique<select::VariableMap>();
     DocumentCalculator calc(getRepo(), "4.0");
 
-    Document doc(*_testRepo.getDocumentType("testdoctype1"),
-                 DocumentId("doc:test:foo"));
+    Document doc(*_testRepo.getDocumentType("testdoctype1"), DocumentId("doc:test:foo"));
     CPPUNIT_ASSERT_EQUAL(4.0, calc.evaluate(doc, std::move(variables)));
 }
 
 void
 DocumentCalculatorTest::testSimple() {
-    DocumentCalculator::VariableMap variables;
+    auto variables = std::make_unique<select::VariableMap>();
     DocumentCalculator calc(getRepo(), "(3 + 5) / 2");
 
-    Document doc(*_testRepo.getDocumentType("testdoctype1"),
-                 DocumentId("doc:test:foo"));
+    Document doc(*_testRepo.getDocumentType("testdoctype1"), DocumentId("doc:test:foo"));
     CPPUNIT_ASSERT_EQUAL(4.0, calc.evaluate(doc, std::move(variables)));
 }
 
 void
 DocumentCalculatorTest::testVariables() {
-    DocumentCalculator::VariableMap variables;
-    variables["x"] = 3.0;
-    variables["y"] = 5.0;
+    auto variables = std::make_unique<select::VariableMap>();
+    (*variables)["x"] = 3.0;
+    (*variables)["y"] = 5.0;
     DocumentCalculator calc(getRepo(), "($x + $y) / 2");
 
-    Document doc(*_testRepo.getDocumentType("testdoctype1"),
-                 DocumentId("doc:test:foo"));
+    Document doc(*_testRepo.getDocumentType("testdoctype1"), DocumentId("doc:test:foo"));
     CPPUNIT_ASSERT_EQUAL(4.0, calc.evaluate(doc, std::move(variables)));
 }
 
 void
 DocumentCalculatorTest::testFields() {
-    DocumentCalculator::VariableMap variables;
-    variables["x"] = 3.0;
-    variables["y"] = 5.0;
+    auto variables = std::make_unique<select::VariableMap>();
+    (*variables)["x"] = 3.0;
+    (*variables)["y"] = 5.0;
     DocumentCalculator calc(getRepo(), "(testdoctype1.headerval + testdoctype1"
                             ".hfloatval) / testdoctype1.headerlongval");
 
-    Document doc(*_testRepo.getDocumentType("testdoctype1"),
-                 DocumentId("doc:test:foo"));
+    Document doc(*_testRepo.getDocumentType("testdoctype1"), DocumentId("doc:test:foo"));
     doc.setValue(doc.getField("headerval"), IntFieldValue(5));
     doc.setValue(doc.getField("hfloatval"), FloatFieldValue(3.0));
     doc.setValue(doc.getField("headerlongval"), LongFieldValue(2));
@@ -100,14 +97,13 @@ DocumentCalculatorTest::testFields() {
 
 void
 DocumentCalculatorTest::testFieldsDivZero() {
-    DocumentCalculator::VariableMap variables;
-    variables["x"] = 3.0;
-    variables["y"] = 5.0;
+    auto variables = std::make_unique<select::VariableMap>();
+    (*variables)["x"] = 3.0;
+    (*variables)["y"] = 5.0;
     DocumentCalculator calc(getRepo(), "(testdoctype1.headerval + testdoctype1"
                             ".hfloatval) / testdoctype1.headerlongval");
 
-    Document doc(*_testRepo.getDocumentType("testdoctype1"),
-                 DocumentId("doc:test:foo"));
+    Document doc(*_testRepo.getDocumentType("testdoctype1"), DocumentId("doc:test:foo"));
     doc.setValue(doc.getField("headerval"), IntFieldValue(5));
     doc.setValue(doc.getField("hfloatval"), FloatFieldValue(3.0));
     doc.setValue(doc.getField("headerlongval"), LongFieldValue(0));
@@ -121,11 +117,10 @@ DocumentCalculatorTest::testFieldsDivZero() {
 
 void
 DocumentCalculatorTest::testDivideByZero() {
-    DocumentCalculator::VariableMap variables;
+    auto variables = std::make_unique<select::VariableMap>();
     DocumentCalculator calc(getRepo(), "(3 + 5) / 0");
 
-    Document doc(*_testRepo.getDocumentType("testdoctype1"),
-                 DocumentId("doc:test:foo"));
+    Document doc(*_testRepo.getDocumentType("testdoctype1"), DocumentId("doc:test:foo"));
     try {
         calc.evaluate(doc, std::move(variables));
         CPPUNIT_ASSERT(false);
@@ -136,11 +131,10 @@ DocumentCalculatorTest::testDivideByZero() {
 
 void
 DocumentCalculatorTest::testModByZero() {
-    DocumentCalculator::VariableMap variables;
+    auto variables = std::make_unique<select::VariableMap>();
     DocumentCalculator calc(getRepo(), "(3 + 5) % 0");
 
-    Document doc(*_testRepo.getDocumentType("testdoctype1"),
-                 DocumentId("doc:test:foo"));
+    Document doc(*_testRepo.getDocumentType("testdoctype1"), DocumentId("doc:test:foo"));
     try {
         calc.evaluate(doc, std::move(variables));
         CPPUNIT_ASSERT(false);
@@ -151,12 +145,11 @@ DocumentCalculatorTest::testModByZero() {
 
 void
 DocumentCalculatorTest::testFieldNotSet() {
-    DocumentCalculator::VariableMap variables;
+    auto variables = std::make_unique<select::VariableMap>();
     DocumentCalculator calc(getRepo(), "(testdoctype1.headerval + testdoctype1"
                             ".hfloatval) / testdoctype1.headerlongval");
 
-    Document doc(*_testRepo.getDocumentType("testdoctype1"),
-                 DocumentId("doc:test:foo"));
+    Document doc(*_testRepo.getDocumentType("testdoctype1"), DocumentId("doc:test:foo"));
     doc.setValue(doc.getField("hfloatval"), FloatFieldValue(3.0));
     doc.setValue(doc.getField("headerlongval"), LongFieldValue(2));
     try {
@@ -169,13 +162,12 @@ DocumentCalculatorTest::testFieldNotSet() {
 
 void
 DocumentCalculatorTest::testFieldNotFound() {
-    DocumentCalculator::VariableMap variables;
+    auto variables = std::make_unique<select::VariableMap>();
     DocumentCalculator calc(getRepo(),
                             "(testdoctype1.mynotfoundfield + testdoctype1"
                             ".hfloatval) / testdoctype1.headerlongval");
 
-    Document doc(*_testRepo.getDocumentType("testdoctype1"),
-                 DocumentId("doc:test:foo"));
+    Document doc(*_testRepo.getDocumentType("testdoctype1"), DocumentId("doc:test:foo"));
     doc.setValue(doc.getField("hfloatval"), FloatFieldValue(3.0));
     doc.setValue(doc.getField("headerlongval"), LongFieldValue(2));
     try {
@@ -188,11 +180,10 @@ DocumentCalculatorTest::testFieldNotFound() {
 
 void
 DocumentCalculatorTest::testByteSubtractionZeroResult() {
-    DocumentCalculator::VariableMap variables;
+    auto variables = std::make_unique<select::VariableMap>();
     DocumentCalculator calc(getRepo(), "testdoctype1.byteval - 3");
 
-    Document doc(*_testRepo.getDocumentType("testdoctype1"),
-                 DocumentId("doc:test:foo"));
+    Document doc(*_testRepo.getDocumentType("testdoctype1"), DocumentId("doc:test:foo"));
     doc.setValue(doc.getField("byteval"), ByteFieldValue(3));
     CPPUNIT_ASSERT_EQUAL(0.0, calc.evaluate(doc, std::move(variables)));
 }
diff --git a/document/src/tests/documenttestcase.cpp b/document/src/tests/documenttestcase.cpp
index b100487a94e..3bbd5178055 100644
--- a/document/src/tests/documenttestcase.cpp
+++ b/document/src/tests/documenttestcase.cpp
@@ -11,6 +11,7 @@
 #include <vespa/vespalib/objects/nbostream.h>
 #include <vespa/vespalib/testkit/test_kit.h>
 #include <vespa/document/util/serializableexceptions.h>
+#include <vespa/document/util/bytebuffer.h>
 
 using vespalib::nbostream;
 
@@ -75,7 +76,7 @@ void DocumentTest::testSizeOf()
     CPPUNIT_ASSERT_EQUAL(120ul, sizeof(Document));
     CPPUNIT_ASSERT_EQUAL(64ul, sizeof(StructFieldValue));
     CPPUNIT_ASSERT_EQUAL(16ul, sizeof(StructuredFieldValue));
-    CPPUNIT_ASSERT_EQUAL(120ul, sizeof(SerializableArray));
+    CPPUNIT_ASSERT_EQUAL(64ul, sizeof(SerializableArray));
 }
 
 void DocumentTest::testFieldPath()
diff --git a/document/src/tests/primitivefieldvaluetest.cpp b/document/src/tests/primitivefieldvaluetest.cpp
index 15fd6e00959..018408a2d02 100644
--- a/document/src/tests/primitivefieldvaluetest.cpp
+++ b/document/src/tests/primitivefieldvaluetest.cpp
@@ -4,6 +4,7 @@
 #include <vespa/document/serialization/vespadocumentdeserializer.h>
 #include <vespa/vdstestlib/cppunit/macros.h>
 #include <vespa/vespalib/objects/nbostream.h>
+#include <vespa/document/util/bytebuffer.h>
 
 using vespalib::nbostream;
 
diff --git a/document/src/tests/structfieldvaluetest.cpp b/document/src/tests/structfieldvaluetest.cpp
index d6e585e0987..1b2fa17a79a 100644
--- a/document/src/tests/structfieldvaluetest.cpp
+++ b/document/src/tests/structfieldvaluetest.cpp
@@ -1,10 +1,12 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include <vespa/document/fieldvalue/fieldvalues.h>
+#include <vespa/document/datatype/documenttype.h>
 #include <vespa/document/repo/configbuilder.h>
 #include <vespa/document/serialization/vespadocumentdeserializer.h>
 #include <vespa/vdstestlib/cppunit/macros.h>
 #include <vespa/vespalib/objects/nbostream.h>
+#include <vespa/document/util/bytebuffer.h>
 
 using vespalib::nbostream;
 using document::config_builder::Struct;
diff --git a/document/src/tests/testxml.cpp b/document/src/tests/testxml.cpp
index da328c7a63f..a9abae531fa 100644
--- a/document/src/tests/testxml.cpp
+++ b/document/src/tests/testxml.cpp
@@ -1,6 +1,5 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
-/* $Id$*/
 
 #include <vespa/document/base/testdocrepo.h>
 #include <cppunit/extensions/HelperMacros.h>
@@ -8,6 +7,7 @@
 #include <vespa/document/update/addvalueupdate.h>
 #include <vespa/document/update/assignvalueupdate.h>
 #include <vespa/document/update/removevalueupdate.h>
+#include <vespa/document/datatype/documenttype.h>
 #include <vespa/document/fieldvalue/fieldvalues.h>
 #include <vespa/vespalib/text/stringtokenizer.h>
 #include <vespa/vespalib/testkit/test_kit.h>
diff --git a/document/src/tests/weightedsetfieldvaluetest.cpp b/document/src/tests/weightedsetfieldvaluetest.cpp
index 81b98da297f..68b94cd2f16 100644
--- a/document/src/tests/weightedsetfieldvaluetest.cpp
+++ b/document/src/tests/weightedsetfieldvaluetest.cpp
@@ -4,6 +4,7 @@
 #include <vespa/document/serialization/vespadocumentdeserializer.h>
 #include <vespa/vdstestlib/cppunit/macros.h>
 #include <vespa/vespalib/objects/nbostream.h>
+#include <vespa/document/util/bytebuffer.h>
 
 using vespalib::nbostream;
 
diff --git a/document/src/vespa/document/annotation/spantree.cpp b/document/src/vespa/document/annotation/spantree.cpp
index 199040ea8f6..61c7ac48817 100644
--- a/document/src/vespa/document/annotation/spantree.cpp
+++ b/document/src/vespa/document/annotation/spantree.cpp
@@ -1,9 +1,7 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "spantree.h"
-#include "annotation.h"
 #include "spannode.h"
-#include <vespa/vespalib/stllike/string.h>
 #include <vespa/vespalib/stllike/asciistream.h>
 
 using std::unique_ptr;
diff --git a/document/src/vespa/document/annotation/spantree.h b/document/src/vespa/document/annotation/spantree.h
index d53e9002598..06558cf6887 100644
--- a/document/src/vespa/document/annotation/spantree.h
+++ b/document/src/vespa/document/annotation/spantree.h
@@ -4,6 +4,7 @@
 
 #include <vespa/document/annotation/annotation.h>
 #include <vector>
+#include <cassert>
 
 namespace document {
 class SpanNode;
diff --git a/document/src/vespa/document/base/documentcalculator.cpp b/document/src/vespa/document/base/documentcalculator.cpp
index 7a83d80764c..24afdee3bd1 100644
--- a/document/src/vespa/document/base/documentcalculator.cpp
+++ b/document/src/vespa/document/base/documentcalculator.cpp
@@ -5,6 +5,7 @@
 #include <vespa/document/select/compare.h>
 #include <vespa/document/select/parser.h>
 #include <vespa/document/select/valuenode.h>
+#include <vespa/document/select/variablemap.h>
 #include <vespa/vespalib/util/exceptions.h>
 
 namespace document {
@@ -21,13 +22,13 @@ DocumentCalculator::DocumentCalculator(
 DocumentCalculator::~DocumentCalculator() { }
 
 double
-DocumentCalculator::evaluate(const Document& doc, VariableMap && variables)
+DocumentCalculator::evaluate(const Document& doc, std::unique_ptr<select::VariableMap> variables)
 {
     select::Compare& compare(static_cast<select::Compare&>(*_selectionNode));
     const select::ValueNode& left = compare.getLeft();
 
     select::Context context(doc);
-    context._variables = std::move(variables);
+    context.setVariableMap(std::move(variables));
     std::unique_ptr<select::Value> value = left.getValue(context);
 
     select::NumberValue* num = dynamic_cast<select::NumberValue*>(value.get());
diff --git a/document/src/vespa/document/base/documentcalculator.h b/document/src/vespa/document/base/documentcalculator.h
index 7e1feaf5688..fc9eb73dea2 100644
--- a/document/src/vespa/document/base/documentcalculator.h
+++ b/document/src/vespa/document/base/documentcalculator.h
@@ -2,18 +2,18 @@
 #pragma once
 
 #include <vespa/document/select/node.h>
-#include <vespa/vespalib/stllike/hash_map.h>
 
 namespace document {
+
+namespace select { class VariableMap; }
+
 class DocumentTypeRepo;
 
 class DocumentCalculator {
 public:
-    using VariableMap = vespalib::hash_map<vespalib::string, double>;
-
     DocumentCalculator(const DocumentTypeRepo& repo, const vespalib::string& expression);
     ~DocumentCalculator();
-    double evaluate(const Document& doc, VariableMap && variables);
+    double evaluate(const Document& doc, std::unique_ptr<select::VariableMap> variables);
 
 private:
     std::unique_ptr<select::Node> _selectionNode;
diff --git a/document/src/vespa/document/base/exceptions.cpp b/document/src/vespa/document/base/exceptions.cpp
index 3e5c464b8b4..dbd3f2bd998 100644
--- a/document/src/vespa/document/base/exceptions.cpp
+++ b/document/src/vespa/document/base/exceptions.cpp
@@ -3,6 +3,7 @@
 #include "exceptions.h"
 #include <vespa/document/datatype/datatype.h>
 #include <vespa/document/fieldvalue/document.h>
+#include <vespa/vespalib/util/stringfmt.h>
 
 namespace document {
 
diff --git a/document/src/vespa/document/base/field.cpp b/document/src/vespa/document/base/field.cpp
index d62623b64b9..d00fd42081c 100644
--- a/document/src/vespa/document/base/field.cpp
+++ b/document/src/vespa/document/base/field.cpp
@@ -4,6 +4,7 @@
 #include <vespa/document/fieldvalue/fieldvalue.h>
 #include <vespa/vespalib/util/exceptions.h>
 #include <vespa/vespalib/stllike/asciistream.h>
+#include <vespa/vespalib/util/stringfmt.h>
 #include <vespa/vespalib/util/bobhash.h>
 
 namespace document {
diff --git a/document/src/vespa/document/base/forcelink.cpp b/document/src/vespa/document/base/forcelink.cpp
index b03433081d6..f17c8f582b3 100644
--- a/document/src/vespa/document/base/forcelink.cpp
+++ b/document/src/vespa/document/base/forcelink.cpp
@@ -3,6 +3,8 @@
 #include "forcelink.h"
 #include <vespa/document/update/updates.h>
 #include <vespa/document/fieldvalue/fieldvalues.h>
+#include <vespa/document/datatype/documenttype.h>
+
 
 namespace document {
 
diff --git a/document/src/vespa/document/base/globalid.cpp b/document/src/vespa/document/base/globalid.cpp
index 7fa22d27971..ab8736999aa 100644
--- a/document/src/vespa/document/base/globalid.cpp
+++ b/document/src/vespa/document/base/globalid.cpp
@@ -3,6 +3,7 @@
 #include <vespa/document/base/globalid.h>
 #include <vespa/vespalib/util/exceptions.h>
 #include <vespa/vespalib/stllike/asciistream.h>
+#include <vespa/vespalib/util/stringfmt.h>
 #include <vespa/vespalib/stllike/hash_set.hpp>
 #include <cassert>
 
diff --git a/document/src/vespa/document/base/idstring.cpp b/document/src/vespa/document/base/idstring.cpp
index f5f9f7377b5..d837f512637 100644
--- a/document/src/vespa/document/base/idstring.cpp
+++ b/document/src/vespa/document/base/idstring.cpp
@@ -3,6 +3,7 @@
 #include "idstring.h"
 #include <vespa/document/bucket/bucketid.h>
 #include <vespa/vespalib/util/md5.h>
+#include <vespa/vespalib/util/stringfmt.h>
 
 using vespalib::string;
 using vespalib::stringref;
diff --git a/document/src/vespa/document/bucket/bucketid.cpp b/document/src/vespa/document/bucket/bucketid.cpp
index 8554450fb22..c9efe7cd152 100644
--- a/document/src/vespa/document/bucket/bucketid.cpp
+++ b/document/src/vespa/document/bucket/bucketid.cpp
@@ -6,6 +6,7 @@
 #include <vespa/vespalib/objects/nbostream.h>
 #include <vespa/vespalib/stllike/asciistream.h>
 #include <vespa/vespalib/stllike/hash_set.hpp>
+#include <vespa/vespalib/util/stringfmt.h>
 
 using vespalib::nbostream;
 using vespalib::asciistream;
diff --git a/document/src/vespa/document/bucket/bucketselector.cpp b/document/src/vespa/document/bucket/bucketselector.cpp
index 8b5e7ada91b..b41b1c284a6 100644
--- a/document/src/vespa/document/bucket/bucketselector.cpp
+++ b/document/src/vespa/document/bucket/bucketselector.cpp
@@ -8,6 +8,7 @@
 #include <vespa/document/select/visitor.h>
 #include <vespa/document/select/branch.h>
 #include <vespa/document/select/compare.h>
+#include <vespa/vespalib/util/stringfmt.h>
 
 namespace document {
 
diff --git a/document/src/vespa/document/datatype/annotationreferencedatatype.cpp b/document/src/vespa/document/datatype/annotationreferencedatatype.cpp
index 2591cdbf37c..344fc31ed33 100644
--- a/document/src/vespa/document/datatype/annotationreferencedatatype.cpp
+++ b/document/src/vespa/document/datatype/annotationreferencedatatype.cpp
@@ -3,6 +3,7 @@
 #include "annotationreferencedatatype.h"
 #include <vespa/document/fieldvalue/annotationreferencefieldvalue.h>
 #include <ostream>
+#include <cassert>
 
 using std::unique_ptr;
 using std::ostream;
diff --git a/document/src/vespa/document/datatype/documenttype.cpp b/document/src/vespa/document/datatype/documenttype.cpp
index 883c4df8f9b..dcf28a9fabf 100644
--- a/document/src/vespa/document/datatype/documenttype.cpp
+++ b/document/src/vespa/document/datatype/documenttype.cpp
@@ -3,6 +3,7 @@
 #include <vespa/document/datatype/documenttype.h>
 #include <vespa/document/fieldvalue/document.h>
 #include <vespa/vespalib/util/exceptions.h>
+#include <vespa/vespalib/util/stringfmt.h>
 #include <iomanip>
 #include <vespa/log/log.h>
 
diff --git a/document/src/vespa/document/datatype/primitivedatatype.cpp b/document/src/vespa/document/datatype/primitivedatatype.cpp
index 821a22aad4a..afc32e291ea 100644
--- a/document/src/vespa/document/datatype/primitivedatatype.cpp
+++ b/document/src/vespa/document/datatype/primitivedatatype.cpp
@@ -3,6 +3,7 @@
 #include "primitivedatatype.h"
 #include <vespa/document/fieldvalue/fieldvalues.h>
 #include <vespa/vespalib/util/exceptions.h>
+#include <vespa/vespalib/util/stringfmt.h>
 #include <sstream>
 
 namespace document {
diff --git a/document/src/vespa/document/datatype/structdatatype.cpp b/document/src/vespa/document/datatype/structdatatype.cpp
index 6cf9e18656b..f6bd1fc5fa6 100644
--- a/document/src/vespa/document/datatype/structdatatype.cpp
+++ b/document/src/vespa/document/datatype/structdatatype.cpp
@@ -1,12 +1,13 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "structdatatype.h"
-
-#include <iomanip>
 #include <vespa/document/base/exceptions.h>
 #include <vespa/document/fieldvalue/structfieldvalue.h>
 #include <vespa/document/fieldvalue/document.h>
+#include <vespa/vespalib/util/stringfmt.h>
 #include <vespa/vespalib/stllike/hash_map.hpp>
+#include <iomanip>
+
 #include <vespa/log/log.h>
 LOG_SETUP(".document.datatype.struct");
 
diff --git a/document/src/vespa/document/fieldset/fieldsets.cpp b/document/src/vespa/document/fieldset/fieldsets.cpp
index 4ac09820979..51166adec86 100644
--- a/document/src/vespa/document/fieldset/fieldsets.cpp
+++ b/document/src/vespa/document/fieldset/fieldsets.cpp
@@ -2,6 +2,7 @@
 
 #include "fieldsets.h"
 #include <vespa/document/fieldvalue/document.h>
+#include <vespa/document/datatype/documenttype.h>
 
 namespace document {
 
diff --git a/document/src/vespa/document/fieldvalue/arrayfieldvalue.cpp b/document/src/vespa/document/fieldvalue/arrayfieldvalue.cpp
index 866c82a63c9..8c05c25e651 100644
--- a/document/src/vespa/document/fieldvalue/arrayfieldvalue.cpp
+++ b/document/src/vespa/document/fieldvalue/arrayfieldvalue.cpp
@@ -4,6 +4,7 @@
 #include "stringfieldvalue.h"
 #include "predicatefieldvalue.h"
 #include <vespa/document/util/serializableexceptions.h>
+#include <vespa/vespalib/util/stringfmt.h>
 #include <vespa/log/log.h>
 
 LOG_SETUP(".document.fieldvalue.array");
diff --git a/document/src/vespa/document/fieldvalue/document.cpp b/document/src/vespa/document/fieldvalue/document.cpp
index f8358cfb544..29f487df989 100644
--- a/document/src/vespa/document/fieldvalue/document.cpp
+++ b/document/src/vespa/document/fieldvalue/document.cpp
@@ -1,6 +1,7 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "document.h"
+#include <vespa/document/datatype/documenttype.h>
 #include <vespa/vespalib/util/crc.h>
 #include <vespa/document/repo/fixedtyperepo.h>
 #include <vespa/document/serialization/vespadocumentdeserializer.h>
@@ -8,6 +9,8 @@
 #include <vespa/vespalib/objects/nbostream.h>
 #include <vespa/document/util/serializableexceptions.h>
 #include <vespa/document/base/exceptions.h>
+#include <vespa/document/util/bytebuffer.h>
+
 #include <sstream>
 
 using vespalib::nbostream;
@@ -15,7 +18,6 @@ using vespalib::make_string;
 using vespalib::IllegalArgumentException;
 using vespalib::IllegalStateException;
 
-
 namespace document {
 namespace {
 
@@ -153,6 +155,11 @@ Document::swap(Document & rhs)
     std::swap(_lastModified, rhs._lastModified);
 }
 
+const DocumentType&
+Document::getType() const {
+    return static_cast<const DocumentType &>(StructuredFieldValue::getType());
+}
+
 Document& Document::operator=(const Document& doc)
 {
     StructuredFieldValue::operator=(doc);
diff --git a/document/src/vespa/document/fieldvalue/document.h b/document/src/vespa/document/fieldvalue/document.h
index f159e9be1d5..02b97238cb2 100644
--- a/document/src/vespa/document/fieldvalue/document.h
+++ b/document/src/vespa/document/fieldvalue/document.h
@@ -18,7 +18,6 @@
 #include "structfieldvalue.h"
 #include <vespa/document/base/documentid.h>
 #include <vespa/document/base/field.h>
-#include <vespa/document/datatype/documenttype.h>
 
 namespace document {
 
@@ -71,10 +70,7 @@ public:
     void accept(FieldValueVisitor &visitor) override { visitor.visit(*this); }
     void accept(ConstFieldValueVisitor &visitor) const override { visitor.visit(*this); }
 
-    const DocumentType& getType() const {
-        return static_cast<const DocumentType &>(StructuredFieldValue::getType());
-    }
-
+    const DocumentType& getType() const;
     const DocumentId& getId() const { return _id; }
     DocumentId & getId() { return _id; }
 
diff --git a/document/src/vespa/document/fieldvalue/fieldvalue.cpp b/document/src/vespa/document/fieldvalue/fieldvalue.cpp
index fd3093488ea..3279eb77a64 100644
--- a/document/src/vespa/document/fieldvalue/fieldvalue.cpp
+++ b/document/src/vespa/document/fieldvalue/fieldvalue.cpp
@@ -10,7 +10,7 @@
 #include "doublefieldvalue.h"
 #include "bytefieldvalue.h"
 #include "predicatefieldvalue.h"
-
+#include <vespa/document/util/bytebuffer.h>
 #include <vespa/document/base/exceptions.h>
 #include <vespa/document/serialization/vespadocumentserializer.h>
 #include <vespa/vespalib/objects/nbostream.h>
@@ -51,6 +51,21 @@ FieldValue::hash() const
     return vespalib::hashValue(os.c_str(), os.size()) ;
 }
 
+bool
+FieldValue::isA(const FieldValue& other) const {
+    return (getDataType()->isA(*other.getDataType()));
+}
+int
+FieldValue::compare(const FieldValue& other) const {
+    const DataType & a = *getDataType();
+    const DataType & b = *other.getDataType();
+    return (a < b)
+           ? -1
+           : (b < a)
+             ? 1
+             : 0;
+}
+
 FieldValue&
 FieldValue::assign(const FieldValue& value)
 {
diff --git a/document/src/vespa/document/fieldvalue/fieldvalue.h b/document/src/vespa/document/fieldvalue/fieldvalue.h
index bb2a10a3f53..81e207bc31a 100644
--- a/document/src/vespa/document/fieldvalue/fieldvalue.h
+++ b/document/src/vespa/document/fieldvalue/fieldvalue.h
@@ -15,7 +15,6 @@
 #include <vespa/document/datatype/datatype.h>
 #include <vespa/document/util/xmlserializable.h>
 #include <vespa/vespalib/util/polymorphicarrays.h>
-#include <vespa/document/util/bytebuffer.h>
 #include <vespa/vespalib/objects/cloneable.h>
 #include <map>
 
@@ -25,6 +24,8 @@ namespace vespalib {
 
 namespace document {
 
+class ByteBuffer;
+
 class FieldValue : public vespalib::Identifiable
 {
 protected:
@@ -168,27 +169,18 @@ public:
     virtual const DataType *getDataType() const = 0;
 
     /** Wrapper for datatypes isA() function. See DataType. */
-    virtual bool isA(const FieldValue& other) const
-        { return (getDataType()->isA(*other.getDataType())); }
+    virtual bool isA(const FieldValue& other) const;
 
     void serialize(vespalib::nbostream &stream) const;
     void serialize(ByteBuffer& buffer) const;
-    ByteBuffer::UP serialize() const;
+    std::unique_ptr<ByteBuffer> serialize() const;
 
     /**
      * Compares this fieldvalue with another fieldvalue.
      * Should return 0 if the two are equal, <0 if this object is "less" than
      * the other, and >0 if this object is more than the other.
      */
-    virtual int compare(const FieldValue& other) const {
-        const DataType & a = *getDataType();
-        const DataType & b = *other.getDataType();
-        return (a < b)
-                   ? -1
-                   : (b < a)
-                       ? 1
-                       : 0;
-    }
+    virtual int compare(const FieldValue& other) const;
 
     /**
      * Returns true if this object have been altered since last
diff --git a/document/src/vespa/document/fieldvalue/serializablearray.cpp b/document/src/vespa/document/fieldvalue/serializablearray.cpp
index 7a174f1ab52..c8beea5e79c 100644
--- a/document/src/vespa/document/fieldvalue/serializablearray.cpp
+++ b/document/src/vespa/document/fieldvalue/serializablearray.cpp
@@ -1,16 +1,29 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 #include "serializablearray.h"
 #include <vespa/document/util/serializableexceptions.h>
+#include <vespa/document/util/bytebuffer.h>
 #include <vespa/vespalib/stllike/hash_map.hpp>
+#include <vespa/vespalib/data/databuffer.h>
+#include <vespa/document/util/compressor.h>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".document.serializable-array");
 
-
 using std::vector;
 
 namespace document {
 
+namespace serializablearray {
+
+using BufferMapT = vespalib::hash_map<int, ByteBuffer::UP>;
+
+class BufferMap : public BufferMapT {
+public:
+    using BufferMapT::BufferMapT;
+};
+
+}
+
 SerializableArray::Statistics SerializableArray::_stats;
 
 SerializableArray::SerializableArray()
@@ -19,6 +32,13 @@ SerializableArray::SerializableArray()
 {
 }
 
+serializablearray::BufferMap & ensure(std::unique_ptr<serializablearray::BufferMap> & owned) {
+    if (!owned) {
+        owned = std::make_unique<serializablearray::BufferMap>();
+    }
+    return *owned;
+}
+
 SerializableArray::SerializableArray(const SerializableArray& other)
     : Cloneable(),
       _entries(other._entries),
@@ -34,7 +54,7 @@ SerializableArray::SerializableArray(const SerializableArray& other)
             // Pointing to a buffer in the _owned structure.
             ByteBuffer::UP buf(ByteBuffer::copyBuffer(e.getBuffer(_uncompSerData.get()), e.size()));
             e.setBuffer(buf->getBuffer());
-            _owned[e.id()] = std::move(buf);
+            ensure(_owned)[e.id()] = std::move(buf);
         } else {
             // If not it is relative to the buffer _uncompSerData, and hence it is valid as is.
         }
@@ -79,7 +99,7 @@ SerializableArray::set(int id, ByteBuffer::UP buffer)
 {
     maybeDecompress();
     Entry e(id, buffer->getRemaining(), buffer->getBuffer());
-    _owned[id] = std::move(buffer);
+    ensure(_owned)[id] = std::move(buffer);
     EntryMap::iterator it = find(id);
     if (it == _entries.end()) {
         _entries.push_back(e);
@@ -149,7 +169,9 @@ SerializableArray::clear(int id)
     EntryMap::iterator it = find(id);
     if (it != _entries.end()) {
         _entries.erase(it);
-        _owned.erase(id);
+        if (_owned) {
+            _owned->erase(id);
+        }
         invalidate();
     }
 }
@@ -215,4 +237,14 @@ void SerializableArray::assign(EntryMap & entries,
     }
 }
 
+CompressionInfo
+SerializableArray::getCompressionInfo() const {
+    return CompressionInfo(_uncompressedLength, _compSerData->getRemaining());
+}
+
+const char *
+SerializableArray::Entry::getBuffer(const ByteBuffer * readOnlyBuffer) const {
+    return hasBuffer() ? _data._buffer : readOnlyBuffer->getBuffer() + getOffset();
+}
+
 } // document
diff --git a/document/src/vespa/document/fieldvalue/serializablearray.h b/document/src/vespa/document/fieldvalue/serializablearray.h
index 9c0978f1476..1d211a50ab3 100644
--- a/document/src/vespa/document/fieldvalue/serializablearray.h
+++ b/document/src/vespa/document/fieldvalue/serializablearray.h
@@ -16,20 +16,22 @@
 
 #pragma once
 
-#include <vespa/document/util/bytebuffer.h>
 #include <vespa/document/util/compressionconfig.h>
-#include <vespa/document/util/compressor.h>
-#include <vespa/document/util/serializable.h>
-#include <vector>
 #include <vespa/vespalib/objects/cloneable.h>
-#include <vespa/vespalib/stllike/hash_map.h>
 #include <vespa/vespalib/util/buffer.h>
-#include <vespa/fastos/dynamiclibrary.h>
+#include <vespa/vespalib/util/memory.h>
+#include <vector>
 
-namespace document
-{
+#define VESPA_DLL_LOCAL  __attribute__ ((visibility("hidden")))
+
+namespace document {
 
 class SerializableArrayIterator;
+class ByteBuffer;
+
+namespace serializablearray {
+    class BufferMap;
+}
 
 class SerializableArray : public vespalib::Cloneable
 {
@@ -71,7 +73,7 @@ public:
         bool operator < (const Entry & e) const { return cmp(e) < 0; }
         int cmp(const Entry & e) const { return _id - e._id; }
         void setBuffer(const char * buffer) { _data._buffer = buffer; _sz |= BUFFER_MASK; }
-        const char * getBuffer(const ByteBuffer * readOnlyBuffer) const { return hasBuffer() ? _data._buffer : readOnlyBuffer->getBuffer() + getOffset(); }
+        VESPA_DLL_LOCAL const char * getBuffer(const ByteBuffer * readOnlyBuffer) const;
     private:
         uint32_t getOffset() const { return _data._offset; }
         enum { BUFFER_MASK=0x80000000 };
@@ -100,12 +102,11 @@ public:
 private:
     static Statistics _stats;
 
-    typedef vespalib::hash_map<int, uint32_t> HashMap;
-
 public:
     static Statistics& getStatistics() { return _stats; }
-    typedef vespalib::CloneablePtr<SerializableArray> CP;
-    typedef std::unique_ptr<SerializableArray> UP;
+    using CP = vespalib::CloneablePtr<SerializableArray>;
+    using UP = std::unique_ptr<SerializableArray>;
+    using ByteBufferUP = std::unique_ptr<ByteBuffer>;
 
     SerializableArray();
     virtual ~SerializableArray();
@@ -122,7 +123,7 @@ public:
     void set(int id, const char* value, int len);
 
     /** Stores a value in the array. */
-    void set(int id, std::unique_ptr<ByteBuffer> buffer);
+    void set(int id, ByteBufferUP buffer);
 
     /**
      * Gets a value from the array. This is the faster version of the above.
@@ -152,19 +153,17 @@ public:
     void clear();
 
     CompressionConfig::Type getCompression() const { return _serializedCompression; }
-    CompressionInfo getCompressionInfo() const {
-        return CompressionInfo(_uncompressedLength, _compSerData->getRemaining());
-    }
+    CompressionInfo getCompressionInfo() const;
 
     /**
      * Sets the serialized data that is the basis for this object's
      * content. This is used by deserialization. Any existing entries
      * are cleared.
      */
-    VESPA_DLL_LOCAL void assign(EntryMap &entries,
-                                ByteBuffer::UP buffer,
-                                CompressionConfig::Type comp_type,
-                                uint32_t uncompressed_length);
+    void assign(EntryMap &entries,
+                ByteBufferUP buffer,
+                CompressionConfig::Type comp_type,
+                uint32_t uncompressed_length);
 
     bool empty() const { return _entries.empty(); }
 
@@ -189,22 +188,22 @@ private:
         return false;
     }
 
-    VESPA_DLL_LOCAL bool deCompressAndCatch() const;
+    bool deCompressAndCatch() const;
     void maybeDecompress() const {
         if ( shouldDecompress() ) {
             const_cast<SerializableArray *>(this)->deCompress();
         }
     }
-    VESPA_DLL_LOCAL void deCompress(); // throw (DeserializeException);
+    void deCompress(); // throw (DeserializeException);
 
     /** Contains the stored attributes, with reference to the real data.. */
     EntryMap _entries;
     /** The buffers we own. */
-    vespalib::hash_map<int, ByteBuffer::UP > _owned;
+    std::unique_ptr<serializablearray::BufferMap> _owned;
 
     /** Data we deserialized from, if applicable. */
-    ByteBuffer::UP           _uncompSerData;
-    ByteBuffer::UP           _compSerData;
+    ByteBufferUP             _uncompSerData;
+    ByteBufferUP             _compSerData;
     CompressionConfig::Type  _serializedCompression;
 
     uint32_t     _uncompressedLength;
@@ -215,4 +214,3 @@ private:
 };
 
 } // document
-
diff --git a/document/src/vespa/document/fieldvalue/structfieldvalue.cpp b/document/src/vespa/document/fieldvalue/structfieldvalue.cpp
index 079b730c8cc..74805ce3f10 100644
--- a/document/src/vespa/document/fieldvalue/structfieldvalue.cpp
+++ b/document/src/vespa/document/fieldvalue/structfieldvalue.cpp
@@ -8,9 +8,9 @@
 #include <vespa/vespalib/objects/nbostream.h>
 #include <vespa/vespalib/util/crc.h>
 #include <vespa/document/datatype/positiondatatype.h>
-#include <vespa/vespalib/util/vstringfmt.h>
 #include <vespa/document/util/serializableexceptions.h>
 #include <vespa/document/base/exceptions.h>
+#include <vespa/document/util/bytebuffer.h>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".document.structfieldvalue");
@@ -43,6 +43,26 @@ StructFieldValue::~StructFieldValue() { }
 StructFieldValue::Chunks::~Chunks() { }
 
 void
+StructFieldValue::Chunks::push_back(SerializableArray::UP item) {
+    assert(_sz < 2);
+    _chunks[_sz++].reset(item.release());
+}
+
+void
+StructFieldValue::Chunks::clear() {
+    _chunks[0].reset();
+    _chunks[1].reset();
+    _sz = 0;
+}
+
+void
+StructFieldValue::Chunks::swap(Chunks & rhs) {
+    _chunks[0].swap(rhs._chunks[0]);
+    _chunks[1].swap(rhs._chunks[1]);
+    std::swap(_sz, rhs._sz);
+}
+
+void
 StructFieldValue::swap(StructFieldValue & rhs)
 {
     StructuredFieldValue::swap(rhs);
@@ -53,6 +73,16 @@ StructFieldValue::swap(StructFieldValue & rhs)
     std::swap(_version, _version);
 }
 
+const StructDataType &
+StructFieldValue::getStructType() const {
+    return static_cast<const StructDataType &>(getType());
+}
+
+const CompressionConfig &
+StructFieldValue::getCompressionConfig() const {
+    return getStructType().getCompressionConfig();
+}
+
 void
 StructFieldValue::lazyDeserialize(const FixedTypeRepo &repo,
                                   uint16_t version,
diff --git a/document/src/vespa/document/fieldvalue/structfieldvalue.h b/document/src/vespa/document/fieldvalue/structfieldvalue.h
index a46a36c477d..01a0e732e43 100644
--- a/document/src/vespa/document/fieldvalue/structfieldvalue.h
+++ b/document/src/vespa/document/fieldvalue/structfieldvalue.h
@@ -11,17 +11,16 @@
 
 #include "structuredfieldvalue.h"
 #include "serializablearray.h"
-#include <vespa/document/util/compressionconfig.h>
-#include <vespa/document/datatype/structdatatype.h>
-#include <vector>
 
 namespace document {
+
 class Document;
 class DocumentType;
 class DocumentTypeRepo;
 class FieldValueWriter;
 class FixedTypeRepo;
 class FieldSet;
+class StructDataType;
 
 class StructFieldValue : public StructuredFieldValue
 {
@@ -32,24 +31,13 @@ public:
         ~Chunks();
         SerializableArray & operator [] (size_t i) { return *_chunks[i]; }
         const SerializableArray & operator [] (size_t i) const { return *_chunks[i]; }
-        void push_back(SerializableArray::UP item) {
-            assert(_sz < 2);
-            _chunks[_sz++].reset(item.release());
-        }
+        VESPA_DLL_LOCAL void push_back(SerializableArray::UP item);
         SerializableArray & back() { return *_chunks[_sz-1]; }
         const SerializableArray & back() const { return *_chunks[_sz-1]; }
         size_t size() const { return _sz; }
         bool empty() const { return _sz == 0; }
-        void clear() {
-            _chunks[0].reset();
-            _chunks[1].reset();
-            _sz = 0;
-        }
-        void swap(Chunks & rhs) {
-            _chunks[0].swap(rhs._chunks[0]);
-            _chunks[1].swap(rhs._chunks[1]);
-            std::swap(_sz, rhs._sz);
-        }
+        VESPA_DLL_LOCAL void clear();
+        VESPA_DLL_LOCAL void swap(Chunks & rhs);
     private:
         SerializableArray::CP _chunks[2];
         size_t _sz;
@@ -73,12 +61,12 @@ public:
     const DocumentTypeRepo * getRepo() const { return _repo; }
     void setDocumentType(const DocumentType & docType) { _doc_type = & docType; }
 
-    const StructDataType & getStructType() const { return static_cast<const StructDataType &>(getType()); }
+    const StructDataType & getStructType() const;
 
     void lazyDeserialize(const FixedTypeRepo &repo,
                          uint16_t version,
                          SerializableArray::EntryMap && fields,
-                         ByteBuffer::UP buffer,
+                         std::unique_ptr<ByteBuffer> buffer,
                          CompressionConfig::Type comp_type,
                          int32_t uncompressed_length);
 
@@ -99,8 +87,7 @@ public:
     const Field& getField(const vespalib::stringref & name) const override;
     void clear() override;
 
-    const CompressionConfig &getCompressionConfig() const
-    { return getStructType().getCompressionConfig(); }
+    const CompressionConfig &getCompressionConfig() const;
 
     // FieldValue implementation.
     FieldValue& assign(const FieldValue&) override;
@@ -146,4 +133,3 @@ private:
 };
 
 } // document
-
diff --git a/document/src/vespa/document/fieldvalue/tensorfieldvalue.cpp b/document/src/vespa/document/fieldvalue/tensorfieldvalue.cpp
index d6396c43a46..03b7d50c1c1 100644
--- a/document/src/vespa/document/fieldvalue/tensorfieldvalue.cpp
+++ b/document/src/vespa/document/fieldvalue/tensorfieldvalue.cpp
@@ -3,6 +3,7 @@
 #include "tensorfieldvalue.h"
 #include <vespa/eval/tensor/tensor.h>
 #include <ostream>
+#include <cassert>
 
 using vespalib::tensor::Tensor;
 
@@ -164,7 +165,6 @@ TensorFieldValue::compare(const FieldValue &other) const
     return ((_tensor.get()  < rhs._tensor.get()) ? -1 : 1);
 }
 
-
 IMPLEMENT_IDENTIFIABLE(TensorFieldValue, FieldValue);
 
 } // document
diff --git a/document/src/vespa/document/repo/documenttyperepo.cpp b/document/src/vespa/document/repo/documenttyperepo.cpp
index 78d668a4886..93adce76efd 100644
--- a/document/src/vespa/document/repo/documenttyperepo.cpp
+++ b/document/src/vespa/document/repo/documenttyperepo.cpp
@@ -3,7 +3,6 @@
 #include "documenttyperepo.h"
 
 #include <vespa/document/datatype/annotationreferencedatatype.h>
-#include <vespa/document/datatype/annotationtype.h>
 #include <vespa/document/datatype/arraydatatype.h>
 #include <vespa/document/datatype/documenttype.h>
 #include <vespa/document/datatype/mapdatatype.h>
@@ -11,15 +10,10 @@
 #include <vespa/document/datatype/urldatatype.h>
 #include <vespa/document/datatype/weightedsetdatatype.h>
 #include <vespa/document/datatype/referencedatatype.h>
-#include <vespa/vespalib/objects/identifiable.h>
 #include <vespa/vespalib/stllike/hash_map.hpp>
-#include <vespa/vespalib/util/closure.h>
 #include <vespa/vespalib/util/exceptions.h>
-#include <vespa/vespalib/util/stringfmt.h>
 #include <vespa/document/config/config-documenttypes.h>
 #include <fstream>
-#include <memory>
-#include <utility>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".documenttyperepo");
@@ -39,6 +33,20 @@ using vespalib::stringref;
 
 namespace document {
 
+namespace internal {
+
+using DocumentTypeMapT = vespalib::hash_map<int32_t, DataTypeRepo *>;
+
+class DocumentTypeMap : public DocumentTypeMapT
+{
+public:
+    using DocumentTypeMapT::DocumentTypeMapT;
+};
+
+}
+
+using DocumentTypeMap = internal::DocumentTypeMap;
+
 namespace {
 template <typename Container>
 void DeleteContent(Container &c) {
@@ -359,7 +367,6 @@ void addDataTypes(const vector<Datatype> &types, Repo &repo,
     }
 }
 
-typedef hash_map<int32_t, DataTypeRepo *> DocumentTypeMap;
 void addDocumentTypes(const DocumentTypeMap &type_map, Repo &repo) {
     for (DocumentTypeMap::const_iterator
              it = type_map.begin(); it != type_map.end(); ++it) {
@@ -515,49 +522,55 @@ void configureAllRepos(const DocumenttypesConfig::DocumenttypeVector &t,
 
 }  // namespace
 
-DocumentTypeRepo::DocumentTypeRepo() {
-    addDefaultDocument(_doc_types);
+DocumentTypeRepo::DocumentTypeRepo() :
+    _doc_types(std::make_unique<internal::DocumentTypeMap>())
+{
+    addDefaultDocument(*_doc_types);
 }
 
-DocumentTypeRepo::DocumentTypeRepo(const DocumentType & type) {
-    addDefaultDocument(_doc_types);
+DocumentTypeRepo::DocumentTypeRepo(const DocumentType & type) :
+    _doc_types(std::make_unique<internal::DocumentTypeMap>())
+{
+    addDefaultDocument(*_doc_types);
     try {
-        addDataTypeRepo(makeDataTypeRepo(type, _doc_types), _doc_types);
+        addDataTypeRepo(makeDataTypeRepo(type, *_doc_types), *_doc_types);
     } catch (...) {
-        DeleteMapContent(_doc_types);
+        DeleteMapContent(*_doc_types);
         throw;
     }
 }
 
-DocumentTypeRepo::DocumentTypeRepo(const DocumenttypesConfig &config) {
-    addDefaultDocument(_doc_types);
+DocumentTypeRepo::DocumentTypeRepo(const DocumenttypesConfig &config) :
+    _doc_types(std::make_unique<internal::DocumentTypeMap>())
+{
+    addDefaultDocument(*_doc_types);
     try {
-        createAllDocumentTypes(config.documenttype, _doc_types);
-        addAllDocumentTypesToRepos(_doc_types);
-        configureAllRepos(config.documenttype, _doc_types);
+        createAllDocumentTypes(config.documenttype, *_doc_types);
+        addAllDocumentTypesToRepos(*_doc_types);
+        configureAllRepos(config.documenttype, *_doc_types);
     } catch (...) {
-        DeleteMapContent(_doc_types);
+        DeleteMapContent(*_doc_types);
         throw;
     }
 }
 
 DocumentTypeRepo::~DocumentTypeRepo() {
-    DeleteMapContent(_doc_types);
+    DeleteMapContent(*_doc_types);
 }
 
 const DocumentType *DocumentTypeRepo::getDocumentType(int32_t type_id) const {
-    const DataTypeRepo *repo = FindPtr(_doc_types, type_id);
+    const DataTypeRepo *repo = FindPtr(*_doc_types, type_id);
     return repo ? repo->doc_type : nullptr;
 }
 
 const DocumentType *DocumentTypeRepo::getDocumentType(const stringref &name) const {
     DocumentTypeMap::const_iterator it =
-        _doc_types.find(DocumentType::createId(name));
+        _doc_types->find(DocumentType::createId(name));
 
-    if (it != _doc_types.end() && it->second->doc_type->getName() == name) {
+    if (it != _doc_types->end() && it->second->doc_type->getName() == name) {
         return it->second->doc_type;
     }
-    for (it = _doc_types.begin(); it != _doc_types.end(); ++it) {
+    for (it = _doc_types->begin(); it != _doc_types->end(); ++it) {
         if (it->second->doc_type->getName() == name) {
             return it->second->doc_type;
         }
@@ -567,27 +580,27 @@ const DocumentType *DocumentTypeRepo::getDocumentType(const stringref &name) con
 
 const DataType *
 DocumentTypeRepo::getDataType(const DocumentType &doc_type, int32_t id) const {
-    const DataTypeRepo *dt_repo = FindPtr(_doc_types, doc_type.getId());
+    const DataTypeRepo *dt_repo = FindPtr(*_doc_types, doc_type.getId());
     return dt_repo ? dt_repo->repo.lookup(id) : nullptr;
 }
 
 const DataType *
 DocumentTypeRepo::getDataType(
         const DocumentType &doc_type, const stringref &name) const {
-    const DataTypeRepo *dt_repo = FindPtr(_doc_types, doc_type.getId());
+    const DataTypeRepo *dt_repo = FindPtr(*_doc_types, doc_type.getId());
     return dt_repo ? dt_repo->repo.lookup(name) : nullptr;
 }
 
 const AnnotationType *DocumentTypeRepo::getAnnotationType(
         const DocumentType &doc_type, int32_t id) const {
-    const DataTypeRepo *dt_repo = FindPtr(_doc_types, doc_type.getId());
+    const DataTypeRepo *dt_repo = FindPtr(*_doc_types, doc_type.getId());
     return dt_repo ? dt_repo->annotations.lookup(id) : nullptr;
 }
 
 void DocumentTypeRepo::forEachDocumentType(
         Closure1<const DocumentType &> &c) const {
     for (DocumentTypeMap::const_iterator
-             it = _doc_types.begin(); it != _doc_types.end(); ++it) {
+             it = _doc_types->begin(); it != _doc_types->end(); ++it) {
         c.call(*it->second->doc_type);
     }
 }
diff --git a/document/src/vespa/document/repo/documenttyperepo.h b/document/src/vespa/document/repo/documenttyperepo.h
index 55655fe169e..4955f3562db 100644
--- a/document/src/vespa/document/repo/documenttyperepo.h
+++ b/document/src/vespa/document/repo/documenttyperepo.h
@@ -3,7 +3,6 @@
 #pragma once
 
 #include <memory>
-#include <vespa/vespalib/stllike/hash_map.h>
 #include <vespa/vespalib/stllike/string.h>
 #include <vespa/vespalib/util/closure.h>
 
@@ -11,6 +10,7 @@ namespace document {
 
 namespace internal {
     class InternalDocumenttypesType;
+    class DocumentTypeMap;
 }
 
 class AnnotationType;
@@ -19,9 +19,7 @@ class DataTypeRepo;
 class DocumentType;
 
 class DocumentTypeRepo {
-    typedef vespalib::hash_map<int32_t, DataTypeRepo *> DocumentTypeMap;
-
-    DocumentTypeMap _doc_types;
+    std::unique_ptr<internal::DocumentTypeMap> _doc_types;
 
 public:
     using DocumenttypesConfig = const internal::InternalDocumenttypesType;
diff --git a/document/src/vespa/document/select/branch.cpp b/document/src/vespa/document/select/branch.cpp
index ddbad69b14d..ea0d0ef5646 100644
--- a/document/src/vespa/document/select/branch.cpp
+++ b/document/src/vespa/document/select/branch.cpp
@@ -2,6 +2,8 @@
 
 #include "branch.h"
 #include "visitor.h"
+#include <cassert>
+#include <ostream>
 
 namespace document {
 namespace select {
diff --git a/document/src/vespa/document/select/constant.cpp b/document/src/vespa/document/select/constant.cpp
index 686ab05055c..0e2fdc2ecec 100644
--- a/document/src/vespa/document/select/constant.cpp
+++ b/document/src/vespa/document/select/constant.cpp
@@ -2,9 +2,10 @@
 
 #include "constant.h"
 #include "visitor.h"
+#include <cassert>
+#include <ostream>
 
-namespace document {
-namespace select {
+namespace document::select {
 
 Constant::Constant(const vespalib::stringref & value)
     : Node(value),
@@ -46,13 +47,11 @@ Constant::visit(Visitor &v) const
 
 
 void
-Constant::print(std::ostream& out, bool,
-                const std::string&) const
+Constant::print(std::ostream& out, bool, const std::string&) const
 {
     if (_parentheses) out << '(';
     out << _name;
     if (_parentheses) out << ')';
 }
 
-} // select
-} // document
+}
diff --git a/document/src/vespa/document/select/context.cpp b/document/src/vespa/document/select/context.cpp
index 44d005e9db2..ef29770ddc8 100644
--- a/document/src/vespa/document/select/context.cpp
+++ b/document/src/vespa/document/select/context.cpp
@@ -1,11 +1,12 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "context.h"
+#include "variablemap.h"
+#include <vespa/document/select/value.h>
 
-namespace document {
-namespace select {
+namespace document::select {
 
-Context::Context(void)
+Context::Context()
     : _doc(NULL),
       _docId(NULL),
       _docUpdate(NULL),
@@ -35,5 +36,20 @@ Context::Context(const DocumentUpdate& docUpdate)
 
 Context::~Context() { }
 
-} // select
-} // document
+std::unique_ptr<Value>
+Context::getValue(const vespalib::string & value) const {
+    VariableMap::const_iterator iter = _variables->find(value);
+
+    if (iter != _variables->end()) {
+        return std::make_unique<FloatValue>(iter->second);
+    } else {
+        return std::make_unique<FloatValue>(0.0);
+    }
+}
+
+void
+Context::setVariableMap(std::unique_ptr<VariableMap> map) {
+    _variables = std::move(map);
+}
+
+}
diff --git a/document/src/vespa/document/select/context.h b/document/src/vespa/document/select/context.h
index 11a3a2be51c..0e347e22bb5 100644
--- a/document/src/vespa/document/select/context.h
+++ b/document/src/vespa/document/select/context.h
@@ -2,9 +2,9 @@
 #pragma once
 
 #include <vespa/vespalib/stllike/string.h>
-#include <vespa/vespalib/stllike/hash_map.h>
+#include <memory>
 
-namespace document{
+namespace document {
 
 class Document;
 class DocumentId;
@@ -12,24 +12,26 @@ class DocumentUpdate;
 
 namespace select {
 
-class Context
-{
-public:
-    typedef vespalib::hash_map<vespalib::string, double> VariableMap;
+class Value;
+class VariableMap;
 
+class Context {
+public:
     Context();
-    Context(const Document& doc);
-    Context(const DocumentId& docId);
-    Context(const DocumentUpdate& docUpdate);
+    Context(const Document & doc);
+    Context(const DocumentId & docId);
+    Context(const DocumentUpdate & docUpdate);
     virtual ~Context();
 
-    const Document       * _doc;
-    const DocumentId     * _docId;
-    const DocumentUpdate * _docUpdate;
-    VariableMap            _variables;
-};
-
-} // select
-} // document
+    void setVariableMap(std::unique_ptr<VariableMap> map);
+    std::unique_ptr<Value> getValue(const vespalib::string & value) const;
 
+    const Document *_doc;
+    const DocumentId *_docId;
+    const DocumentUpdate *_docUpdate;
+private:
+    std::unique_ptr<VariableMap> _variables;
+};
 
+}
+}
diff --git a/document/src/vespa/document/select/doctype.cpp b/document/src/vespa/document/select/doctype.cpp
index 3c23b2e5ba1..117f97ea1bc 100644
--- a/document/src/vespa/document/select/doctype.cpp
+++ b/document/src/vespa/document/select/doctype.cpp
@@ -5,9 +5,9 @@
 
 #include <vespa/document/update/documentupdate.h>
 #include <vespa/document/fieldvalue/document.h>
+#include <vespa/document/datatype/documenttype.h>
 
-namespace document {
-namespace select {
+namespace document::select {
 
 namespace {
     bool documentTypeEqualsName(const DocumentType& type,
@@ -88,5 +88,4 @@ DocType::print(std::ostream& out, bool verbose,
     if (_parentheses) out << ')';
 }
 
-} // select
-} // document
+}
diff --git a/document/src/vespa/document/select/invalidconstant.cpp b/document/src/vespa/document/select/invalidconstant.cpp
index 793e34c7bc5..53e3a6e1647 100644
--- a/document/src/vespa/document/select/invalidconstant.cpp
+++ b/document/src/vespa/document/select/invalidconstant.cpp
@@ -2,9 +2,9 @@
 
 #include "invalidconstant.h"
 #include "visitor.h"
+#include <ostream>
 
-namespace document {
-namespace select {
+namespace document::select {
 
 InvalidConstant::InvalidConstant(const vespalib::stringref & value)
     : Node(value)
@@ -26,13 +26,11 @@ InvalidConstant::visit(Visitor &v) const
 
 
 void
-InvalidConstant::print(std::ostream& out, bool,
-                       const std::string&) const
+InvalidConstant::print(std::ostream& out, bool, const std::string&) const
 {
     if (_parentheses) out << '(';
     out << _name;
     if (_parentheses) out << ')';
 }
 
-} // select
-} // document
+}
diff --git a/document/src/vespa/document/select/operator.cpp b/document/src/vespa/document/select/operator.cpp
index b127e29c1ad..3c09a482cf0 100644
--- a/document/src/vespa/document/select/operator.cpp
+++ b/document/src/vespa/document/select/operator.cpp
@@ -4,9 +4,9 @@
 #include <vespa/vespalib/util/regexp.h>
 #include <vespa/vespalib/stllike/asciistream.h>
 #include <vespa/vespalib/stllike/hash_map.hpp>
+#include <cassert>
 
-namespace document {
-namespace select {
+namespace document::select {
 
 Operator::OperatorMap Operator::_operators;
 
@@ -228,5 +228,4 @@ GlobOperator::containsVariables(const vespalib::stringref & expression)
 
 const GlobOperator GlobOperator::GLOB("=");
 
-} // select
-} // document
+}
diff --git a/document/src/vespa/document/select/value.cpp b/document/src/vespa/document/select/value.cpp
index 7638ba3c36e..7a453fe5cf3 100644
--- a/document/src/vespa/document/select/value.cpp
+++ b/document/src/vespa/document/select/value.cpp
@@ -2,7 +2,6 @@
 
 #include "value.h"
 #include "operator.h"
-#include <cstdint>
 
 namespace document {
 namespace select {
diff --git a/document/src/vespa/document/select/valuenode.cpp b/document/src/vespa/document/select/valuenode.cpp
index ebf0a466ad9..96bc46bd54e 100644
--- a/document/src/vespa/document/select/valuenode.cpp
+++ b/document/src/vespa/document/select/valuenode.cpp
@@ -4,6 +4,7 @@
 #include "parser.h"
 #include <vespa/document/base/exceptions.h>
 #include <vespa/document/fieldvalue/fieldvalues.h>
+#include <vespa/document/datatype/documenttype.h>
 #include <vespa/vespalib/util/md5.h>
 #include <vespa/document/util/stringutil.h>
 #include <vespa/vespalib/text/lowercase.h>
@@ -35,6 +36,13 @@ namespace {
     }
 }
 
+std::unique_ptr<Value>
+ValueNode::defaultTrace(std::unique_ptr<Value> val, std::ostream& out) const
+{
+    out << "Returning value " << *val << ".\n";
+    return std::move(val);
+}
+
 InvalidValueNode::InvalidValueNode(const vespalib::stringref & name)
     : _name(name)
 { }
@@ -146,13 +154,7 @@ CurrentTimeValueNode::print(std::ostream& out, bool verbose,
 
 std::unique_ptr<Value>
 VariableValueNode::getValue(const Context& context) const {
-    VariableMap::const_iterator iter = context._variables.find(_value);
-
-    if (iter != context._variables.end()) {
-        return std::unique_ptr<Value>(new FloatValue(iter->second));
-    } else {
-        return std::unique_ptr<Value>(new FloatValue(0.0));
-    }
+    return context.getValue(_value);
 }
 
 
diff --git a/document/src/vespa/document/select/valuenode.h b/document/src/vespa/document/select/valuenode.h
index e3f0b4c4a94..d56b1468508 100644
--- a/document/src/vespa/document/select/valuenode.h
+++ b/document/src/vespa/document/select/valuenode.h
@@ -29,34 +29,25 @@ namespace select {
 class ValueNode : public Printable
 {
 public:
-    typedef std::unique_ptr<ValueNode> UP;
-    typedef Context::VariableMap VariableMap;
+    using UP = std::unique_ptr<ValueNode>;
 
     ValueNode() : _parentheses(false) {}
     virtual ~ValueNode() {}
 
     void setParentheses() { _parentheses = true; }
-
-    void clearParentheses()
-    {
-        _parentheses = false;
-    }
-
+    void clearParentheses() { _parentheses = false; }
     bool hadParentheses() const { return _parentheses; }
 
     virtual std::unique_ptr<Value>
     getValue(const Context& context) const  = 0;
 
     virtual std::unique_ptr<Value>
-    traceValue(const Context &context,
-               std::ostream &out) const {
+    traceValue(const Context &context, std::ostream &out) const {
         return defaultTrace(getValue(context), out);
     }
 
     virtual void print(std::ostream& out, bool verbose, const std::string& indent) const override = 0;
-
     virtual void visit(Visitor&) const = 0;
-
     virtual ValueNode::UP clone() const = 0;
 private:
     bool _parentheses; // Set to true if parentheses was used around this part
@@ -70,12 +61,7 @@ protected:
         return ret;
     }
 
-    std::unique_ptr<Value> defaultTrace(std::unique_ptr<Value> val,
-                                      std::ostream& out) const
-    {
-        out << "Returning value " << *val << ".\n";
-        return std::move(val);
-    }
+    std::unique_ptr<Value> defaultTrace(std::unique_ptr<Value> val, std::ostream& out) const;
 };
 
 class InvalidValueNode : public ValueNode
diff --git a/document/src/vespa/document/select/variablemap.h b/document/src/vespa/document/select/variablemap.h
new file mode 100644
index 00000000000..86a1cb85e63
--- /dev/null
+++ b/document/src/vespa/document/select/variablemap.h
@@ -0,0 +1,15 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/vespalib/stllike/hash_map.h>
+#include <vespa/vespalib/stllike/string.h>
+
+namespace document::select {
+
+using VariableMapT = vespalib::hash_map<vespalib::string, double>;
+
+class VariableMap : public VariableMapT {
+public:
+    using VariableMapT::VariableMapT;
+};
+
+}
diff --git a/document/src/vespa/document/serialization/vespadocumentdeserializer.cpp b/document/src/vespa/document/serialization/vespadocumentdeserializer.cpp
index 93856b376aa..b2a2bd098a6 100644
--- a/document/src/vespa/document/serialization/vespadocumentdeserializer.cpp
+++ b/document/src/vespa/document/serialization/vespadocumentdeserializer.cpp
@@ -27,6 +27,7 @@
 #include <vespa/document/util/serializableexceptions.h>
 #include <vespa/document/base/exceptions.h>
 #include <vespa/vespalib/objects/nbostream.h>
+#include <vespa/document/util/bytebuffer.h>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".vespadocumentdeserializer");
diff --git a/document/src/vespa/document/serialization/vespadocumentserializer.cpp b/document/src/vespa/document/serialization/vespadocumentserializer.cpp
index 11317f2ff0a..1f6a4a4f0b0 100644
--- a/document/src/vespa/document/serialization/vespadocumentserializer.cpp
+++ b/document/src/vespa/document/serialization/vespadocumentserializer.cpp
@@ -24,7 +24,9 @@
 #include <vespa/document/update/fieldpathupdates.h>
 #include <vespa/vespalib/data/slime/binary_format.h>
 #include <vespa/vespalib/objects/nbostream.h>
+#include <vespa/vespalib/data/databuffer.h>
 #include <vespa/eval/tensor/serialization/typed_binary_format.h>
+#include <vespa/document/util/compressor.h>
 
 using std::make_pair;
 using std::pair;
diff --git a/document/src/vespa/document/update/assignfieldpathupdate.cpp b/document/src/vespa/document/update/assignfieldpathupdate.cpp
index 0a676c88df2..4b8ee510576 100644
--- a/document/src/vespa/document/update/assignfieldpathupdate.cpp
+++ b/document/src/vespa/document/update/assignfieldpathupdate.cpp
@@ -1,9 +1,10 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "assignfieldpathupdate.h"
 #include <vespa/document/fieldvalue/fieldvalues.h>
-#include <vespa/document/repo/fixedtyperepo.h>
 #include <vespa/document/select/parser.h>
+#include <vespa/document/select/variablemap.h>
 #include <vespa/document/serialization/vespadocumentdeserializer.h>
-#include <vespa/document/update/assignfieldpathupdate.h>
 #include <vespa/vespalib/objects/nbostream.h>
 #include <vespa/vespalib/util/exceptions.h>
 #include <boost/numeric/conversion/cast.hpp>
@@ -108,7 +109,8 @@ FieldValue::IteratorHandler::ModificationStatus
 AssignFieldPathUpdate::AssignExpressionIteratorHandler::doModify(FieldValue& fv) {
     LOG(spam, "fv = %s", fv.toString().c_str());
     if (fv.inherits(NumericFieldValueBase::classId)) {
-        DocumentCalculator::VariableMap vars;
+        std::unique_ptr<select::VariableMap> varHolder = std::make_unique<select::VariableMap>();
+        select::VariableMap & vars = *varHolder;
         for (VariableMap::const_iterator i(getVariables().begin()),
                  e(getVariables().end()); i != e; ++i)
         {
@@ -122,7 +124,7 @@ AssignFieldPathUpdate::AssignExpressionIteratorHandler::doModify(FieldValue& fv)
         vars["value"] = fv.getAsDouble();
 
         try {
-            double res = _calc.evaluate(_doc, std::move(vars));
+            double res = _calc.evaluate(_doc, std::move(varHolder));
             if (_removeIfZero && static_cast<uint64_t>(res) == 0) {
                 return REMOVED;
             } else {
diff --git a/document/src/vespa/document/update/clearvalueupdate.cpp b/document/src/vespa/document/update/clearvalueupdate.cpp
index 13143e004ee..cb526852299 100644
--- a/document/src/vespa/document/update/clearvalueupdate.cpp
+++ b/document/src/vespa/document/update/clearvalueupdate.cpp
@@ -1,15 +1,15 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
+#include "clearvalueupdate.h"
 #include <vespa/document/base/field.h>
-#include <vespa/document/update/clearvalueupdate.h>
 #include <vespa/document/fieldvalue/document.h>
 #include <vespa/vespalib/util/exceptions.h>
+#include <ostream>
 
 using vespalib::IllegalArgumentException;
 using vespalib::IllegalStateException;
 
-namespace document
-{
+namespace document {
 
 IMPLEMENT_IDENTIFIABLE(ClearValueUpdate, ValueUpdate);
 
diff --git a/document/src/vespa/document/update/documentupdate.cpp b/document/src/vespa/document/update/documentupdate.cpp
index 866a215505c..d91b33650d4 100644
--- a/document/src/vespa/document/update/documentupdate.cpp
+++ b/document/src/vespa/document/update/documentupdate.cpp
@@ -1,4 +1,5 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
 #include "documentupdate.h"
 #include "documentupdateflags.h"
 #include <vespa/document/fieldvalue/fieldvalues.h>
@@ -7,6 +8,7 @@
 #include <vespa/vespalib/objects/nbostream.h>
 #include <vespa/document/util/bufferexceptions.h>
 #include <vespa/document/base/exceptions.h>
+#include <vespa/document/datatype/documenttype.h>
 
 using vespalib::IllegalArgumentException;
 using vespalib::IllegalStateException;
@@ -103,7 +105,28 @@ DocumentUpdate::affectsDocumentBody() const
     return false;
 }
 
-// Print the content of this document update.
+const DocumentType&
+DocumentUpdate::getType() const {
+    return static_cast<const DocumentType &> (*_type);
+}
+
+DocumentUpdate&
+DocumentUpdate::addUpdate(const FieldUpdate& update) {
+    _updates.push_back(update);
+    return *this;
+}
+
+DocumentUpdate&
+DocumentUpdate::addFieldPathUpdate(const FieldPathUpdate::CP& update) {
+    _fieldPathUpdates.push_back(update);
+    return *this;
+}
+
+DocumentUpdate*
+DocumentUpdate::clone() const {
+    return new DocumentUpdate(*this);
+}
+
 void
 DocumentUpdate::print(std::ostream& out, bool verbose,
                       const std::string& indent) const
@@ -230,8 +253,7 @@ DocumentUpdate::deserialize42(const DocumentTypeRepo& repo, ByteBuffer& buffer)
     try{
         buffer.getShortNetwork(_version);
 
-        std::pair<const DocumentType *, DocumentId> typeAndId(
-                deserializeTypeAndId(repo, buffer));
+        std::pair<const DocumentType *, DocumentId> typeAndId(deserializeTypeAndId(repo, buffer));
         _type = typeAndId.first;
         _documentId = typeAndId.second;
         // Read field updates, if any.
diff --git a/document/src/vespa/document/update/documentupdate.h b/document/src/vespa/document/update/documentupdate.h
index 5d09bd36816..a1d3e910cb5 100644
--- a/document/src/vespa/document/update/documentupdate.h
+++ b/document/src/vespa/document/update/documentupdate.h
@@ -29,7 +29,6 @@
 #include "fieldpathupdate.h"
 #include <vespa/document/base/documentid.h>
 #include <vespa/document/base/field.h>
-#include <vespa/document/datatype/documenttype.h>
 #include <vespa/document/fieldvalue/fieldvalue.h>
 #include <vespa/document/util/bytebuffer.h>
 
@@ -118,19 +117,13 @@ public:
      * Add a field update to this document update.
      * @return A reference to this.
      */
-    DocumentUpdate& addUpdate(const FieldUpdate& update) {
-        _updates.push_back(update);
-        return *this;
-    }
+    DocumentUpdate& addUpdate(const FieldUpdate& update);
 
     /**
      * Add a fieldpath update to this document update.
      * @return A reference to this.
      */
-    DocumentUpdate& addFieldPathUpdate(const FieldPathUpdate::CP& update) {
-        _fieldPathUpdates.push_back(update);
-        return *this;
-    }
+    DocumentUpdate& addFieldPathUpdate(const FieldPathUpdate::CP& update);
 
     /** @return The list of updates. */
     const FieldUpdateV & getUpdates() const { return _updates; }
@@ -141,9 +134,8 @@ public:
     bool affectsDocumentBody() const;
 
     /** @return The type of document this update is for. */
-    const DocumentType& getType() const { return static_cast<const DocumentType &> (*_type); }
+    const DocumentType& getType() const;
 	
-    // Printable implementation
     void print(std::ostream& out, bool verbose, const std::string& indent) const override;
 
     void deserialize42(const DocumentTypeRepo&, ByteBuffer&);
@@ -155,13 +147,9 @@ public:
     void serialize42(vespalib::nbostream &stream) const;
     void serializeHEAD(vespalib::nbostream &stream) const;
 
-    // XmlSerializable implementation
     void printXml(XmlOutputStream&) const override;
 
-    // Cloneable implementation
-    virtual DocumentUpdate* clone() const {
-        return new DocumentUpdate(*this);
-    }
+    virtual DocumentUpdate* clone() const;
 
     /**
      * Sets whether this update should create the document it updates if that document does not exist.
@@ -199,7 +187,6 @@ private:
     DocumentUpdate();
 
     int deserializeFlags(int sizeAndFlags);
-
 };
 
 } // document
diff --git a/document/src/vespa/document/update/fieldupdate.cpp b/document/src/vespa/document/update/fieldupdate.cpp
index 3141fdf3a4f..e25b95e56c1 100644
--- a/document/src/vespa/document/update/fieldupdate.cpp
+++ b/document/src/vespa/document/update/fieldupdate.cpp
@@ -3,6 +3,7 @@
 #include "fieldupdate.h"
 #include <vespa/document/base/exceptions.h>
 #include <vespa/document/fieldvalue/document.h>
+#include <vespa/document/datatype/documenttype.h>
 
 namespace document {
 
diff --git a/documentapi/src/tests/messagebus/messagebus_test.cpp b/documentapi/src/tests/messagebus/messagebus_test.cpp
index 58fc96b2e78..e4b04b0afc7 100644
--- a/documentapi/src/tests/messagebus/messagebus_test.cpp
+++ b/documentapi/src/tests/messagebus/messagebus_test.cpp
@@ -1,6 +1,7 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 #include <vespa/document/base/testdocrepo.h>
 #include <vespa/document/fieldvalue/document.h>
+#include <vespa/document/datatype/documenttype.h>
 #include <vespa/documentapi/documentapi.h>
 #include <vespa/vdslib/state/clusterstate.h>
 #include <vespa/vespalib/testkit/testapp.h>
diff --git a/documentapi/src/tests/messages/messages50test.cpp b/documentapi/src/tests/messages/messages50test.cpp
index 44aea1cb169..3c9f068468f 100644
--- a/documentapi/src/tests/messages/messages50test.cpp
+++ b/documentapi/src/tests/messages/messages50test.cpp
@@ -4,6 +4,7 @@
 #include <vespa/document/datatype/datatype.h>
 #include <vespa/document/fieldvalue/document.h>
 #include <vespa/document/update/fieldpathupdates.h>
+#include <vespa/document/datatype/documenttype.h>
 #include <vespa/documentapi/documentapi.h>
 #include <vespa/vdslib/container/writabledocumentlist.h>
 
diff --git a/documentapi/src/tests/messages/messages52test.cpp b/documentapi/src/tests/messages/messages52test.cpp
index 9339d23ee86..37a133add91 100644
--- a/documentapi/src/tests/messages/messages52test.cpp
+++ b/documentapi/src/tests/messages/messages52test.cpp
@@ -5,6 +5,7 @@
 #include "messages52test.h"
 #include <vespa/documentapi/documentapi.h>
 #include <vespa/document/update/fieldpathupdates.h>
+#include <vespa/document/datatype/documenttype.h>
 
 using document::DocumentTypeRepo;
 
diff --git a/documentapi/src/tests/policies/policies_test.cpp b/documentapi/src/tests/policies/policies_test.cpp
index 3527e62840b..5d92b495c48 100644
--- a/documentapi/src/tests/policies/policies_test.cpp
+++ b/documentapi/src/tests/policies/policies_test.cpp
@@ -23,7 +23,9 @@
 #include <vespa/vdslib/state/clusterstate.h>
 #include <vespa/document/base/testdocrepo.h>
 #include <vespa/document/fieldvalue/longfieldvalue.h>
+#include <vespa/document/datatype/documenttype.h>
 #include <vespa/vespalib/testkit/testapp.h>
+#include <vespa/vespalib/util/stringfmt.h>
 
 #include <vespa/log/log.h>
 LOG_SETUP("policies_test");
diff --git a/documentapi/src/vespa/documentapi/loadtypes/loadtypeset.cpp b/documentapi/src/vespa/documentapi/loadtypes/loadtypeset.cpp
index 4f0f3508d20..062ed58ca87 100644
--- a/documentapi/src/vespa/documentapi/loadtypes/loadtypeset.cpp
+++ b/documentapi/src/vespa/documentapi/loadtypes/loadtypeset.cpp
@@ -3,6 +3,7 @@
 #include "loadtypeset.h"
 #include <vespa/config-load-type.h>
 #include <vespa/config/config.h>
+#include <vespa/vespalib/util/stringfmt.h>
 #include <vespa/vespalib/stllike/hash_map.hpp>
 #include <vespa/config/helper/configgetter.hpp>
 
diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/documentrouteselectorpolicy.cpp b/documentapi/src/vespa/documentapi/messagebus/policies/documentrouteselectorpolicy.cpp
index 61c6648aee6..4dae20e52d5 100644
--- a/documentapi/src/vespa/documentapi/messagebus/policies/documentrouteselectorpolicy.cpp
+++ b/documentapi/src/vespa/documentapi/messagebus/policies/documentrouteselectorpolicy.cpp
@@ -12,6 +12,7 @@
 #include <vespa/messagebus/emptyreply.h>
 #include <vespa/messagebus/routing/routingtable.h>
 #include <vespa/messagebus/messagebus.h>
+#include <vespa/vespalib/util/stringfmt.h>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".documentrouteselectorpolicy");
diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/externpolicy.cpp b/documentapi/src/vespa/documentapi/messagebus/policies/externpolicy.cpp
index 3a35e995805..d1a2e7b135b 100644
--- a/documentapi/src/vespa/documentapi/messagebus/policies/externpolicy.cpp
+++ b/documentapi/src/vespa/documentapi/messagebus/policies/externpolicy.cpp
@@ -3,9 +3,7 @@
 #include <boost/tokenizer.hpp>
 #include <vespa/documentapi/messagebus/documentprotocol.h>
 #include <vespa/messagebus/emptyreply.h>
-#include <vespa/messagebus/errorcode.h>
-#include <vespa/messagebus/routing/route.h>
-#include <vespa/messagebus/routing/routingcontext.h>
+#include <vespa/vespalib/util/stringfmt.h>
 #include <vespa/slobrok/sbmirror.h>
 #include <vespa/fnet/frt/frt.h>
 
diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/messagetypepolicy.cpp b/documentapi/src/vespa/documentapi/messagebus/policies/messagetypepolicy.cpp
index 4d260a8e81f..ba4d0cff079 100644
--- a/documentapi/src/vespa/documentapi/messagebus/policies/messagetypepolicy.cpp
+++ b/documentapi/src/vespa/documentapi/messagebus/policies/messagetypepolicy.cpp
@@ -1,12 +1,26 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
 #include "messagetypepolicy.h"
 #include <vespa/documentapi/messagebus/documentprotocol.h>
+#include <vespa/messagebus/routing/route.h>
+#include <vespa/messagebus/routing/routingcontext.h>
 #include <vespa/vespalib/stllike/hash_map.hpp>
 
 using vespa::config::content::MessagetyperouteselectorpolicyConfig;
 
 namespace documentapi {
 
+namespace policy {
+
+using MessageTypeMapT = vespalib::hash_map<int, mbus::Route>;
+
+class MessageTypeMap : public MessageTypeMapT {
+public:
+    using MessageTypeMapT::MessageTypeMapT;
+};
+
+}
+
 MessageTypePolicy::MessageTypePolicy(const config::ConfigUri & configUri) :
     mbus::IRoutingPolicy(),
     config::IFetcherCallback<MessagetyperouteselectorpolicyConfig>(),
@@ -23,7 +37,7 @@ MessageTypePolicy::~MessageTypePolicy() {}
 void
 MessageTypePolicy::configure(std::unique_ptr<MessagetyperouteselectorpolicyConfig> cfg)
 {
-    std::unique_ptr<MessageTypeMap> map(new MessageTypeMap);
+    auto map = std::make_unique<policy::MessageTypeMap>();
     for (size_t i(0), m(cfg->route.size()); i < m; i++) {
         const MessagetyperouteselectorpolicyConfig::Route & r = cfg->route[i];
         (*map)[r.messagetype] = mbus::Route::parse(r.name);
@@ -38,8 +52,8 @@ void
 MessageTypePolicy::select(mbus::RoutingContext & context)
 {
     int messageType = context.getMessage().getType();
-    std::shared_ptr<MessageTypeMap> map = _map.get();
-    MessageTypeMap::const_iterator found = map->find(messageType);
+    std::shared_ptr<policy::MessageTypeMap> map = _map.get();
+    policy::MessageTypeMap::const_iterator found = map->find(messageType);
     if (found != map->end()) {
          context.addChild(found->second);
     } else {
diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/messagetypepolicy.h b/documentapi/src/vespa/documentapi/messagebus/policies/messagetypepolicy.h
index d9a324a79c5..18ae4d1acb8 100644
--- a/documentapi/src/vespa/documentapi/messagebus/policies/messagetypepolicy.h
+++ b/documentapi/src/vespa/documentapi/messagebus/policies/messagetypepolicy.h
@@ -1,20 +1,20 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 #pragma once
 
-#include <vespa/document/select/node.h>
-#include <map>
 #include <vespa/messagebus/routing/iroutingpolicy.h>
-#include <vespa/messagebus/routing/route.h>
-#include <vespa/messagebus/routing/routingcontext.h>
-#include <vespa/vespalib/util/sync.h>
 #include <vespa/vespalib/util/ptrholder.h>
 #include <vespa/config-messagetyperouteselectorpolicy.h>
 #include <vespa/config/config.h>
 #include <vespa/config/helper/configfetcher.h>
 #include <vespa/documentapi/common.h>
 
+namespace mbus {
+    class RoutingContext;
+    class Route;
+}
 namespace documentapi {
 
+namespace policy {class MessageTypeMap; }
 /**
  * This policy is responsible for selecting among the given recipient routes
  * according to the configured document selection properties. To factilitate
@@ -26,8 +26,7 @@ class MessageTypePolicy : public mbus::IRoutingPolicy,
                           public config::IFetcherCallback<vespa::config::content::MessagetyperouteselectorpolicyConfig>
 {
 private:
-    typedef vespalib::hash_map<int, mbus::Route> MessageTypeMap;
-    typedef vespalib::PtrHolder<MessageTypeMap> MessageTypeHolder;
+    typedef vespalib::PtrHolder<policy::MessageTypeMap> MessageTypeHolder;
     typedef vespalib::PtrHolder<mbus::Route> RouteHolder;
 
     MessageTypeHolder     _map;
diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/storagepolicy.cpp b/documentapi/src/vespa/documentapi/messagebus/policies/storagepolicy.cpp
index 5aaf7919d71..f19e0c4c85f 100644
--- a/documentapi/src/vespa/documentapi/messagebus/policies/storagepolicy.cpp
+++ b/documentapi/src/vespa/documentapi/messagebus/policies/storagepolicy.cpp
@@ -7,6 +7,7 @@
 #include <vespa/documentapi/documentapi.h>
 #include <vespa/vdslib/state/clusterstate.h>
 #include <vespa/vespalib/stllike/asciistream.h>
+#include <vespa/vespalib/util/stringfmt.h>
 #include <vespa/config-stor-distribution.h>
 #include <vespa/config/helper/ifetchercallback.h>
 #include <vespa/config/helper/configfetcher.h>
diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor_builder.cpp b/eval/src/vespa/eval/tensor/dense/dense_tensor_builder.cpp
index a00537ffa50..3c6e182a143 100644
--- a/eval/src/vespa/eval/tensor/dense/dense_tensor_builder.cpp
+++ b/eval/src/vespa/eval/tensor/dense/dense_tensor_builder.cpp
@@ -2,6 +2,7 @@
 
 #include "dense_tensor_builder.h"
 #include <vespa/vespalib/util/exceptions.h>
+#include <vespa/vespalib/util/stringfmt.h>
 #include <cassert>
 
 using vespalib::IllegalArgumentException;
diff --git a/fastos/src/tests/tests.h b/fastos/src/tests/tests.h
index cc54e66292f..6807741f86f 100644
--- a/fastos/src/tests/tests.h
+++ b/fastos/src/tests/tests.h
@@ -2,6 +2,7 @@
 
 #include <vespa/fastos/app.h>
 #include <vespa/fastos/socket.h>
+#include <vespa/fastos/thread.h>
 
 class BaseTest : public FastOS_Application
 {
diff --git a/fastos/src/vespa/fastos/app.h b/fastos/src/vespa/fastos/app.h
index 75fb5c30d43..faa6ff55c7d 100644
--- a/fastos/src/vespa/fastos/app.h
+++ b/fastos/src/vespa/fastos/app.h
@@ -13,8 +13,8 @@
 #include <vespa/fastos/types.h>
 
 class FastOS_ProcessInterface;
+class FastOS_ThreadPool;
 
-#include <vespa/fastos/thread.h>
 #include <vespa/fastos/mutex.h>
 
 /**
diff --git a/fastos/src/vespa/fastos/unix_process.h b/fastos/src/vespa/fastos/unix_process.h
index d3b3f0f3d96..7438b96d434 100644
--- a/fastos/src/vespa/fastos/unix_process.h
+++ b/fastos/src/vespa/fastos/unix_process.h
@@ -13,7 +13,7 @@
 #include <string>
 #include <memory>
 
-class FastOS_ThreadPool;
+class FastOS_BoolCond;
 class FastOS_UNIX_RealProcess;
 
 #include <vespa/fastos/ringbuffer.h>
diff --git a/filedistribution/src/vespa/filedistribution/distributor/filedownloader.cpp b/filedistribution/src/vespa/filedistribution/distributor/filedownloader.cpp
index cbd93eb2e57..14cb8844d8b 100644
--- a/filedistribution/src/vespa/filedistribution/distributor/filedownloader.cpp
+++ b/filedistribution/src/vespa/filedistribution/distributor/filedownloader.cpp
@@ -2,6 +2,7 @@
 #include "filedownloader.h"
 #include "hostname.h"
 #include <vespa/filedistribution/model/zkfacade.h>
+#include <vespa/vespalib/util/stringfmt.h>
 
 #include <boost/filesystem.hpp>
 #include <boost/filesystem/fstream.hpp>
diff --git a/filedistribution/src/vespa/filedistribution/distributor/hostname.cpp b/filedistribution/src/vespa/filedistribution/distributor/hostname.cpp
index 7b33632df81..f307ae1fca3 100644
--- a/filedistribution/src/vespa/filedistribution/distributor/hostname.cpp
+++ b/filedistribution/src/vespa/filedistribution/distributor/hostname.cpp
@@ -1,6 +1,7 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-#include "hostname.h"
 
+#include "hostname.h"
+#include <vespa/vespalib/util/stringfmt.h>
 #include <vespa/vespalib/net/socket_address.h>
 #include <vespa/log/log.h>
 LOG_SETUP(".hostname");
diff --git a/filedistribution/src/vespa/filedistribution/model/filedistributionmodelimpl.cpp b/filedistribution/src/vespa/filedistribution/model/filedistributionmodelimpl.cpp
index 3b4d043a30b..15ff198b8d5 100644
--- a/filedistribution/src/vespa/filedistribution/model/filedistributionmodelimpl.cpp
+++ b/filedistribution/src/vespa/filedistribution/model/filedistributionmodelimpl.cpp
@@ -4,6 +4,7 @@
 #include "zkfiledbmodel.h"
 #include "deployedfilestodownload.h"
 #include "filedistributionmodelimpl.h"
+#include <vespa/vespalib/util/stringfmt.h>
 #include <boost/filesystem.hpp>
 #include <zookeeper/zookeeper.h>
 
diff --git a/filedistribution/src/vespa/filedistribution/model/zkfacade.cpp b/filedistribution/src/vespa/filedistribution/model/zkfacade.cpp
index 2400eae9eb7..28e0e82f79a 100644
--- a/filedistribution/src/vespa/filedistribution/model/zkfacade.cpp
+++ b/filedistribution/src/vespa/filedistribution/model/zkfacade.cpp
@@ -2,15 +2,15 @@
 
 #include "zkfacade.h"
 #include <vespa/vespalib/net/socket_address.h>
-#include <sstream>
-#include <thread>
-#include <boost/function_output_iterator.hpp>
-
-#include <zookeeper/zookeeper.h>
 #include <vespa/filedistribution/common/logfwd.h>
 #include <vespa/defaults.h>
 #include <vespa/vespalib/util/sync.h>
 #include <vespa/vespalib/text/stringtokenizer.h>
+#include <vespa/vespalib/util/stringfmt.h>
+#include <zookeeper/zookeeper.h>
+#include <sstream>
+#include <thread>
+#include <boost/function_output_iterator.hpp>
 
 typedef std::unique_lock<std::mutex> UniqueLock;
 
diff --git a/filedistribution/src/vespa/filedistribution/model/zkfiledbmodel.cpp b/filedistribution/src/vespa/filedistribution/model/zkfiledbmodel.cpp
index 5198b6bbeb7..4030b777afa 100644
--- a/filedistribution/src/vespa/filedistribution/model/zkfiledbmodel.cpp
+++ b/filedistribution/src/vespa/filedistribution/model/zkfiledbmodel.cpp
@@ -5,6 +5,7 @@
 #include "zkfiledbmodel.h"
 #include "deployedfilestodownload.h"
 #include <vespa/filedistribution/common/logfwd.h>
+#include <vespa/vespalib/util/stringfmt.h>
 #include <sys/file.h>
 #include <ostream>
 #include <algorithm>
diff --git a/fnet/src/examples/frt/rpc/CMakeLists.txt b/fnet/src/examples/frt/rpc/CMakeLists.txt
index aae76bdcd4f..338fe761b97 100644
--- a/fnet/src/examples/frt/rpc/CMakeLists.txt
+++ b/fnet/src/examples/frt/rpc/CMakeLists.txt
@@ -2,21 +2,18 @@
 vespa_add_executable(fnet_rpc_server_app
     SOURCES
     rpc_server.cpp
-    INSTALL bin
     DEPENDS
     fnet
 )
 vespa_add_executable(fnet_rpc_client_app
     SOURCES
     rpc_client.cpp
-    INSTALL bin
     DEPENDS
     fnet
 )
 vespa_add_executable(fnet_echo_client_app
     SOURCES
     echo_client.cpp
-    INSTALL bin
     DEPENDS
     fnet
 )
@@ -24,7 +21,6 @@ vespa_add_executable(fnet_rpc_info_app
     SOURCES
     rpc_info.cpp
     OUTPUT_NAME rpc_info
-    INSTALL bin
     DEPENDS
     fnet
 )
@@ -39,14 +35,12 @@ vespa_add_executable(fnet_rpc_proxy_app
 vespa_add_executable(fnet_rpc_callback_server_app
     SOURCES
     rpc_callback_server.cpp
-    INSTALL bin
     DEPENDS
     fnet
 )
 vespa_add_executable(fnet_rpc_callback_client_app
     SOURCES
     rpc_callback_client.cpp
-    INSTALL bin
     DEPENDS
     fnet
 )
diff --git a/fnet/src/examples/ping/CMakeLists.txt b/fnet/src/examples/ping/CMakeLists.txt
index 7354ec9cf48..ff635799848 100644
--- a/fnet/src/examples/ping/CMakeLists.txt
+++ b/fnet/src/examples/ping/CMakeLists.txt
@@ -3,7 +3,6 @@ vespa_add_executable(fnet_pingserver_app
     SOURCES
     packets.cpp
     pingserver.cpp
-    INSTALL bin
     DEPENDS
     fnet
 )
@@ -11,7 +10,6 @@ vespa_add_executable(fnet_pingclient_app
     SOURCES
     packets.cpp
     pingclient.cpp
-    INSTALL bin
     DEPENDS
     fnet
 )
diff --git a/fnet/src/examples/proxy/CMakeLists.txt b/fnet/src/examples/proxy/CMakeLists.txt
index bc5dce755a7..45710fc0779 100644
--- a/fnet/src/examples/proxy/CMakeLists.txt
+++ b/fnet/src/examples/proxy/CMakeLists.txt
@@ -2,7 +2,6 @@
 vespa_add_executable(fnet_proxy_app
     SOURCES
     proxy.cpp
-    INSTALL bin
     DEPENDS
     fnet
 )
diff --git a/fnet/src/examples/timeout/CMakeLists.txt b/fnet/src/examples/timeout/CMakeLists.txt
index 5b9514ab0c8..2f4a710c1f2 100644
--- a/fnet/src/examples/timeout/CMakeLists.txt
+++ b/fnet/src/examples/timeout/CMakeLists.txt
@@ -2,7 +2,6 @@
 vespa_add_executable(fnet_timeout_app
     SOURCES
     timeout.cpp
-    INSTALL bin
     DEPENDS
     fnet
 )
diff --git a/fnet/src/tests/sync_execute/sync_execute.cpp b/fnet/src/tests/sync_execute/sync_execute.cpp
index ca77ab6f73b..8b0edb4476f 100644
--- a/fnet/src/tests/sync_execute/sync_execute.cpp
+++ b/fnet/src/tests/sync_execute/sync_execute.cpp
@@ -2,6 +2,7 @@
 #include <vespa/vespalib/testkit/test_kit.h>
 #include <vespa/fnet/transport.h>
 #include <vespa/fnet/iexecutable.h>
+#include <vespa/fastos/thread.h>
 
 struct DoIt : public FNET_IExecutable {
     vespalib::Gate gate;
diff --git a/install_java.cmake b/install_java.cmake
index 361a07dad5d..408582b387e 100644
--- a/install_java.cmake
+++ b/install_java.cmake
@@ -46,6 +46,7 @@ install_fat_java_artifact(simplemetrics)
 install_fat_java_artifact(standalone-container)
 install_fat_java_artifact(vespa-http-client)
 install_fat_java_artifact(vespaclient-container-plugin)
+install_fat_java_artifact(vespaclient-java)
 install_fat_java_artifact(zkfacade)
 
 vespa_install_script(application-preprocessor/src/main/sh/vespa-preprocess-application bin)
@@ -57,6 +58,15 @@ vespa_install_script(config-model/src/main/perl/expand-config.pl bin)
 vespa_install_script(config-model/src/main/perl/vespa-replicate-log-stream bin)
 vespa_install_script(config-model/src/main/sh/validate-application bin)
 vespa_install_script(container-disc/src/main/sh/vespa-start-container-daemon.sh vespa-start-container-daemon bin)
+vespa_install_script(vespaclient-java/src/main/sh/vds-document-statistics.sh vds-document-statistics bin)
+vespa_install_script(vespaclient-java/src/main/sh/vdsstat.sh vdsstat bin)
+vespa_install_script(vespaclient-java/src/main/sh/vespa-query-profile-dump-tool.sh vespa-query-profile-dump-tool bin)
+vespa_install_script(vespaclient-java/src/main/sh/vespa-summary-benchmark.sh vespa-summary-benchmark bin)
+vespa_install_script(vespaclient-java/src/main/sh/vespadestination.sh vespadestination bin)
+vespa_install_script(vespaclient-java/src/main/sh/vespafeeder.sh vespafeeder bin)
+vespa_install_script(vespaclient-java/src/main/sh/vespaget.sh vespaget bin)
+vespa_install_script(vespaclient-java/src/main/sh/vespavisit.sh vespavisit bin)
+vespa_install_script(vespaclient-java/src/main/sh/vespavisittarget.sh vespavisittarget bin)
 
 vespa_install_script(logserver/bin/logserver-start.sh logserver-start bin)
 
diff --git a/memfilepersistence/src/tests/device/devicemanagertest.cpp b/memfilepersistence/src/tests/device/devicemanagertest.cpp
index f5e16032361..50a4e8ef56c 100644
--- a/memfilepersistence/src/tests/device/devicemanagertest.cpp
+++ b/memfilepersistence/src/tests/device/devicemanagertest.cpp
@@ -3,6 +3,7 @@
 #include <vespa/memfilepersistence/device/devicemanager.h>
 #include <vespa/vdstestlib/cppunit/macros.h>
 #include <vespa/vespalib/util/exception.h>
+#include <vespa/vespalib/util/stringfmt.h>
 #include <vespa/storageframework/defaultimplementation/clock/fakeclock.h>
 
 namespace storage {
diff --git a/memfilepersistence/src/vespa/memfilepersistence/common/types.cpp b/memfilepersistence/src/vespa/memfilepersistence/common/types.cpp
index f02423cb194..2bc85fbc422 100644
--- a/memfilepersistence/src/vespa/memfilepersistence/common/types.cpp
+++ b/memfilepersistence/src/vespa/memfilepersistence/common/types.cpp
@@ -1,11 +1,11 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "types.h"
-#include <sstream>
 #include <vespa/vespalib/util/exceptions.h>
+#include <cassert>
+#include <sstream>
 
-namespace storage {
-namespace memfile {
+namespace storage::memfile {
 
 const framework::MicroSecTime Types::MAX_TIMESTAMP(framework::MicroSecTime::max());
 const framework::MicroSecTime Types::UNSET_TIMESTAMP(0);
@@ -34,5 +34,17 @@ operator<<(std::ostream& os, const DataLocation& loc)
     return os;
 }
 
-} // memfile
-} // storage
+const char*
+Types::getMemFileFlagName(MemFileFlag flag) {
+    switch (flag) {
+        case FILE_EXIST: return "FILE_EXIST";
+        case HEADER_BLOCK_READ: return "HEADER_BLOCK_READ";
+        case BODY_BLOCK_READ: return "BODY_BLOCK_READ";
+        case BUCKET_INFO_OUTDATED: return "BUCKET_INFO_OUTDATED";
+        case SLOTS_ALTERED: return "SLOTS_ALTERED";
+        case LEGAL_MEMFILE_FLAGS: assert(false); // Not a single flag
+        default: return "INVALID";
+    }
+}
+
+}
diff --git a/memfilepersistence/src/vespa/memfilepersistence/common/types.h b/memfilepersistence/src/vespa/memfilepersistence/common/types.h
index 7758e6091ac..c1f29571037 100644
--- a/memfilepersistence/src/vespa/memfilepersistence/common/types.h
+++ b/memfilepersistence/src/vespa/memfilepersistence/common/types.h
@@ -173,21 +173,9 @@ struct Types {
         }
     }
 
-    static const char* getMemFileFlagName(MemFileFlag flag) {
-        switch (flag) {
-            case FILE_EXIST: return "FILE_EXIST";
-            case HEADER_BLOCK_READ: return "HEADER_BLOCK_READ";
-            case BODY_BLOCK_READ: return "BODY_BLOCK_READ";
-            case BUCKET_INFO_OUTDATED: return "BUCKET_INFO_OUTDATED";
-            case SLOTS_ALTERED: return "SLOTS_ALTERED";
-            case LEGAL_MEMFILE_FLAGS: assert(false); // Not a single flag
-            default: return "INVALID";
-        }
-    }
-
-    static void verifyLegalFlags(uint32_t flags, uint32_t legal,
-                                 const char* operation);
+    static const char* getMemFileFlagName(MemFileFlag flag);
 
+    static void verifyLegalFlags(uint32_t flags, uint32_t legal, const char* operation);
 protected:
     ~Types() {} // Noone should refer to objects as Types objects
 };
diff --git a/memfilepersistence/src/vespa/memfilepersistence/mapper/simplememfileiobuffer.cpp b/memfilepersistence/src/vespa/memfilepersistence/mapper/simplememfileiobuffer.cpp
index 1f5344d466e..6bda3d69113 100644
--- a/memfilepersistence/src/vespa/memfilepersistence/mapper/simplememfileiobuffer.cpp
+++ b/memfilepersistence/src/vespa/memfilepersistence/mapper/simplememfileiobuffer.cpp
@@ -2,10 +2,11 @@
 
 #include "simplememfileiobuffer.h"
 #include <vespa/document/repo/documenttyperepo.h>
+#include <vespa/document/datatype/documenttype.h>
 #include <vespa/memfilepersistence/common/environment.h>
 #include <vespa/vespalib/util/exceptions.h>
+#include <vespa/document/util/bytebuffer.h>
 #include <vespa/log/log.h>
-
 LOG_SETUP(".memfile.simpleiobuffer");
 
 namespace storage::memfile {
diff --git a/memfilepersistence/src/vespa/memfilepersistence/spi/joinoperationhandler.cpp b/memfilepersistence/src/vespa/memfilepersistence/spi/joinoperationhandler.cpp
index 1f82ffb2172..0fc2aa2ece6 100644
--- a/memfilepersistence/src/vespa/memfilepersistence/spi/joinoperationhandler.cpp
+++ b/memfilepersistence/src/vespa/memfilepersistence/spi/joinoperationhandler.cpp
@@ -3,6 +3,7 @@
 #include "joinoperationhandler.h"
 #include "cacheevictionguard.h"
 #include <vespa/memfilepersistence/mapper/memfilemapper.h>
+#include <vespa/vespalib/util/stringfmt.h>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".persistence.memfile.handler.join");
diff --git a/messagebus/src/vespa/messagebus/testlib/slobrok.cpp b/messagebus/src/vespa/messagebus/testlib/slobrok.cpp
index 10c501db854..12f53e31764 100644
--- a/messagebus/src/vespa/messagebus/testlib/slobrok.cpp
+++ b/messagebus/src/vespa/messagebus/testlib/slobrok.cpp
@@ -4,6 +4,7 @@
 #include <vespa/slobrok/server/sbenv.h>
 #include <vespa/fnet/frt/supervisor.h>
 #include <vespa/fnet/transport.h>
+#include <vespa/vespalib/util/stringfmt.h>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".slobrok");
diff --git a/metrics/src/vespa/metrics/metric.cpp b/metrics/src/vespa/metrics/metric.cpp
index 57ce8e5ffc1..0c99409a8bb 100644
--- a/metrics/src/vespa/metrics/metric.cpp
+++ b/metrics/src/vespa/metrics/metric.cpp
@@ -8,6 +8,7 @@
 #include <vespa/vespalib/text/stringtokenizer.h>
 #include <vespa/vespalib/util/exceptions.h>
 #include <vespa/vespalib/stllike/asciistream.h>
+#include <vespa/vespalib/util/stringfmt.h>
 #include <iterator>
 
 namespace metrics {
diff --git a/metrics/src/vespa/metrics/metricmanager.cpp b/metrics/src/vespa/metrics/metricmanager.cpp
index 572779be855..db05ea16488 100644
--- a/metrics/src/vespa/metrics/metricmanager.cpp
+++ b/metrics/src/vespa/metrics/metricmanager.cpp
@@ -10,7 +10,7 @@
 #include "xmlwriter.h"
 #include <vespa/config/print/ostreamconfigwriter.h>
 #include <vespa/vespalib/text/stringtokenizer.h>
-
+#include <vespa/vespalib/util/stringfmt.h>
 #include <vespa/vespalib/util/exceptions.h>
 #include <vespa/vespalib/stllike/asciistream.h>
 #include <vespa/log/log.h>
diff --git a/metrics/src/vespa/metrics/metricset.cpp b/metrics/src/vespa/metrics/metricset.cpp
index 15ee22ba1f2..c9acbaafce2 100644
--- a/metrics/src/vespa/metrics/metricset.cpp
+++ b/metrics/src/vespa/metrics/metricset.cpp
@@ -4,10 +4,10 @@
 #include "memoryconsumption.h"
 #include <vespa/vespalib/stllike/hash_map.hpp>
 #include <vespa/vespalib/util/exceptions.h>
+#include <vespa/vespalib/util/stringfmt.h>
 #include <list>
 
 #include <vespa/log/log.h>
-
 LOG_SETUP(".metrics.metricsset");
 
 namespace metrics {
diff --git a/metrics/src/vespa/metrics/printutils.cpp b/metrics/src/vespa/metrics/printutils.cpp
index 2373d2391ff..2209ebd5da7 100644
--- a/metrics/src/vespa/metrics/printutils.cpp
+++ b/metrics/src/vespa/metrics/printutils.cpp
@@ -1,6 +1,7 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "printutils.h"
+#include <vespa/vespalib/util/stringfmt.h>
 
 namespace metrics {
 namespace printutils {
diff --git a/metrics/src/vespa/metrics/summetric.hpp b/metrics/src/vespa/metrics/summetric.hpp
index 1a54067b01f..3e85aa0e215 100644
--- a/metrics/src/vespa/metrics/summetric.hpp
+++ b/metrics/src/vespa/metrics/summetric.hpp
@@ -5,6 +5,7 @@
 #include "metricset.h"
 #include "memoryconsumption.h"
 #include <vespa/vespalib/util/exceptions.h>
+#include <vespa/vespalib/util/stringfmt.h>
 #include <ostream>
 
 namespace metrics {
diff --git a/metrics/src/vespa/metrics/valuemetric.hpp b/metrics/src/vespa/metrics/valuemetric.hpp
index 65a0c2bb5b2..b09f35e3272 100644
--- a/metrics/src/vespa/metrics/valuemetric.hpp
+++ b/metrics/src/vespa/metrics/valuemetric.hpp
@@ -4,6 +4,7 @@
 #include "valuemetric.h"
 #include "memoryconsumption.h"
 #include <vespa/vespalib/util/exceptions.h>
+#include <vespa/vespalib/util/stringfmt.h>
 #include <sstream>
 
 namespace metrics {
diff --git a/metrics/src/vespa/metrics/valuemetricvalues.hpp b/metrics/src/vespa/metrics/valuemetricvalues.hpp
index d8becca3e72..e7f3e9e3239 100644
--- a/metrics/src/vespa/metrics/valuemetricvalues.hpp
+++ b/metrics/src/vespa/metrics/valuemetricvalues.hpp
@@ -3,6 +3,7 @@
 
 #include "valuemetricvalues.h"
 #include <vespa/vespalib/util/exceptions.h>
+#include <vespa/vespalib/util/stringfmt.h>
 #include <ostream>
 
 namespace metrics {
diff --git a/persistence/src/tests/proxy/providerproxy_test.cpp b/persistence/src/tests/proxy/providerproxy_test.cpp
index 514d9516c65..28ce33a2b5b 100644
--- a/persistence/src/tests/proxy/providerproxy_test.cpp
+++ b/persistence/src/tests/proxy/providerproxy_test.cpp
@@ -8,6 +8,7 @@
 #include <vespa/document/repo/documenttyperepo.h>
 #include <vespa/document/fieldvalue/document.h>
 #include <vespa/document/update/documentupdate.h>
+#include <vespa/document/datatype/documenttype.h>
 #include <vespa/persistence/proxy/providerproxy.h>
 #include <vespa/persistence/proxy/providerstub.h>
 #include <vespa/persistence/spi/abstractpersistenceprovider.h>
diff --git a/persistence/src/tests/proxy/providerstub_test.cpp b/persistence/src/tests/proxy/providerstub_test.cpp
index 1028e06b8bc..c750a3c94af 100644
--- a/persistence/src/tests/proxy/providerstub_test.cpp
+++ b/persistence/src/tests/proxy/providerstub_test.cpp
@@ -7,6 +7,7 @@
 #include <vespa/document/util/bytebuffer.h>
 #include <vespa/document/fieldvalue/document.h>
 #include <vespa/document/update/documentupdate.h>
+#include <vespa/document/datatype/documenttype.h>
 #include <vespa/persistence/proxy/buildid.h>
 #include <vespa/persistence/proxy/providerstub.h>
 #include <vespa/persistence/spi/abstractpersistenceprovider.h>
diff --git a/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp b/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp
index d2a3ab3be65..05e21096fef 100644
--- a/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp
+++ b/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp
@@ -9,6 +9,7 @@
 #include <vespa/document/fieldset/fieldsetrepo.h>
 #include <vespa/vespalib/stllike/asciistream.h>
 #include <vespa/vespalib/util/exceptions.h>
+#include <vespa/vespalib/util/atomic.h>
 #include <vespa/vespalib/stllike/hash_map.hpp>
 #include <vespa/log/log.h>
 
diff --git a/persistence/src/vespa/persistence/proxy/providerproxy.cpp b/persistence/src/vespa/persistence/proxy/providerproxy.cpp
index 1f6d0b19be6..29b09af9953 100644
--- a/persistence/src/vespa/persistence/proxy/providerproxy.cpp
+++ b/persistence/src/vespa/persistence/proxy/providerproxy.cpp
@@ -8,9 +8,8 @@
 #include <vespa/document/fieldset/fieldsetrepo.h>
 #include <vespa/document/serialization/vespadocumentdeserializer.h>
 #include <vespa/document/serialization/vespadocumentserializer.h>
-#include <vespa/document/util/bytebuffer.h>
+#include <vespa/vespalib/util/stringfmt.h>
 #include <vespa/vespalib/objects/nbostream.h>
-#include <vespa/vespalib/util/noncopyable.hpp>
 #include <vespa/fnet/frt/frt.h>
 #include <vespa/log/log.h>
 LOG_SETUP(".providerproxy");
diff --git a/persistence/src/vespa/persistence/spi/docentry.cpp b/persistence/src/vespa/persistence/spi/docentry.cpp
index c9ceda982e0..1061a06fd28 100644
--- a/persistence/src/vespa/persistence/spi/docentry.cpp
+++ b/persistence/src/vespa/persistence/spi/docentry.cpp
@@ -3,6 +3,7 @@
 #include "docentry.h"
 #include <vespa/document/fieldvalue/document.h>
 #include <sstream>
+#include <cassert>
 
 namespace storage {
 namespace spi {
diff --git a/pom.xml b/pom.xml
index b45a4f31c4b..2b12f720eae 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1168,6 +1168,7 @@
         <module>vdslib</module>
         <module>vespaclient-core</module>
         <module>vespaclient-container-plugin</module>
+        <module>vespaclient-java</module>
         <module>vespa-application-maven-plugin</module>
         <module>vespa-documentgen-plugin</module>
         <module>vespa_feed_perf</module>
diff --git a/sample-apps/blog-recommendation/README.md b/sample-apps/blog-recommendation/README.md
index 0b6a00c9743..705c4e6879c 100644
--- a/sample-apps/blog-recommendation/README.md
+++ b/sample-apps/blog-recommendation/README.md
@@ -1,55 +1 @@
-Basic Search Application
-==================
-Start by [deploying a sample application](http://vespa.corp.yahoo.com/6/documentation/developing-with-vespa.html).
-
-### Find the endpoint
-
-When you have successfully deployed your own compiled version of the application above, you need to find the name of the "endpoint".
-The endpoint is used for feeding and searching for data.
-**Please allow a few minutes for the endpoint to appear after deployment**
-You can find this endpoint by doing:
- ```sh
-
- mvn vespa:endpoints | grep Endpoints
- ```
-
-You can also find it by looking at the [Hosted Vespa Dashboard](http://dashboard.vespa.corp.yahoo.com).
-
-
-### Feed and search
- 1. **Feed** the data that is to be searched
- ```sh
-
- # Feeding two documents
- curl -X POST --data-binary  @music-data-1.json <endpoint url>/document/v1/music/music/docid/1 | python -m json.tool
- curl -X POST --data-binary  @music-data-2.json <endpoint url>/document/v1/music/music/docid/2 | python -m json.tool
-
-  ```
-
-For feeding many documents fast and reliable, checkout [feeding example](https://git.corp.yahoo.com/vespa-samples/basic-feeding-client)
-
- 2. **Visit documents
-
- Since we do not have many documents we can list them all
- ```sh
-
- # All documents
- curl <endpoint url>/document/v1/music/music/docid | python -m json.tool
-
- # Document with id 1
- curl <endpoint url>/document/v1/music/music/docid/1 | python -m json.tool
-
-  ```
-
- 3. **Search**
- We can also search for documents:
-    ```sh
-
-    curl '<endpoint url>/search/?query=bad' | python -m json.tool
-
-
-    ```
-
-### Next step: from development to production
-See [continuous deployments](http://vespa.corp.yahoo.com/6/documentation/continuous-deployment.html) for how to implement continuous deployments for production.
-See [RESTified Document Operation API](http://vespa.corp.yahoo.com/6/documentation/document_api_v1.html) for documentation about the REST API for document operations.
+Blog Recommendation
diff --git a/sample-apps/blog-recommendation/src/pig/feed_content_and_tensor_vespa.pig b/sample-apps/blog-recommendation/src/pig/feed_content_and_tensor_vespa.pig
new file mode 100644
index 00000000000..9a536f38779
--- /dev/null
+++ b/sample-apps/blog-recommendation/src/pig/feed_content_and_tensor_vespa.pig
@@ -0,0 +1,100 @@
+REGISTER vespa-hadoop.jar
+
+-- Create valid Vespa put operations
+DEFINE VespaPutOperationDoc
+       com.yahoo.vespa.hadoop.pig.VespaDocumentOperation(
+            'operation=put',
+            'docid=id:blog-recommendation:blog_post::<post_id>',
+            'create-tensor-fields=user_item_cf',
+            'simple-array-fields=tags,categories'
+       );
+
+DEFINE VespaPutOperationUser
+       com.yahoo.vespa.hadoop.pig.VespaDocumentOperation(
+            'operation=put',
+            'docid=id:blog-recommendation:user::<user_id>',
+            'create-tensor-fields=user_item_cf',
+            'simple-array-fields=has_read_items'
+       );
+
+-- Transform tabular data to a Vespa document operation JSON format
+DEFINE VespaStorage
+       com.yahoo.vespa.hadoop.pig.VespaStorage();
+
+-- DEFINE VespaStorage
+--        com.yahoo.vespa.hadoop.pig.VespaStorage(
+--             'create-document-operation=true',
+--             'operation=put',
+--             'docid=id:blog-recommendation:blog_post::<post_id>'
+--        );
+
+-- Load data from any source - here we load using PigStorage
+data = LOAD 'blog-recommendation/trainPostsFinal' USING JsonLoader('date_gmt:chararray, language:chararray, author:chararray, url:chararray, title:chararray, blog:chararray, post_id:chararray, tags:{T:(tag_name:chararray)}, blogname:chararray, date:chararray, content:chararray, categories:{T:(category_name:chararray)}, likes:{T:(dt:chararray, uid:chararray)}');
+
+data_for_feed = FOREACH data GENERATE 
+  date_gmt, 
+  language, 
+  author, 
+  url, 
+  title, 
+  blog, 
+  post_id, 
+  tags, 
+  blogname,
+  content, 
+  categories;
+
+data_doc = LOAD 'blog-recommendation/user_item_cf/product_features' USING JsonLoader('post_id:chararray, user_item_cf:[double]');
+
+data_content_and_doc_tensor = JOIN data_for_feed BY post_id LEFT, data_doc BY post_id;
+data_content_and_doc_tensor = FOREACH data_content_and_doc_tensor GENERATE
+  date_gmt AS date_gmt, 
+  language AS language, 
+  author AS author, 
+  url AS url, 
+  title AS title, 
+  blog AS blog, 
+  data_for_feed::post_id as post_id, 
+  tags AS tags, 
+  blogname AS blogname,
+  content AS content, 
+  categories AS categories,
+  user_item_cf AS user_item_cf,
+  (user_item_cf IS NOT NULL ? 1 : 0) AS has_user_item_cf;
+
+data_content_and_doc_tensor_feed = FOREACH data_content_and_doc_tensor GENERATE VespaPutOperationDoc(*);
+
+-- use cf latent factor
+data_user = LOAD 'blog-recommendation/user_item_cf/user_features' USING JsonLoader('user_id:chararray, user_item_cf:[double]');
+data_user = FOREACH data_user GENERATE  
+  user_id AS user_id,
+  user_item_cf AS user_item_cf;
+  
+
+-- Articles already liked
+data_likes = FOREACH data GENERATE post_id, FLATTEN(likes) AS (dt, uid); 
+
+post_liked_per_user = GROUP data_likes BY uid;
+post_liked_per_user = FOREACH post_liked_per_user GENERATE group AS user_id, data_likes.post_id AS has_read_items;
+
+-- Join user data
+data_user = JOIN post_liked_per_user BY user_id FULL, data_user BY user_id;
+
+data_user = FOREACH data_user GENERATE (post_liked_per_user::user_id IS NOT NULL ? post_liked_per_user::user_id : data_user::user_id) AS user_id,
+                                       user_item_cf AS user_item_cf,
+                                       (user_item_cf IS NOT NULL ? 1 : 0) AS has_user_item_cf,
+                                       has_read_items AS has_read_items;
+data_user = FILTER data_user BY user_id IS NOT NULL;
+
+data_user_for_feed = FOREACH data_user GENERATE VespaPutOperationUser(*);
+
+joint_content_tensors = UNION data_content_and_doc_tensor_feed, data_user_for_feed;
+
+-- STORE data_for_feed into 'vespa_put_operation';
+
+-- Store into Vespa
+STORE joint_content_tensors INTO '$ENDPOINT' USING VespaStorage();
+
+
+
+
diff --git a/sample-apps/blog-recommendation/src/pig/feed_content_vespa.pig b/sample-apps/blog-recommendation/src/pig/feed_content_vespa.pig
new file mode 100644
index 00000000000..59b173e16f4
--- /dev/null
+++ b/sample-apps/blog-recommendation/src/pig/feed_content_vespa.pig
@@ -0,0 +1,71 @@
+REGISTER vespa-hadoop.jar
+
+-- UDF to create valid Vespa document operation in JSON format
+DEFINE VespaPutOperationDoc
+       com.yahoo.vespa.hadoop.pig.VespaDocumentOperation(
+            'operation=put',
+            'docid=id:blog-search:blog_post::<post_id>',
+            'simple-array-fields=tags,categories'
+            );
+
+-- UDF to send data to a Vespa endpoint
+DEFINE VespaStorage
+       com.yahoo.vespa.hadoop.pig.VespaStorage();
+
+-- Load data from any source - here we load using JsonLoader
+data_first_release = LOAD 'blog-recommendation/first_release/trainPosts.json' USING 
+  JsonLoader('date_gmt:chararray, 
+              language:chararray, 
+              author:chararray, 
+              url:chararray, 
+              title:chararray, 
+              blog:chararray, 
+              post_id:chararray, 
+              tags:{T:(tag_name:chararray)}, 
+              blogname:chararray, 
+              date:chararray, 
+              content:chararray, 
+              categories:{T:(category_name:chararray)}, 
+              likes:{T:(dt:chararray, uid:chararray)}');
+
+data_second_release = LOAD 'blog-recommendation/second_release/trainPosts.json' USING 
+  JsonLoader('date_gmt:chararray, 
+              language:chararray, 
+              author:chararray, 
+              url:chararray, 
+              title:chararray, 
+              blog:chararray, 
+              post_id:chararray, 
+              tags:{T:(tag_name:chararray)}, 
+              blogname:chararray, 
+              date:chararray, 
+              content:chararray, 
+              categories:{T:(category_name:chararray)}, 
+              likes:{T:(dt:chararray, uid:chararray)}');
+
+data = UNION data_first_release, data_second_release;
+
+-- Select fields that will be sent to Vespa.
+-- This should follow blog_post.sd
+data_for_feed = FOREACH data GENERATE 
+  date_gmt,  
+  language, 
+  author, 
+  url, 
+  title, 
+  blog, 
+  post_id, 
+  tags, 
+  blogname,
+  content, 
+  categories;
+
+-- Create valid Vespa put operations in JSON format
+data_for_feed_json = FOREACH data_for_feed GENERATE VespaPutOperationDoc(*);
+
+-- Sample Vespa operations
+-- data_for_feed_json_sample = SAMPLE data_for_feed_json 0.0005;
+-- STORE data_for_feed_json_sample INTO 'blog-sample';
+
+-- Store into Vespa
+STORE data_for_feed_json INTO '$ENDPOINT' USING VespaStorage();
+\ No newline at end of file
diff --git a/sample-apps/blog-recommendation/src/pig/feed_user_item_cf_vespa.pig b/sample-apps/blog-recommendation/src/pig/feed_user_item_cf_vespa.pig
new file mode 100644
index 00000000000..fd06394c3af
--- /dev/null
+++ b/sample-apps/blog-recommendation/src/pig/feed_user_item_cf_vespa.pig
@@ -0,0 +1,37 @@
+REGISTER vespa-hadoop.jar
+
+-- Create valid Vespa put operations
+DEFINE VespaPutOperationDoc
+       com.yahoo.vespa.hadoop.pig.VespaDocumentOperation(
+            'operation=put',
+            'docid=id:blog-recommendation:blog_post::<post_id>',
+            'create-tensor-fields=user_item_cf'
+       );
+
+DEFINE VespaPutOperationUser
+       com.yahoo.vespa.hadoop.pig.VespaDocumentOperation(
+            'operation=put',
+            'docid=id:blog-recommendation:user::<user_id>',
+            'create-tensor-fields=user_item_cf'
+       );
+
+-- Transform tabular data to a Vespa document operation JSON format
+DEFINE VespaStorage
+       com.yahoo.vespa.hadoop.pig.VespaStorage();
+
+
+data_doc = LOAD 'blog-recommendation/user_item_cf/product_features' USING JsonLoader('post_id:chararray, user_item_cf:[double]');
+data_doc_for_feed = FOREACH data_doc GENERATE VespaPutOperationDoc(*);
+
+
+data_user = LOAD 'blog-recommendation/user_item_cf/user_features' USING JsonLoader('user_id:chararray, user_item_cf:[double]');
+data_user_for_feed = FOREACH data_user GENERATE VespaPutOperationUser(*);
+
+
+-- Store into Vespa
+STORE data_doc_for_feed INTO '$ENDPOINT' USING VespaStorage();
+STORE data_user_for_feed INTO '$ENDPOINT' USING VespaStorage();
+
+
+
+
diff --git a/sample-apps/blog-recommendation/src/pig/generate_user_item_cf_dataset.pig b/sample-apps/blog-recommendation/src/pig/generate_user_item_cf_dataset.pig
new file mode 100644
index 00000000000..2e71dcbe9e3
--- /dev/null
+++ b/sample-apps/blog-recommendation/src/pig/generate_user_item_cf_dataset.pig
@@ -0,0 +1,15 @@
+-- Load data from any source - here we load using PigStorage
+data = LOAD 'blog-recommendation/trainPostsFinal' USING JsonLoader('date_gmt:chararray, language:chararray, author:chararray, url:chararray, title:chararray, blog:chararray, post_id:chararray, tags:{T:(tag_name:chararray)}, blogname:chararray, date:chararray, content:chararray, categories:{T:(category_name:chararray)}, likes:{T:(dt:chararray, uid:chararray)}');
+
+data_likes = FOREACH data GENERATE post_id, FLATTEN(likes) AS (dt, uid);
+
+data_cf = FOREACH data_likes GENERATE uid, post_id, 1 as rate;
+
+data_cf = FILTER data_cf BY (uid IS NOT NULL) AND (uid != '') AND (post_id IS NOT NULL) AND (post_id != ''); 
+
+-- data_cf_sample = SAMPLE data_cf 0.001;
+
+-- data_cf = LIMIT data_cf 10;
+
+STORE data_cf INTO 'blog-recommendation/trainPostsFinal_user_item_cf';
+
diff --git a/sample-apps/blog-recommendation/src/pig/get_recommendations.pig b/sample-apps/blog-recommendation/src/pig/get_recommendations.pig
new file mode 100644
index 00000000000..00b03b0f49a
--- /dev/null
+++ b/sample-apps/blog-recommendation/src/pig/get_recommendations.pig
@@ -0,0 +1,29 @@
+-- REGISTER $VESPA_HADOOP_JAR
+REGISTER vespa-hadoop.jar
+-- REGISTER parquet-pig-bundle-1.8.1.jar
+
+-- Define Vespa query for retrieving blog posts
+DEFINE  BlogPostRecommendations
+        com.yahoo.vespa.hadoop.pig.VespaQuery(
+            'query=http://ENDPOINT:8080/search/?user_id=<userid>&hits=100',
+            'schema=rank:int,id:chararray,relevance:double,fields/post_id:chararray'
+        );
+
+-- Load test_set data from a local file
+test_set = LOAD 'data/cv/test_set_exploded' AS (post_id:chararray, userid:chararray);
+users = FOREACH test_set GENERATE userid;
+users = FILTER users BY userid IS NOT null;
+users = DISTINCT users;
+
+users_limit = LIMIT users 10;
+
+-- Run a set of queries against Vespa
+recommendations = FOREACH users_limit GENERATE userid, 
+                                               FLATTEN(BlogPostRecommendations(*)) AS (rank, id, relevance, post_id);
+recommendations = FOREACH recommendations GENERATE userid, rank, post_id;
+
+recommendations = FILTER recommendations BY rank IS NOT NULL AND post_id IS NOT NULL;                                              
+
+-- Output recommendations
+STORE recommendations INTO 'data/recommendations' USING PigStorage('\t', '-schema');
+-- STORE recommendations INTO 'data/recommendations' USING org.apache.parquet.pig.ParquetStorer();
diff --git a/sample-apps/blog-recommendation/src/pig/tutorial_blog_popularity.pig b/sample-apps/blog-recommendation/src/pig/tutorial_blog_popularity.pig
new file mode 100644
index 00000000000..4dac36a717f
--- /dev/null
+++ b/sample-apps/blog-recommendation/src/pig/tutorial_blog_popularity.pig
@@ -0,0 +1,55 @@
+REGISTER '$VESPA_HADOOP_JAR'
+
+-- UDF to create valid Vespa document operation in JSON format
+DEFINE VespaUpdateOperationDoc
+       com.yahoo.vespa.hadoop.pig.VespaDocumentOperation(
+            'operation=update',
+            'docid=id:blog-search:blog_post::<post_id>'
+       );
+
+-- UDF to send data to a Vespa endpoint
+DEFINE VespaStorage
+       com.yahoo.vespa.hadoop.pig.VespaStorage();
+
+-- Load data from any source - here we load using JsonLoader
+data = LOAD '$DATA_PATH' USING 
+  JsonLoader('date_gmt:chararray, 
+              language:chararray, 
+              author:chararray, 
+              url:chararray, 
+              title:chararray, 
+              blog:chararray, 
+              post_id:chararray, 
+              tags:{T:(tag_name:chararray)}, 
+              blogname:chararray, 
+              date:chararray, 
+              content:chararray, 
+              categories:{T:(category_name:chararray)}, 
+              likes:{T:(dt:chararray, uid:chararray)}');
+
+data = FILTER data BY likes IS NOT NULL;
+
+data_likes = FOREACH data GENERATE 
+  blog, 
+  post_id, 
+  blogname,
+  FLATTEN(likes) AS (dt, uid);
+
+-- data_likes_limit = LIMIT data_likes 10;
+
+likes = FOREACH (GROUP data_likes ALL) 
+     GENERATE COUNT(data_likes) as total_number;
+
+blog_popularity = FOREACH (GROUP data_likes BY blog) GENERATE 
+  group as blog,
+  (double)COUNT(data_likes)/(double)likes.total_number AS popularity;
+
+data_update = JOIN data_likes BY blog, blog_popularity BY blog;
+data_update = FOREACH data_update GENERATE
+  post_id, popularity;
+
+-- Create valid Vespa put operations in JSON format
+data_for_feed_json = FOREACH data_update GENERATE VespaUpdateOperationDoc(*);
+
+-- Store into Vespa
+STORE data_for_feed_json INTO '$ENDPOINT' USING VespaStorage();
diff --git a/sample-apps/blog-recommendation/src/pig/tutorial_feed_content_and_tensor_vespa.pig b/sample-apps/blog-recommendation/src/pig/tutorial_feed_content_and_tensor_vespa.pig
new file mode 100644
index 00000000000..77943fd842a
--- /dev/null
+++ b/sample-apps/blog-recommendation/src/pig/tutorial_feed_content_and_tensor_vespa.pig
@@ -0,0 +1,116 @@
+REGISTER '$VESPA_HADOOP_JAR'
+
+-- Create valid Vespa put operations
+DEFINE VespaPutOperationDoc
+       com.yahoo.vespa.hadoop.pig.VespaDocumentOperation(
+            'operation=put',
+            'docid=id:blog-recommendation:blog_post::<post_id>',
+            'create-tensor-fields=user_item_cf',
+            'simple-array-fields=tags,categories'
+       );
+
+DEFINE VespaPutOperationUser
+       com.yahoo.vespa.hadoop.pig.VespaDocumentOperation(
+            'operation=put',
+            'docid=id:blog-recommendation:user::<user_id>',
+            'create-tensor-fields=user_item_cf',
+            'simple-array-fields=has_read_items'
+       );
+
+-- Transform tabular data to a Vespa document operation JSON format
+DEFINE VespaStorage
+       com.yahoo.vespa.hadoop.pig.VespaStorage();
+
+-- Load data 
+data = LOAD '$DATA_PATH' USING 
+  JsonLoader('date_gmt:chararray, 
+              language:chararray, 
+              author:chararray, 
+              url:chararray, 
+              title:chararray, 
+              blog:chararray, 
+              post_id:chararray, 
+              tags:{T:(tag_name:chararray)}, 
+              blogname:chararray, 
+              date:chararray, 
+              content:chararray, 
+              categories:{T:(category_name:chararray)}, 
+              likes:{T:(dt:chararray, uid:chararray)}');
+
+data_for_feed = FOREACH data GENERATE 
+  date_gmt, 
+  language, 
+  author, 
+  url, 
+  title, 
+  blog, 
+  post_id, 
+  tags, 
+  blogname,
+  content, 
+  categories;
+
+-- Load Blog post CF latent factors
+data_doc = LOAD '$BLOG_POST_FACTORS' USING 
+  JsonLoader('post_id:chararray, 
+              user_item_cf:[double]');
+
+-- Join data and latent factors
+data_content_and_doc_tensor = JOIN data_for_feed BY post_id LEFT, data_doc BY post_id;
+data_content_and_doc_tensor = FOREACH data_content_and_doc_tensor GENERATE
+  date_gmt AS date_gmt, 
+  language AS language, 
+  author AS author, 
+  url AS url, 
+  title AS title, 
+  blog AS blog, 
+  data_for_feed::post_id as post_id, 
+  tags AS tags, 
+  blogname AS blogname,
+  content AS content, 
+  categories AS categories,
+  user_item_cf AS user_item_cf,
+  (user_item_cf IS NOT NULL ? 1 : 0) AS has_user_item_cf;
+
+-- Generate valid Vespa JSON format
+data_content_and_doc_tensor_feed = FOREACH data_content_and_doc_tensor GENERATE VespaPutOperationDoc(*);
+
+-- Load User CF latent factors
+data_user = LOAD '$USER_FACTORS' USING 
+  JsonLoader('user_id:chararray, 
+              user_item_cf:[double]');
+data_user = FOREACH data_user GENERATE  
+  user_id AS user_id,
+  user_item_cf AS user_item_cf;
+  
+-- Articles already liked
+data_likes = FOREACH data GENERATE post_id, FLATTEN(likes) AS (dt, uid); 
+
+post_liked_per_user = GROUP data_likes BY uid;
+post_liked_per_user = FOREACH post_liked_per_user GENERATE 
+  group AS user_id, 
+  data_likes.post_id AS has_read_items;
+
+-- Join user data
+data_user = JOIN post_liked_per_user BY user_id FULL, 
+                 data_user BY user_id;
+
+data_user = FOREACH data_user GENERATE 
+  (post_liked_per_user::user_id IS NOT NULL ? post_liked_per_user::user_id : data_user::user_id) AS user_id,
+  user_item_cf AS user_item_cf,
+  (user_item_cf IS NOT NULL ? 1 : 0) AS has_user_item_cf,
+  has_read_items AS has_read_items;
+
+data_user = FILTER data_user BY user_id IS NOT NULL;
+
+-- Generate valid Vespa JSON format
+data_user_for_feed = FOREACH data_user GENERATE VespaPutOperationUser(*);
+
+joint_content_tensors = UNION data_content_and_doc_tensor_feed, data_user_for_feed;
+
+-- Store into Vespa
+STORE joint_content_tensors INTO '$ENDPOINT' USING VespaStorage();
+
+
+
+
diff --git a/sample-apps/blog-recommendation/src/pig/tutorial_feed_content_vespa.pig b/sample-apps/blog-recommendation/src/pig/tutorial_feed_content_vespa.pig
new file mode 100644
index 00000000000..d20ccf505a9
--- /dev/null
+++ b/sample-apps/blog-recommendation/src/pig/tutorial_feed_content_vespa.pig
@@ -0,0 +1,51 @@
+REGISTER '$VESPA_HADOOP_JAR'
+-- REGISTER vespa-hadoop.jar
+
+-- UDF to create valid Vespa document operation in JSON format
+DEFINE VespaPutOperationDoc
+       com.yahoo.vespa.hadoop.pig.VespaDocumentOperation(
+            'operation=put',
+            'docid=id:blog-search:blog_post::<post_id>',
+            'simple-array-fields=tags,categories'
+       );
+
+-- UDF to send data to a Vespa endpoint
+DEFINE VespaStorage
+       com.yahoo.vespa.hadoop.pig.VespaStorage();
+
+-- Load data from any source - here we load using JsonLoader
+data = LOAD '$DATA_PATH' USING 
+  JsonLoader('date_gmt:chararray, 
+              language:chararray, 
+              author:chararray, 
+              url:chararray, 
+              title:chararray, 
+              blog:chararray, 
+              post_id:chararray, 
+              tags:{T:(tag_name:chararray)}, 
+              blogname:chararray, 
+              date:chararray, 
+              content:chararray, 
+              categories:{T:(category_name:chararray)}, 
+              likes:{T:(dt:chararray, uid:chararray)}');
+
+-- Select fields that will be sent to Vespa.
+-- This should follow blog_post.sd
+data_for_feed = FOREACH data GENERATE 
+  date_gmt, 
+  language, 
+  author, 
+  url, 
+  title, 
+  blog, 
+  post_id, 
+  tags, 
+  blogname,
+  content, 
+  categories;
+
+-- Create valid Vespa put operations in JSON format
+data_for_feed_json = FOREACH data_for_feed GENERATE VespaPutOperationDoc(*);
+
+-- Store into Vespa
+STORE data_for_feed_json INTO '$ENDPOINT' USING VespaStorage();
diff --git a/sample-apps/blog-recommendation/src/spark/collaborative_filtering_example.scala b/sample-apps/blog-recommendation/src/spark/collaborative_filtering_example.scala
new file mode 100644
index 00000000000..1a2c8f92730
--- /dev/null
+++ b/sample-apps/blog-recommendation/src/spark/collaborative_filtering_example.scala
@@ -0,0 +1,59 @@
+import org.apache.spark.mllib.recommendation.ALS
+import org.apache.spark.mllib.recommendation.MatrixFactorizationModel
+import org.apache.spark.mllib.recommendation.Rating
+import scala.util.parsing.json.JSONObject
+
+// Load and parse the data
+val data = sc.textFile("blog-recommendation/trainPostsFinal_user_item_cf")
+val ratings = data.map(_.split('\t') match { case Array(user, item, rate) =>
+  Rating(user.toInt, item.toInt, rate.toDouble)
+})
+
+// Build the recommendation model using ALS
+val rank = 10
+val numIterations = 10
+val model = ALS.train(ratings, rank, numIterations, 0.01)
+
+// Evaluate the model on rating data
+val usersProducts = ratings.map { case Rating(user, product, rate) =>
+  (user, product)
+}
+val predictions =
+  model.predict(usersProducts).map { case Rating(user, product, rate) =>
+    ((user, product), rate)
+  }
+val ratesAndPreds = ratings.map { case Rating(user, product, rate) =>
+  ((user, product), rate)
+}.join(predictions)
+val MSE = ratesAndPreds.map { case ((user, product), (r1, r2)) =>
+  val err = (r1 - r2)
+  err * err
+}.mean()
+println("Mean Squared Error = " + MSE)
+
+def writeModelFeaturesAsTensor (modelFeatures:(Int, Array[Double]), id_string:String) = {
+
+  val id = modelFeatures._1
+  val latentVector = modelFeatures._2
+  var latentVectorMap:Map[String,Double] = Map()
+  var output:Map[String,Any] = Map()
+
+  for ( i <- 0 until latentVector.length ){
+
+    latentVectorMap += (("user_item_cf:" + i.toString, latentVector(i)))      
+
+  }
+
+  output += ((id_string, id))
+  output += (("user_item_cf", scala.util.parsing.json.JSONObject(latentVectorMap)))
+
+  JSONObject(output)
+
+}
+
+val product_features = model.productFeatures.map(x => writeModelFeaturesAsTensor(x, "post_id"))
+product_features.saveAsTextFile("blog-recommendation/user_item_cf/product_features")
+val user_features = model.userFeatures.map(x => writeModelFeaturesAsTensor(x, "user_id"))
+user_features.saveAsTextFile("blog-recommendation/user_item_cf/user_features")
+
+
diff --git a/sample-apps/blog-recommendation/src/spark/data_exploration.scala b/sample-apps/blog-recommendation/src/spark/data_exploration.scala
new file mode 100644
index 00000000000..228834cfb4b
--- /dev/null
+++ b/sample-apps/blog-recommendation/src/spark/data_exploration.scala
@@ -0,0 +1,63 @@
+// sc is an existing SparkContext.
+val sqlContext = new org.apache.spark.sql.SQLContext(sc)
+
+val original_train_post_path = "blog-recommendation-support/data/original_data/trainPosts.json"
+val original_train_post_thin_path = "blog-recommendation-support/data/original_data/trainPostsThin.json"
+val original_test_post_thin_path = "blog-recommendation-support/data/original_data/testPostsThin.json"
+
+val original_train_post = sqlContext.read.json(original_train_post_path)
+val original_train_post_thin = sqlContext.read.json(original_train_post_thin_path)
+val original_test_post_thin = sqlContext.read.json(original_test_post_thin_path)
+
+val count_original_train = original_train_post.count()
+val count_original_train_thin = original_train_post_thin.count()
+val count_original_test_thin = original_test_post_thin.count()
+
+// The inferred schema can be visualized using the printSchema() method.
+original_train_post.printSchema()
+original_train_post_thin.printSchema()
+original_test_post_thin.printSchema()
+
+// No intersection between train and test data
+original_train_post_thin.join(original_test_post_thin, original_train_post_thin("post_id") == original_test_post_thin("post_id")).count(2)
+
+// original_train_minimal_df
+var original_train_minimal_df = original_train_post.select($"date_gmt", $"post_id", size($"likes").as("number_likes"), $"likes")
+// no duplicate post_id
+original_train_minimal_df.select("post_id").dropDuplicates().count() - original_train_minimal_df.select("post_id").count()
+
+// CHECK THIS DECISION - I SHOULD NOT EXLUDE POST_ID WITH ZERO LIKES
+// OTHERWISE THERE WILL BE NO DOCUMENT IN THE TEST SET THAT NO ONE HAS LIKED,
+// WHICH MAKES THE EXERCISE MUCH EASIER
+// only post_id with at least one like
+// original_train_minimal_df = original_train_minimal_df.filter("number_likes > 0")
+
+// Set some post_id aside to be present only on the test set 
+var sets = original_train_minimal_df.randomSplit(Array(0.95, 0.05), 123)
+
+var training_set = sets(0)
+var test_set = sets(1)
+
+// flat dataframe so that each line is a combination of post_id and user
+training_set = training_set.select($"post_id", explode($"likes").as("likes_flat"))
+training_set = training_set.select("post_id", "likes_flat.uid")
+
+test_set = test_set.select($"post_id", explode($"likes").as("likes_flat"))
+test_set = test_set.select("post_id", "likes_flat.uid")
+
+// randomly move some (post_id, uid) from training set to test set
+sets = training_set.randomSplit(Array(0.85, 0.15), 123)
+
+training_set = sets(0)
+var additional_test_set = sets(1)
+
+// concatenate test_set and additional_test_set
+test_set = test_set.unionAll(additional_test_set)
+
+// see number of likes distribution
+val like_dist = original_train_minimal_df.groupBy("number_likes").count().orderBy(asc("number_likes")).collect()
+like_dist.map(println)
+
+
+
+
diff --git a/sample-apps/blog-recommendation/src/spark/expected_percentile.scala b/sample-apps/blog-recommendation/src/spark/expected_percentile.scala
new file mode 100644
index 00000000000..986a3eb79f4
--- /dev/null
+++ b/sample-apps/blog-recommendation/src/spark/expected_percentile.scala
@@ -0,0 +1,39 @@
+val test_file_path = "data/cv/test_set_exploded"
+val blog_recom_file_path = "data/recommendations"
+val size_recommendation_list = 100
+
+val sqlContext = new org.apache.spark.sql.SQLContext(sc)
+
+val test_set = sc.textFile(test_file_path).
+  map(_.split("\t")).map(p => (p(0).toString, p(1).toString)).
+  toDF("post_id", "user_id")
+
+val recommendations = sc.textFile(blog_recom_file_path).
+  map(_.split("\t")).map(p => (p(0).toString, p(1).toString, p(2).toString)).
+  toDF("user_id", "rank", "post_id")
+
+// val recommendations = sqlContext.createDataFrame(Seq(
+//   ("16966742", "5", "1009088"),
+//   ("30463255", "10", "1044974")
+// )).toDF("user_id", "rank", "post_id")
+
+// join data
+var joined_data = test_set.
+  join(recommendations, 
+       test_set("post_id") === recommendations("post_id") && 
+        test_set("user_id") === recommendations("user_id")).
+  select(test_set("post_id"), 
+         test_set("user_id"), 
+         recommendations("rank"))
+
+// transform and add a column
+joined_data = joined_data.withColumn("percentile", joined_data("rank")/size_recommendation_list)
+
+val expected_percentile = joined_data.
+  // groupBy($"user_id").
+  groupBy().
+  agg(sum($"percentile").as("sum_percentile"), 
+      count($"post_id").as("number_read")).
+  withColumn("expected_percentile", $"sum_percentile" / $"number_read")
+
+expected_percentile.show()
+\ No newline at end of file
diff --git a/sample-apps/blog-recommendation/src/spark/full_dataset_cf.scala b/sample-apps/blog-recommendation/src/spark/full_dataset_cf.scala
new file mode 100644
index 00000000000..0b76e8b8b1c
--- /dev/null
+++ b/sample-apps/blog-recommendation/src/spark/full_dataset_cf.scala
@@ -0,0 +1,60 @@
+import org.apache.spark.mllib.recommendation.ALS
+import org.apache.spark.mllib.recommendation.MatrixFactorizationModel
+import org.apache.spark.mllib.recommendation.Rating
+import scala.util.parsing.json.JSONObject
+
+// Prepare data
+
+val data_path = "data/original_data/trainPosts.json"
+
+val sqlContext = new org.apache.spark.sql.SQLContext(sc)
+
+val full_dataset = sqlContext.read.json(data_path)
+
+var data = full_dataset.select($"post_id", explode($"likes").as("likes_flat"))
+data = data.select($"likes_flat.uid".as("uid"), $"post_id")
+
+data = data.filter("uid is not null and uid != '' and post_id is not null and post_id != ''")
+
+val ratings = data.rdd.map(x => (x(0).toString, x(1).toString) match { 
+  case (user, item) =>  Rating(user.toInt, item.toInt, 1)
+})
+
+// Train the model
+
+val rank = 10
+val numIterations = 10
+val model = ALS.train(ratings, rank, numIterations, 0.01)
+
+// Convert latent vectors from model to Vespa Tensor model
+
+def writeModelFeaturesAsTensor (modelFeatures:(Int, Array[Double]), id_string:String) = {
+
+  val id = modelFeatures._1
+  val latentVector = modelFeatures._2
+  var latentVectorMap:Map[String,Double] = Map()
+  var output:Map[String,Any] = Map()
+
+  for ( i <- 0 until latentVector.length ){
+
+    latentVectorMap += (("user_item_cf:" + i.toString, latentVector(i)))      
+
+  }
+
+  output += ((id_string, id))
+  output += (("user_item_cf", scala.util.parsing.json.JSONObject(latentVectorMap)))
+
+  JSONObject(output)
+
+}
+
+// Write user and item latent factors to disk
+
+val product_features = model.productFeatures.map(x => writeModelFeaturesAsTensor(x, "post_id"))
+product_features.saveAsTextFile("data/user_item_cf/product_features")
+val user_features = model.userFeatures.map(x => writeModelFeaturesAsTensor(x, "user_id"))
+user_features.saveAsTextFile("data/user_item_cf/user_features")
+
+
+
+
diff --git a/sample-apps/blog-recommendation/src/spark/train_test_set_division.scala b/sample-apps/blog-recommendation/src/spark/train_test_set_division.scala
new file mode 100644
index 00000000000..2fc67734386
--- /dev/null
+++ b/sample-apps/blog-recommendation/src/spark/train_test_set_division.scala
@@ -0,0 +1,45 @@
+import org.apache.spark.sql.functions.udf
+
+// Inputs
+val input_file_path = "data/original_data/trainPosts.json"
+val test_perc_stage1 = 0.05
+val test_perc_stage2 = 0.15
+val training_file_path = "data/cv/training_set_exploded"
+val test_file_path = "data/cv/test_set_exploded"
+val seed = 123
+
+val sqlContext = new org.apache.spark.sql.SQLContext(sc)
+
+// Load full dataset
+val full_dataset = sqlContext.read.json(input_file_path)
+val full_dataset_simple = full_dataset.select($"post_id", size($"likes").as("number_likes"), $"likes")
+
+// Set some blog posts aside to be present only on the test set 
+var sets = full_dataset_simple.randomSplit(Array(1 - test_perc_stage1, test_perc_stage1), seed)
+
+var training_set = sets(0)
+val training_set_null = training_set.filter("number_likes == 0")
+var training_set_exploded = training_set.select($"post_id", explode($"likes").as("likes_flat"))
+training_set_exploded = training_set_exploded.select("post_id", "likes_flat.uid")
+
+var test_set = sets(1)
+val test_set_null = test_set.filter("number_likes == 0")
+var test_set_exploded = test_set.select($"post_id", explode($"likes").as("likes_flat"))
+test_set_exploded = test_set_exploded.select("post_id", "likes_flat.uid")
+
+// randomly move some (post_id, uid) from training set to test set
+sets = training_set_exploded.randomSplit(Array(1 - test_perc_stage2, test_perc_stage2), seed)
+
+training_set_exploded = sets(0)
+
+var additional_test_set_exploded = sets(1)
+test_set_exploded = test_set_exploded.unionAll(additional_test_set_exploded)
+
+// concatenate exploded set with null set
+val getNull = udf(() => None: Option[String])
+training_set_exploded = training_set_exploded.unionAll(training_set_null.select("post_id").withColumn("uid", getNull()))
+test_set_exploded = test_set_exploded.unionAll(test_set_null.select("post_id").withColumn("uid", getNull()))
+
+// Write to disk
+training_set_exploded.rdd.map(x => x(0) + "\t" + x(1)).saveAsTextFile(training_file_path)
+test_set_exploded.rdd.map(x => x(0) + "\t" + x(1)).saveAsTextFile(test_file_path)
+\ No newline at end of file
diff --git a/sample-apps/blog-recommendation/training_data_example.json b/sample-apps/blog-recommendation/training_data_example.json
new file mode 100644
index 00000000000..b25e4613fbe
--- /dev/null
+++ b/sample-apps/blog-recommendation/training_data_example.json
@@ -0,0 +1,247 @@
+[
+  {
+    "date_gmt": "2012-08-11 09:44:22",
+    "language": "en",
+    "author": "232",
+    "url": "https://joen.wordpress.com/?p=1339",
+    "title": "Selma and me",
+    "blog": "230",
+    "post_id": "522793",
+    "tags": [
+      "selma"
+    ],
+    "blogname": "Lens Cap",
+    "date": "2012-08-11 09:44:22",
+    "content": "<a href=\"http://joen.files.wordpress.com/2012/08/wpid-imag03021.jpg\"><img title=\"\" class=\"alignnone\" alt=\"image\" src=\"http://joen.files.wordpress.com/2012/08/wpid-imag0302.jpg\" /></a>",
+    "categories": [
+      "Photopool"
+    ],
+    "likes": [
+      {
+        "dt": "2012-08-11 14:26:51",
+        "uid": "5"
+      },
+      {
+        "dt": "2012-08-11 14:06:24",
+        "uid": "144"
+      },
+      {
+        "dt": "2012-08-11 14:02:42",
+        "uid": "414033"
+      },
+      {
+        "dt": "2012-08-11 10:56:57",
+        "uid": "6894686"
+      },
+      {
+        "dt": "2012-08-11 14:16:09",
+        "uid": "20416304"
+      }
+    ]
+  },
+  {
+    "date_gmt": "2012-07-17 16:05:35",
+    "language": "en",
+    "author": "322",
+    "url": "http://markjaquith.wordpress.com/?p=3894",
+    "title": "WP Help 1.0",
+    "blog": "316",
+    "post_id": "618834",
+    "tags": [],
+    "blogname": "Mark on WordPress",
+    "date": "2012-07-17 12:05:35",
+    "content": "My <a href=\"http://wordpress.org/extend/plugins/wp-help/\">WP Help</a> plugin just got a huge update. Version 1.0 is really worth checking out. I'm quite proud of it.\r\n\r\n<img src=\"http://markjaquith.files.wordpress.com/2012/07/screen-shot-2012-07-17-at-12-04-21-pm.png\" alt=\"\" title=\"Screen Shot 2012-07-17 at 12.04.21 PM\" width=\"584\" height=\"284\" class=\"aligncenter size-full wp-image-3896\" />\r\n\r\n<h3>WP Help</h3>\r\n\r\nWP Help is a plugin for creating documentation for display within the WordPress admin. Many WordPress installs are customized, and it's really helpful to have a centralized resource for documenting those features. You can create documents about creating content, editing content, moderating comments, or whatever you want! If you have clients who can't seem to remember how to do X, you should install WP Help and document it for them. WP Help is powered by WordPress Custom Post Types, so you create content using the full WordPress editor.\r\n\r\n<h3>Document Syncing</h3>\r\n\r\nOh yeah. This is the feature you've all been waiting for. If you have a standard set of help documents you want to use on multiple sites, this lets you do that. Create the documents, grab the (secret) sync URL for that site, and then plug that URL in to other sites. Those other sites will automatically pull down those documents, and keep them up-to-date (even handling new documents, deleted documents, renamed documents, and re-parented documents). Any internal links in the original document will be rewritten to be local to the destination WP Help install. So go ahead and use the WP internal linking functionality on your source site and know that those links will work on all the destination sites!\r\n\r\n<h3>Menu Placement</h3>\r\n\r\nThe menu item for the help documents can now be placed in one of four locations:\r\n\r\n<ol>\r\n<li>As a Dashboard submenu</li>\r\n<li>Top level, above the Dashboard</li>\r\n<li>Top level, below the Dashboard</li>\r\n<li>Top level, at the bottom</li>\r\n</ol>\r\n\r\nYou get a live preview of this (yeah, I know, super fancy).\r\n\r\n<h3>Menu Name</h3>\r\n\r\nThe menu name (and page title) can be changed. Just doubleclick the page title, edit it, and hit return. Boom. Again, this has a live preview.\r\n\r\n<h3>Topics List Name</h3>\r\n\r\nLikewise the topics list header can be renamed. Doubleclick, edit, return. Live preview.\r\n\r\n<h3>Edit Links &amp; Management Links</h3>\r\n\r\nIf you can edit help documents, you'll get an edit link. When editing, there is a handy \"Manage\" link to jump you back to the documents management interface. Navigating in general has been improved quite a bit.\r\n\r\n<h3>Dashboard Widget</h3>\r\n\r\nThere is now a simple dashboard widget, listing all of your help documents.\r\n\r\n<h3>Better Default Access</h3>\r\n\r\nA lot of you said that you have Contributor-level users who need documentation just like authors do. So now they can view documentation by default (there's a hook if you want to change the capability required to view help documents).\r\n\r\n<h3>Lots of Little Tweaks</h3>\r\n\r\nThere are numerous little tweaks to improve your experience. <a href=\"http://wordpress.org/extend/plugins/wp-help/\">Check out out</a>!\r\n\r\n<h3>Roadmap</h3>\r\n\r\nThings I'm considering:\r\n\r\n<ul>\r\n<li>Restricting individual documents to users with a certain level of access</li>\r\n<li>Multiple sync sources</li>\r\n</ul>\r\n\r\nAny other ideas?",
+    "categories": [
+      "WordPress"
+    ],
+    "likes": [
+      {
+        "dt": "2012-07-17 21:12:13",
+        "uid": "1234"
+      },
+      {
+        "dt": "2012-07-17 16:14:08",
+        "uid": "249119"
+      },
+      {
+        "dt": "2012-07-17 18:52:43",
+        "uid": "5818061"
+      },
+      {
+        "dt": "2012-07-20 18:00:27",
+        "uid": "10773334"
+      },
+      {
+        "dt": "2012-07-18 10:27:04",
+        "uid": "23406451"
+      },
+      {
+        "dt": "2012-07-17 18:50:47",
+        "uid": "37851431"
+      }
+    ]
+  },
+  {
+    "date_gmt": "2012-07-13 20:49:59",
+    "language": "en",
+    "author": "18342963",
+    "url": "http://justin.wordpress.com/?p=1214",
+    "title": "@ Café Meta",
+    "blog": "415",
+    "post_id": "1648262",
+    "tags": [
+      "checkin"
+    ],
+    "blogname": "Justin G. Shreve",
+    "date": "2012-07-13 16:49:59",
+    "content": "<p><a href=\"Café Meta\"><img src=\"https://is0.4sqi.net/pix/Nox6drHMgdWnt_7m8w1HaINMksVAtKXpLiD0Ocou-rg.jpg\"/></a></p>\n<p>Cheese filled pork rolls with mashed potatoes.</p>",
+    "categories": [
+      "foursquare"
+    ],
+    "likes": [
+      {
+        "dt": "2012-07-14 18:05:10",
+        "uid": "10916751"
+      }
+    ]
+  },
+  {
+    "date_gmt": "2012-08-03 23:55:23",
+    "language": "en",
+    "author": "18342963",
+    "url": "http://justin.wordpress.com/?p=1218",
+    "title": "@ El Campesino",
+    "blog": "415",
+    "post_id": "1194797",
+    "tags": [
+      "checkin"
+    ],
+    "blogname": "Justin G. Shreve",
+    "date": "2012-08-03 19:55:23",
+    "content": "<p><a href=\"http://4sq.com/7wXFrq\"><img src=\"https://is1.4sqi.net/pix/AwOlZW7UMMcfWhzioK7YCZIr-f2o1jJ-aTYTpxAuhHo.jpg\"/></a></p>\n<p>Celebrating a family birthday #burritofriday style.</p>",
+    "categories": [
+      "foursquare"
+    ],
+    "likes": []
+  },
+  {
+    "date_gmt": "2012-08-08 14:35:13",
+    "language": "en",
+    "author": "18342963",
+    "url": "http://justin.wordpress.com/?p=1220",
+    "title": "@ Wild Goats Cafe",
+    "blog": "415",
+    "post_id": "285252",
+    "tags": [
+      "checkin"
+    ],
+    "blogname": "Justin G. Shreve",
+    "date": "2012-08-08 10:35:13",
+    "content": "<p><a href=\"http://4sq.com/aDGp5P\"><img src=\"https://is0.4sqi.net/pix/eylAdOrAsEbe3T0W7345n8nRfoiFv6GuYd4xQ4nu4g0.jpg\"/></a></p>\n<p>Italian Omelet\n with Grits</p>",
+    "categories": [
+      "foursquare"
+    ],
+    "likes": []
+  },
+  {
+    "date_gmt": "2012-08-09 00:00:54",
+    "language": "en",
+    "author": "18342963",
+    "url": "http://justin.wordpress.com/?p=1222",
+    "title": "@ Oxford, OH",
+    "blog": "415",
+    "post_id": "250389",
+    "tags": [
+      "checkin"
+    ],
+    "blogname": "Justin G. Shreve",
+    "date": "2012-08-08 20:00:54",
+    "content": "<p><a href=\"http://4sq.com/95WcwY\"><img src=\"https://is1.4sqi.net/pix/M75ofin4EhckhAszHPHLMzhC4mYWELShAnJKI12Re_c.jpg\"/></a></p>\n<p>Welcome to new home meal</p>",
+    "categories": [
+      "foursquare"
+    ],
+    "likes": []
+  },
+  {
+    "date_gmt": "2012-08-09 19:16:40",
+    "language": "en",
+    "author": "18342963",
+    "url": "http://justin.wordpress.com/?p=1226",
+    "title": "@ IKEA",
+    "blog": "415",
+    "post_id": "1590141",
+    "tags": [
+      "checkin"
+    ],
+    "blogname": "Justin G. Shreve",
+    "date": "2012-08-09 15:16:40",
+    "content": "<p><a href=\"http://4sq.com/6IXrIh\"><img src=\"https://is0.4sqi.net/pix/FmrWa9WdnQi8q7LBd5iaI_oqRhBGyEHaxr4k_6qHETg.jpg\"/></a></p>\n<p></p>",
+    "categories": [
+      "foursquare"
+    ],
+    "likes": []
+  },
+  {
+    "date_gmt": "2012-08-11 17:00:36",
+    "language": "en",
+    "author": "18342963",
+    "url": "http://justin.wordpress.com/?p=1228",
+    "title": "@ Oxford, OH",
+    "blog": "415",
+    "post_id": "583639",
+    "tags": [
+      "checkin"
+    ],
+    "blogname": "Justin G. Shreve",
+    "date": "2012-08-11 13:00:36",
+    "content": "<p><a href=\"http://4sq.com/95WcwY\"><img src=\"https://is0.4sqi.net/pix/X6hSjNcr74mn5aWhEQ73M6Yzw2WFTcP9B4Y0bHrFrrw.jpg\"/></a></p>\n<p>Part of the property</p>",
+    "categories": [
+      "foursquare"
+    ],
+    "likes": []
+  },
+  {
+    "date_gmt": "2012-08-11 18:32:48",
+    "language": "en",
+    "author": "18342963",
+    "url": "http://justin.wordpress.com/?p=1230",
+    "title": "@ Phan-Shin",
+    "blog": "415",
+    "post_id": "1451283",
+    "tags": [
+      "checkin"
+    ],
+    "blogname": "Justin G. Shreve",
+    "date": "2012-08-11 14:32:48",
+    "content": "<p><a href=\"http://4sq.com/90TF72\"><img src=\"https://is1.4sqi.net/pix/TMuuSbyftmgRT98vbeMgVCcMSBnsZzg-PNGRP4EqQqY.jpg\"/></a></p>\n<p>Last meal before leaving. General Tso's</p>",
+    "categories": [
+      "foursquare"
+    ],
+    "likes": []
+  },
+  {
+    "date_gmt": "2012-07-12 17:03:49",
+    "language": "en",
+    "author": "486",
+    "url": "http://rick.wordpress.com/?p=1635",
+    "title": "On Baghdad by the Bay, July 12th at 8PM PDT; Interview with Dogcatcher and Wave Array",
+    "blog": "479",
+    "post_id": "1166238",
+    "tags": [
+      "bay area music",
+      "music"
+    ],
+    "blogname": "STET",
+    "date": "2012-07-12 10:03:49",
+    "content": "<strong>Stream the show archive: [audio http://rick.files.wordpress.com/2012/07/on-baghdad-by-the-bay-july-12th-interview-with-dogcatcher-and-wave-array.mp3] <a href=\"http://rick.files.wordpress.com/2012/07/on-baghdad-by-the-bay-july-12th-interview-with-dogcatcher-and-wave-array.mp3\">Or download the MP3</a>.</strong>\r\n\r\n<a href=\"http://dogcatcherband.com/\"><img src=\"http://rick.files.wordpress.com/2012/07/dogcatcher4panel.jpg?w=450\" alt=\"\" title=\"Dogcatcher4panel\" width=\"450\" height=\"193\" class=\"alignnone size-large wp-image-1636\" /></a>\r\n<strong>Dogcatcher</strong> \r\n<a href=\"http://dogcatcherband.com/\">dogcatcherband.com</a>\r\n8pm\r\nThere’s something magical that happens when four musicians come together from four completely different backgrounds. For Mountain View’s Dogcatcher, the result is a mellow, subtly-intense blend of soul, jazz, folk and indie-rock. Their first album, KILR, was a quiet folk-oriented solo album Heine wrote while living in San Diego and digesting the Iraq War he’d recently returned from. After KILR, Heine returned to his hometown, Mountain View, and formed a proper group, plucking musicians out of totally different bands to create an eclectic, genre-defying sound. On Dogcatcher’s second album, It’s Easy, the four members demonstrate that they have truly come into their own. Heine’s soft, hushed voice and simple-stated lyrics compliment the easy, hip-shaking nature of the music. But for listeners looking for lyrics with depth, Heine will not disappoint. Rather than aim for overt expressions of emotion, Heine always goes for the understated, which makes the emotionality behind the music that much more powerful with repeat listens. \r\n\r\n[youtube http://www.youtube.com/watch?v=ZbI_KdDK8-c?rel=0&amp;w=560&amp;h=315]\r\n<strong>Wave Array </strong>\r\n<a href=\"http://wavearraymusic.com/\">wavearraymusic.com</a>\r\n9:20\r\nWave Array is a four-piece rock band hailing from the East Bay/San Francisco, CA that is stretching the limits of alternative/indie rock into the psychedelic and experimental realms. Using vibrant guitar and bass work, driving rhythms, and palpable melodies, the quartet aims to resonate frequencies of human emotion and thought buried beneath the preoccupation of everyday life. Wave Array's eclectic compositions of melodic vamps and hard hitting jams have already sparked the interest of the Bay Area music community as well as local and nation-wide radio. Their self-produced debut album, Cheapjack Moon, is available on iTunes and Amazon.",
+    "categories": [
+      "Radio Valencia"
+    ],
+    "likes": [
+      {
+        "dt": "2012-07-12 18:27:02",
+        "uid": "34944702"
+      }
+    ]
+  }
+]
+\ No newline at end of file
diff --git a/searchcore/src/apps/fdispatch/.gitignore b/searchcore/src/apps/fdispatch/.gitignore
index 7ef8af69759..6cb8d53bdc1 100644
--- a/searchcore/src/apps/fdispatch/.gitignore
+++ b/searchcore/src/apps/fdispatch/.gitignore
@@ -1,2 +1 @@
-/fdispatch
-fdispatch-bin
+vespa-fdispatch-bin
diff --git a/searchcore/src/apps/fdispatch/CMakeLists.txt b/searchcore/src/apps/fdispatch/CMakeLists.txt
index 728f4e46883..63e1e6ca515 100644
--- a/searchcore/src/apps/fdispatch/CMakeLists.txt
+++ b/searchcore/src/apps/fdispatch/CMakeLists.txt
@@ -2,7 +2,7 @@
 vespa_add_executable(searchcore_fdispatch_app
     SOURCES
     fdispatch.cpp
-    OUTPUT_NAME fdispatch-bin
+    OUTPUT_NAME vespa-fdispatch-bin
     INSTALL sbin
     DEPENDS
     searchcore_fdispatch_program
diff --git a/searchcore/src/apps/proton/.gitignore b/searchcore/src/apps/proton/.gitignore
index 4c244afd919..c2cc445d918 100644
--- a/searchcore/src/apps/proton/.gitignore
+++ b/searchcore/src/apps/proton/.gitignore
@@ -1,4 +1,3 @@
 .depend
 Makefile
-proton
-proton-bin
+vespa-proton-bin
diff --git a/searchcore/src/apps/proton/CMakeLists.txt b/searchcore/src/apps/proton/CMakeLists.txt
index 13f04d2d6f4..c5398feed2e 100644
--- a/searchcore/src/apps/proton/CMakeLists.txt
+++ b/searchcore/src/apps/proton/CMakeLists.txt
@@ -3,7 +3,7 @@ vespa_add_executable(searchcore_proton_app
     SOURCES
     downpersistence.cpp
     proton.cpp
-    OUTPUT_NAME proton-bin
+    OUTPUT_NAME vespa-proton-bin
     INSTALL sbin
     DEPENDS
     searchcore_server
diff --git a/searchcore/src/apps/verify_ranksetup/.gitignore b/searchcore/src/apps/verify_ranksetup/.gitignore
index 269b313f1f0..d32b68f0f38 100644
--- a/searchcore/src/apps/verify_ranksetup/.gitignore
+++ b/searchcore/src/apps/verify_ranksetup/.gitignore
@@ -1,4 +1,3 @@
 .depend
 Makefile
-verify_ranksetup
-verify_ranksetup-bin
+vespa-verify-ranksetup-bin
diff --git a/searchcore/src/apps/verify_ranksetup/CMakeLists.txt b/searchcore/src/apps/verify_ranksetup/CMakeLists.txt
index de777f88182..7dd6d0490da 100644
--- a/searchcore/src/apps/verify_ranksetup/CMakeLists.txt
+++ b/searchcore/src/apps/verify_ranksetup/CMakeLists.txt
@@ -2,7 +2,7 @@
 vespa_add_executable(searchcore_verify_ranksetup_app
     SOURCES
     verify_ranksetup.cpp
-    OUTPUT_NAME verify_ranksetup-bin
+    OUTPUT_NAME vespa-verify-ranksetup-bin
     INSTALL bin
     DEPENDS
     searchcore_fconfig
diff --git a/searchcore/src/apps/vespa-proton-cmd/vespa-proton-cmd.cpp b/searchcore/src/apps/vespa-proton-cmd/vespa-proton-cmd.cpp
index bfbe27cb26c..58844dc969c 100644
--- a/searchcore/src/apps/vespa-proton-cmd/vespa-proton-cmd.cpp
+++ b/searchcore/src/apps/vespa-proton-cmd/vespa-proton-cmd.cpp
@@ -5,6 +5,7 @@
 #include <vespa/config/common/configsystem.h>
 #include <vespa/fnet/frt/frt.h>
 #include <vespa/vespalib/util/host_name.h>
+#include <vespa/vespalib/util/stringfmt.h>
 #include <vespa/fastos/app.h>
 #include <algorithm>
 #include <string>
diff --git a/searchcore/src/tests/proton/common/cachedselect_test.cpp b/searchcore/src/tests/proton/common/cachedselect_test.cpp
index fb5e40767ab..1117234f18f 100644
--- a/searchcore/src/tests/proton/common/cachedselect_test.cpp
+++ b/searchcore/src/tests/proton/common/cachedselect_test.cpp
@@ -23,6 +23,7 @@
 #include <vespa/document/fieldvalue/stringfieldvalue.h>
 #include <vespa/document/fieldvalue/intfieldvalue.h>
 #include <vespa/document/fieldvalue/document.h>
+#include <vespa/document/datatype/documenttype.h>
 
 #include <vespa/log/log.h>
 LOG_SETUP("cachedselect_test");
diff --git a/searchcore/src/tests/proton/common/selectpruner_test.cpp b/searchcore/src/tests/proton/common/selectpruner_test.cpp
index 4ecefde024c..29ef6ef016c 100644
--- a/searchcore/src/tests/proton/common/selectpruner_test.cpp
+++ b/searchcore/src/tests/proton/common/selectpruner_test.cpp
@@ -3,6 +3,7 @@
 #include <vespa/vespalib/testkit/testapp.h>
 #include <vespa/document/repo/configbuilder.h>
 #include <vespa/document/repo/documenttyperepo.h>
+#include <vespa/document/datatype/documenttype.h>
 #include <vespa/searchcore/proton/common/selectpruner.h>
 #include <vespa/document/select/parser.h>
 #include <vespa/document/select/cloningvisitor.h>
diff --git a/searchcore/src/tests/proton/docsummary/docsummary.cpp b/searchcore/src/tests/proton/docsummary/docsummary.cpp
index 5567eb970bb..6c36b004133 100644
--- a/searchcore/src/tests/proton/docsummary/docsummary.cpp
+++ b/searchcore/src/tests/proton/docsummary/docsummary.cpp
@@ -952,13 +952,13 @@ Test::requireThatUrisAreUsed()
                        startIndexField("urisingle").
                        startSubField("all").
                        addUrlTokenizedString(
-                               "http://www.yahoo.com:81/fluke?ab=2#4").
+                               "http://www.example.com:81/fluke?ab=2#4").
                        endSubField().
                        startSubField("scheme").
                        addUrlTokenizedString("http").
                        endSubField().
                        startSubField("host").
-                       addUrlTokenizedString("www.yahoo.com").
+                       addUrlTokenizedString("www.example.com").
                        endSubField().
                        startSubField("port").
                        addUrlTokenizedString("81").
@@ -977,13 +977,13 @@ Test::requireThatUrisAreUsed()
                        startElement(1).
                        startSubField("all").
                        addUrlTokenizedString(
-                               "http://www.yahoo.com:82/fluke?ab=2#8").
+                               "http://www.example.com:82/fluke?ab=2#8").
                        endSubField().
                        startSubField("scheme").
                        addUrlTokenizedString("http").
                        endSubField().
                        startSubField("host").
-                       addUrlTokenizedString("www.yahoo.com").
+                       addUrlTokenizedString("www.example.com").
                        endSubField().
                        startSubField("port").
                        addUrlTokenizedString("82").
@@ -1027,13 +1027,13 @@ Test::requireThatUrisAreUsed()
                        startElement(4).
                        startSubField("all").
                        addUrlTokenizedString(
-                               "http://www.yahoo.com:83/fluke?ab=2#12").
+                               "http://www.example.com:83/fluke?ab=2#12").
                        endSubField().
                        startSubField("scheme").
                        addUrlTokenizedString("http").
                        endSubField().
                        startSubField("host").
-                       addUrlTokenizedString("www.yahoo.com").
+                       addUrlTokenizedString("www.example.com").
                        endSubField().
                        startSubField("port").
                        addUrlTokenizedString("83").
@@ -1086,14 +1086,14 @@ Test::requireThatUrisAreUsed()
                              bc.createFieldCacheRepo(getResultConfig())->getFieldCache("class0"),
                              getMarkupFields());
 
-    EXPECT_TRUE(assertString("http://www.yahoo.com:81/fluke?ab=2#4",
+    EXPECT_TRUE(assertString("http://www.example.com:81/fluke?ab=2#4",
                             "urisingle", dsa, 1));
     GeneralResultPtr res = getResult(dsa, 1);
     {
         vespalib::Slime slime;
         decode(res->GetEntry("uriarray"), slime);
         EXPECT_TRUE(slime.get().valid());
-        EXPECT_EQUAL("http://www.yahoo.com:82/fluke?ab=2#8",  asVstring(slime.get()[0]));
+        EXPECT_EQUAL("http://www.example.com:82/fluke?ab=2#8",  asVstring(slime.get()[0]));
         EXPECT_EQUAL("http://www.flickr.com:82/fluke?ab=2#9", asVstring(slime.get()[1]));
     }
     {
@@ -1104,7 +1104,7 @@ Test::requireThatUrisAreUsed()
         EXPECT_EQUAL(7L, slime.get()[1]["weight"].asLong());
         vespalib::string arr0s = asVstring(slime.get()[0]["item"]);
         vespalib::string arr1s = asVstring(slime.get()[1]["item"]);
-        EXPECT_EQUAL("http://www.yahoo.com:83/fluke?ab=2#12", arr0s);
+        EXPECT_EQUAL("http://www.example.com:83/fluke?ab=2#12", arr0s);
         EXPECT_EQUAL("http://www.flickr.com:85/fluke?ab=2#13", arr1s);
     }
 }
diff --git a/searchcore/src/tests/proton/docsummary/summaryfieldconverter_test.cpp b/searchcore/src/tests/proton/docsummary/summaryfieldconverter_test.cpp
index 878833fa58b..b9b74527dca 100644
--- a/searchcore/src/tests/proton/docsummary/summaryfieldconverter_test.cpp
+++ b/searchcore/src/tests/proton/docsummary/summaryfieldconverter_test.cpp
@@ -369,9 +369,9 @@ Document Test::makeDocument() {
     doc.setValue("position2", LongFieldValue(ZCurve::encode(4, 2)));
 
     StructFieldValue uri(getDataType("url"));
-    uri.setValue("all", annotateTerm("http://www.yahoo.com:42/foobar?q#frag"));
+    uri.setValue("all", annotateTerm("http://www.example.com:42/foobar?q#frag"));
     uri.setValue("scheme", annotateTerm("http"));
-    uri.setValue("host", annotateTerm("www.yahoo.com"));
+    uri.setValue("host", annotateTerm("www.example.com"));
     uri.setValue("port", annotateTerm("42"));
     uri.setValue("path", annotateTerm("foobar"));
     uri.setValue("query", annotateTerm("q"));
@@ -379,10 +379,10 @@ Document Test::makeDocument() {
     doc.setValue("uri", uri);
 
     ArrayFieldValue uri_array(getDataType("Array<url>"));
-    uri.setValue("all", annotateTerm("http://www.yahoo.com:80/foobar?q#frag"));
+    uri.setValue("all", annotateTerm("http://www.example.com:80/foobar?q#frag"));
     uri.setValue("port", annotateTerm("80"));
     uri_array.add(uri);
-    uri.setValue("all", annotateTerm("https://www.yahoo.com:443/foo?q#frag"));
+    uri.setValue("all", annotateTerm("https://www.example.com:443/foo?q#frag"));
     uri.setValue("scheme", annotateTerm("https"));
     uri.setValue("path", annotateTerm("foo"));
     uri.setValue("port", annotateTerm("443"));
diff --git a/searchcore/src/tests/proton/document_iterator/document_iterator_test.cpp b/searchcore/src/tests/proton/document_iterator/document_iterator_test.cpp
index 0ccd5b8d6ee..8261ae80e02 100644
--- a/searchcore/src/tests/proton/document_iterator/document_iterator_test.cpp
+++ b/searchcore/src/tests/proton/document_iterator/document_iterator_test.cpp
@@ -2,6 +2,7 @@
 
 #include <vespa/document/fieldset/fieldsets.h>
 #include <vespa/document/fieldvalue/fieldvalues.h>
+#include <vespa/document/datatype/documenttype.h>
 #include <vespa/persistence/spi/bucket.h>
 #include <vespa/persistence/spi/docentry.h>
 #include <vespa/persistence/spi/result.h>
diff --git a/searchcore/src/tests/proton/documentdb/document_scan_iterator/document_scan_iterator_test.cpp b/searchcore/src/tests/proton/documentdb/document_scan_iterator/document_scan_iterator_test.cpp
index 6a2c283dc8d..db95898e349 100644
--- a/searchcore/src/tests/proton/documentdb/document_scan_iterator/document_scan_iterator_test.cpp
+++ b/searchcore/src/tests/proton/documentdb/document_scan_iterator/document_scan_iterator_test.cpp
@@ -4,6 +4,7 @@
 #include <vespa/vespalib/test/insertion_operators.h>
 #include <vespa/vespalib/testkit/testapp.h>
 #include <vespa/document/base/documentid.h>
+#include <vespa/vespalib/util/stringfmt.h>
 
 using namespace document;
 using namespace proton;
diff --git a/searchcore/src/tests/proton/documentdb/documentdb_test.cpp b/searchcore/src/tests/proton/documentdb/documentdb_test.cpp
index 5b550f9c49d..d757e48e7d0 100644
--- a/searchcore/src/tests/proton/documentdb/documentdb_test.cpp
+++ b/searchcore/src/tests/proton/documentdb/documentdb_test.cpp
@@ -1,10 +1,6 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include <tests/proton/common/dummydbowner.h>
-#include <vespa/document/repo/documenttyperepo.h>
-#include <vespa/messagebus/emptyreply.h>
-#include <vespa/messagebus/testlib/receptor.h>
-#include <vespa/searchcommon/common/schema.h>
 #include <vespa/searchcore/proton/attribute/flushableattribute.h>
 #include <vespa/searchcore/proton/common/feedtoken.h>
 #include <vespa/searchcore/proton/common/hw_info.h>
@@ -26,6 +22,9 @@
 #include <vespa/searchcorespi/index/indexflushtarget.h>
 #include <vespa/searchlib/index/dummyfileheadercontext.h>
 #include <vespa/searchlib/transactionlog/translogserver.h>
+#include <vespa/messagebus/emptyreply.h>
+#include <vespa/messagebus/testlib/receptor.h>
+#include <vespa/document/datatype/documenttype.h>
 #include <vespa/vespalib/data/slime/slime.h>
 #include <vespa/vespalib/testkit/test_kit.h>
 
diff --git a/searchcore/src/tests/proton/feedoperation/feedoperation_test.cpp b/searchcore/src/tests/proton/feedoperation/feedoperation_test.cpp
index 6077ca5fe76..1c8165d9f17 100644
--- a/searchcore/src/tests/proton/feedoperation/feedoperation_test.cpp
+++ b/searchcore/src/tests/proton/feedoperation/feedoperation_test.cpp
@@ -25,6 +25,7 @@
 #include <vespa/document/fieldvalue/fieldvalues.h>
 #include <vespa/document/repo/configbuilder.h>
 #include <vespa/document/repo/documenttyperepo.h>
+#include <vespa/document/datatype/documenttype.h>
 #include <vespa/vespalib/testkit/testapp.h>
 
 using document::BucketId;
diff --git a/searchcore/src/tests/proton/metrics/documentdb_job_trackers/documentdb_job_trackers_test.cpp b/searchcore/src/tests/proton/metrics/documentdb_job_trackers/documentdb_job_trackers_test.cpp
index 45931307c0e..816e214aa0c 100644
--- a/searchcore/src/tests/proton/metrics/documentdb_job_trackers/documentdb_job_trackers_test.cpp
+++ b/searchcore/src/tests/proton/metrics/documentdb_job_trackers/documentdb_job_trackers_test.cpp
@@ -1,12 +1,13 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-#include <vespa/log/log.h>
-LOG_SETUP("documentdb_job_trackers_test");
 
 #include <vespa/searchcore/proton/metrics/documentdb_job_trackers.h>
 #include <vespa/searchcore/proton/metrics/job_tracked_flush_target.h>
 #include <vespa/searchcore/proton/test/dummy_flush_target.h>
 #include <vespa/vespalib/testkit/testapp.h>
 
+#include <vespa/log/log.h>
+LOG_SETUP("documentdb_job_trackers_test");
+
 using namespace proton;
 using namespace searchcorespi;
 
@@ -61,7 +62,7 @@ TEST_F("require that job metrics are updated", Fixture)
     // Update metrics 2 times to ensure that all jobs are running
     // in the last interval we actually care about.
     f._trackers.updateMetrics(f._metrics);
-    FastOS_Thread::Sleep(100);
+    std::this_thread::sleep_for(std::chrono::milliseconds(100));
     f._trackers.updateMetrics(f._metrics);
 
     EXPECT_APPROX(1.0, f._metrics.attributeFlush.getLast(), EPS);
diff --git a/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp b/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp
index 19843cb9531..8dea47e850f 100644
--- a/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp
+++ b/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp
@@ -1,5 +1,6 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 #include <vespa/document/repo/documenttyperepo.h>
+#include <vespa/document/datatype/documenttype.h>
 #include <vespa/persistence/spi/documentselection.h>
 #include <vespa/searchcore/proton/persistenceengine/bucket_guard.h>
 #include <vespa/searchcore/proton/persistenceengine/ipersistenceengineowner.h>
diff --git a/searchcore/src/tests/proton/server/documentretriever_test.cpp b/searchcore/src/tests/proton/server/documentretriever_test.cpp
index bd2fe12e894..b0541f9ac71 100644
--- a/searchcore/src/tests/proton/server/documentretriever_test.cpp
+++ b/searchcore/src/tests/proton/server/documentretriever_test.cpp
@@ -5,6 +5,7 @@
 #include <vespa/document/bucket/bucketid.h>
 #include <vespa/document/datatype/datatype.h>
 #include <vespa/document/datatype/positiondatatype.h>
+#include <vespa/document/datatype/documenttype.h>
 #include <vespa/document/fieldvalue/arrayfieldvalue.h>
 #include <vespa/document/fieldvalue/document.h>
 #include <vespa/document/fieldvalue/doublefieldvalue.h>
@@ -29,11 +30,7 @@
 #include <vespa/searchlib/attribute/integerbase.h>
 #include <vespa/searchlib/attribute/predicate_attribute.h>
 #include <vespa/searchlib/attribute/stringbase.h>
-#include <vespa/searchlib/docstore/cachestats.h>
-#include <vespa/searchlib/docstore/idocumentstore.h>
-#include <vespa/vespalib/stllike/string.h>
 #include <vespa/vespalib/testkit/testapp.h>
-#include <vespa/vespalib/util/stringfmt.h>
 
 using document::ArrayFieldValue;
 using document::FieldValue;
diff --git a/searchcore/src/tests/proton/verify_ranksetup/verify_ranksetup_test.cpp b/searchcore/src/tests/proton/verify_ranksetup/verify_ranksetup_test.cpp
index 2c5e119ecbe..8439c2fb54f 100644
--- a/searchcore/src/tests/proton/verify_ranksetup/verify_ranksetup_test.cpp
+++ b/searchcore/src/tests/proton/verify_ranksetup/verify_ranksetup_test.cpp
@@ -9,7 +9,7 @@
 #include <map>
 #include <initializer_list>
 
-const char *prog = "../../../apps/verify_ranksetup/verify_ranksetup-bin";
+const char *prog = "../../../apps/verify_ranksetup/vespa-verify-ranksetup-bin";
 const std::string gen_dir("generated");
 
 const char *valid_feature = "value(0)";
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp
index 60cc292cbeb..a357fb92980 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp
@@ -5,6 +5,7 @@
 #include <vespa/searchcore/proton/common/attrupdate.h>
 #include <vespa/searchlib/attribute/attributevector.hpp>
 #include <vespa/searchlib/common/isequencedtaskexecutor.h>
+#include <vespa/document/datatype/documenttype.h>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".proton.server.attributeadapter");
diff --git a/searchcore/src/vespa/searchcore/proton/common/selectpruner.cpp b/searchcore/src/vespa/searchcore/proton/common/selectpruner.cpp
index c166dfbe634..001eb7d16f3 100644
--- a/searchcore/src/vespa/searchcore/proton/common/selectpruner.cpp
+++ b/searchcore/src/vespa/searchcore/proton/common/selectpruner.cpp
@@ -2,15 +2,13 @@
 
 #include "selectpruner.h"
 #include <vespa/document/base/exceptions.h>
-#include <vespa/document/base/fieldpath.h>
+#include <vespa/document/datatype/documenttype.h>
 #include <vespa/document/repo/documenttyperepo.h>
 #include <vespa/document/select/compare.h>
-#include <vespa/document/select/operator.h>
 #include <vespa/document/select/constant.h>
 #include <vespa/document/select/branch.h>
 #include <vespa/document/select/doctype.h>
 #include <vespa/document/select/invalidconstant.h>
-#include <vespa/document/select/constant.h>
 #include <vespa/searchlib/attribute/iattributemanager.h>
 
 using document::select::And;
diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/moveoperation.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/moveoperation.cpp
index 2a369cb821b..2cbfee1d5c9 100644
--- a/searchcore/src/vespa/searchcore/proton/feedoperation/moveoperation.cpp
+++ b/searchcore/src/vespa/searchcore/proton/feedoperation/moveoperation.cpp
@@ -1,8 +1,8 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "moveoperation.h"
-#include <vespa/vespalib/util/stringfmt.h>
 #include <vespa/document/fieldvalue/document.h>
+#include <cassert>
 
 using document::BucketId;
 using document::Document;
diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.cpp b/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.cpp
index 9df67a6bf11..79629410b00 100644
--- a/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.cpp
+++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.cpp
@@ -4,6 +4,7 @@
 #include <vespa/document/select/gid_filter.h>
 #include <vespa/document/select/node.h>
 #include <vespa/vespalib/objects/nbostream.h>
+#include <vespa/vespalib/stllike/hash_map.h>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".proton.persistenceengine.document_iterator");
diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/transport_latch.cpp b/searchcore/src/vespa/searchcore/proton/persistenceengine/transport_latch.cpp
index 78eb1cfb3f6..94bac320ef3 100644
--- a/searchcore/src/vespa/searchcore/proton/persistenceengine/transport_latch.cpp
+++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/transport_latch.cpp
@@ -1,6 +1,7 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "transport_latch.h"
+#include <vespa/vespalib/util/stringfmt.h>
 
 using storage::spi::Result;
 
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentretriever.cpp b/searchcore/src/vespa/searchcore/proton/server/documentretriever.cpp
index a3566de66b9..e40d6480cc3 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentretriever.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/documentretriever.cpp
@@ -2,6 +2,7 @@
 
 #include "documentretriever.h"
 #include <vespa/document/datatype/positiondatatype.h>
+#include <vespa/document/datatype/documenttype.h>
 #include <vespa/document/repo/documenttyperepo.h>
 #include <vespa/searchcore/proton/attribute/document_field_retriever.h>
 #include <vespa/vespalib/geo/zcurve.h>
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentretrieverbase.cpp b/searchcore/src/vespa/searchcore/proton/server/documentretrieverbase.cpp
index 07a04905beb..63cfded1962 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentretrieverbase.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/documentretrieverbase.cpp
@@ -2,6 +2,7 @@
 
 #include "documentretrieverbase.h"
 #include <vespa/document/repo/documenttyperepo.h>
+#include <vespa/document/datatype/documenttype.h>
 #include <vespa/vespalib/stllike/lrucache_map.hpp>
 
 using document::DocumentId;
diff --git a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp
index 2f36f5140df..aa1541d766e 100644
--- a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp
@@ -12,15 +12,13 @@
 #include <vespa/documentapi/messagebus/messages/feedreply.h>
 #include <vespa/documentapi/messagebus/messages/removedocumentreply.h>
 #include <vespa/documentapi/messagebus/messages/updatedocumentreply.h>
-#include <vespa/searchcore/proton/common/bucketfactory.h>
-#include <vespa/searchcore/proton/feedoperation/moveoperation.h>
-#include <vespa/searchcore/proton/feedoperation/operations.h>
 #include <vespa/searchcore/proton/persistenceengine/transport_latch.h>
 #include <vespa/searchcore/proton/bucketdb/ibucketdbhandler.h>
 #include <vespa/searchcorespi/index/ithreadingservice.h>
 #include <vespa/vespalib/util/closuretask.h>
 #include <vespa/searchcore/proton/persistenceengine/i_resource_write_filter.h>
 #include <vespa/vespalib/util/exceptions.h>
+#include <vespa/document/datatype/documenttype.h>
 #include <vespa/log/log.h>
 LOG_SETUP(".proton.server.feedhandler");
 
diff --git a/searchcore/src/vespa/searchcore/proton/server/matchview.cpp b/searchcore/src/vespa/searchcore/proton/server/matchview.cpp
index 86ee2fe2ac0..c567930d68d 100644
--- a/searchcore/src/vespa/searchcore/proton/server/matchview.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/matchview.cpp
@@ -4,6 +4,7 @@
 #include "searchcontext.h"
 #include <vespa/searchlib/engine/searchrequest.h>
 #include <vespa/searchlib/engine/searchreply.h>
+#include <vespa/vespalib/util/stringfmt.h>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".proton.server.matchview");
diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp
index be8df97b683..271c127bcee 100644
--- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp
@@ -21,6 +21,7 @@
 #include <vespa/searchlib/util/fileheadertk.h>
 #include <vespa/searchcommon/common/schemaconfigurer.h>
 #include <vespa/document/base/exceptions.h>
+#include <vespa/document/datatype/documenttype.h>
 #include <vespa/vespalib/io/fileutil.h>
 #include <vespa/vespalib/util/closuretask.h>
 #include <vespa/vespalib/util/host_name.h>
diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp
index ac24469e4d2..e27a710022f 100644
--- a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp
@@ -2,23 +2,21 @@
 
 #include "ireplayconfig.h"
 #include "storeonlyfeedview.h"
-#include <vespa/searchcore/proton/common/bucketfactory.h>
-#include <vespa/searchcore/proton/common/commit_time_tracker.h>
-#include <vespa/searchcore/proton/common/feedtoken.h>
-#include <vespa/searchcore/proton/metrics/feed_metrics.h>
-#include <vespa/searchcore/proton/documentmetastore/ilidreusedelayer.h>
-#include <vespa/vespalib/stllike/string.h>
-#include <vespa/vespalib/text/stringtokenizer.h>
-#include <vespa/vespalib/util/closuretask.h>
-#include <vespa/searchcore/proton/feedoperation/moveoperation.h>
 #include "forcecommitcontext.h"
 #include "operationdonecontext.h"
 #include "removedonecontext.h"
 #include "updatedonecontext.h"
 #include "putdonecontext.h"
-#include <vespa/searchlib/common/lambdatask.h>
+#include <vespa/searchcore/proton/common/commit_time_tracker.h>
+#include <vespa/searchcore/proton/common/feedtoken.h>
+#include <vespa/searchcore/proton/metrics/feed_metrics.h>
+#include <vespa/searchcore/proton/documentmetastore/ilidreusedelayer.h>
 #include <vespa/searchlib/common/scheduletaskcallback.h>
+#include <vespa/document/datatype/documenttype.h>
 #include <vespa/vespalib/util/exceptions.h>
+#include <vespa/vespalib/text/stringtokenizer.h>
+#include <vespa/vespalib/util/closuretask.h>
+
 #include <vespa/log/log.h>
 LOG_SETUP(".proton.server.storeonlyfeedview");
 
diff --git a/searchcore/src/vespa/searchcore/proton/test/bucketdocuments.h b/searchcore/src/vespa/searchcore/proton/test/bucketdocuments.h
index b2fc3ff6dbc..f0d10d9ecbd 100644
--- a/searchcore/src/vespa/searchcore/proton/test/bucketdocuments.h
+++ b/searchcore/src/vespa/searchcore/proton/test/bucketdocuments.h
@@ -2,6 +2,7 @@
 #pragma once
 
 #include "document.h"
+#include <cassert>
 
 namespace proton {
 
diff --git a/searchlib/CMakeLists.txt b/searchlib/CMakeLists.txt
index c15ed681d8f..7a340e7800e 100644
--- a/searchlib/CMakeLists.txt
+++ b/searchlib/CMakeLists.txt
@@ -58,11 +58,10 @@ vespa_define_module(
 
     APPS
     src/apps/docstore
-    src/apps/expgolomb
     src/apps/fileheaderinspect
-    src/apps/loadattribute
     src/apps/tests
     src/apps/uniform
+    src/apps/vespa-attribute-inspect
     src/apps/vespa-index-inspect
     src/apps/vespa-ranking-expression-analyzer
 
diff --git a/searchlib/src/apps/expgolomb/.gitignore b/searchlib/src/apps/expgolomb/.gitignore
deleted file mode 100644
index 0886ab154a2..00000000000
--- a/searchlib/src/apps/expgolomb/.gitignore
+++ /dev/null
@@ -1,3 +0,0 @@
-.depend
-Makefile
-expgolomb
diff --git a/searchlib/src/apps/expgolomb/CMakeLists.txt b/searchlib/src/apps/expgolomb/CMakeLists.txt
deleted file mode 100644
index 230718907dd..00000000000
--- a/searchlib/src/apps/expgolomb/CMakeLists.txt
+++ /dev/null
@@ -1,9 +0,0 @@
-# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-vespa_add_executable(searchlib_expgolomb_app
-    SOURCES
-    expgolomb.cpp
-    OUTPUT_NAME expgolomb
-    INSTALL bin
-    DEPENDS
-    searchlib
-)
diff --git a/searchlib/src/apps/expgolomb/expgolomb.cpp b/searchlib/src/apps/expgolomb/expgolomb.cpp
deleted file mode 100644
index 81aa566305c..00000000000
--- a/searchlib/src/apps/expgolomb/expgolomb.cpp
+++ /dev/null
@@ -1,168 +0,0 @@
-// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include <vespa/fastos/app.h>
-#include <vespa/searchlib/bitcompression/compression.h>
-#include <vector>
-
-class ExpGolombApp : public FastOS_Application
-{
-    void usage();
-    int testExpGolomb64(int kValue);
-    int testExpGolomb64le(int kValue);
-    int Main() override;
-};
-
-
-
-void
-ExpGolombApp::usage()
-{
-    printf("Usage: expgolomb testeg64 <kValue>]\n");
-    fflush(stdout);
-}
-
-
-int
-ExpGolombApp::testExpGolomb64(int kValue)
-{
-    std::vector<uint64_t> myrand;
-    for (int i = 0; i < 10000; ++i) {
-        uint64_t rval = rand();
-        rval <<= 30;
-        rval |= rand();
-        myrand.push_back(rval);
-    }
-    for (int i = 0; i < 10000; ++i) {
-        uint64_t rval = rand();
-        rval <<= 30;
-        rval |= rand();
-        uint32_t bits = (rand() & 63);
-        rval &= ((UINT64_C(1) << bits) - 1);
-        myrand.push_back(rval);
-    }
-    typedef search::bitcompression::EncodeContext64BE EC;
-
-    EC e;
-    search::ComprFileWriteContext wc(e);
-    wc.allocComprBuf(32768, 32768);
-    e.setupWrite(wc);
-
-    int rsize = myrand.size();
-    for (int i = 0; i < rsize; ++i) {
-        e.encodeExpGolomb(myrand[i], kValue);
-        if (e._valI >= e._valE)
-            wc.writeComprBuffer(false);
-    }
-    e.flush();
-
-    UC64_DECODECONTEXT(o);
-    unsigned int length;
-    uint64_t val64;
-    UC64BE_SETUPBITS_NS(o, static_cast<const uint64_t *>(wc._comprBuf), 0, EC);
-
-    bool failure = false;
-    for (int i = 0; i < rsize; ++i) {
-        UC64BE_DECODEEXPGOLOMB(oVal, oCompr, oPreRead, oCacheInt,
-                               kValue, EC);
-        if (val64 != myrand[i]) {
-            printf("FAILURE: TestExpGolomb64, val64=%"
-                   PRIu64 ", myrand[%d]=%" PRIu64 "\n",
-                   val64, i, myrand[i]);
-            failure = true;
-        }
-    }
-    if (!failure)
-        printf("SUCCESS: TestExpGolomb64\n");
-    return failure ? 1 : 0;
-}
-
-int
-ExpGolombApp::testExpGolomb64le(int kValue)
-{
-    std::vector<uint64_t> myrand;
-    for (int i = 0; i < 10000; ++i) {
-        uint64_t rval = rand();
-        rval <<= 30;
-        rval |= rand();
-        myrand.push_back(rval);
-    }
-    for (int i = 0; i < 10000; ++i) {
-        uint64_t rval = rand();
-        rval <<= 30;
-        rval |= rand();
-        uint32_t bits = (rand() & 63);
-        rval &= ((UINT64_C(1) << bits) - 1);
-        myrand.push_back(rval);
-    }
-    typedef search::bitcompression::EncodeContext64LE EC;
-
-    EC e;
-    search::ComprFileWriteContext wc(e);
-    wc.allocComprBuf(32768, 32768);
-    e.setupWrite(wc);
-
-    int rsize = myrand.size();
-    for (int i = 0; i < rsize; ++i) {
-        e.encodeExpGolomb(myrand[i], kValue);
-        if (e._valI >= e._valE)
-            wc.writeComprBuffer(false);
-    }
-    e.flush();
-
-    UC64_DECODECONTEXT(o);
-    unsigned int length;
-    uint64_t val64;
-    UC64LE_SETUPBITS_NS(o, static_cast<const uint64_t *>(wc._comprBuf), 0, EC);
-
-    bool failure = false;
-    for (int i = 0; i < rsize; ++i) {
-        UC64LE_DECODEEXPGOLOMB(oVal, oCompr, oPreRead, oCacheInt,
-                               kValue, EC);
-        if (val64 != myrand[i]) {
-            printf("FAILURE: TestExpGolomb64le, val64=%"
-                   PRIu64 ", myrand[%d]=%" PRIu64 "\n",
-                   val64, i, myrand[i]);
-            failure = true;
-        }
-    }
-    if (!failure)
-        printf("SUCCESS: TestExpGolomb64le\n");
-    return failure ? 1 : 0;
-}
-
-
-int
-ExpGolombApp::Main()
-{
-    printf("Hello world\n");
-    if (_argc >= 2) {
-        if (strcmp(_argv[1], "testeg64") == 0) {
-            if (_argc < 3) {
-                fprintf(stderr, "Too few arguments\n");
-                usage();
-                return 1;
-            }
-            return testExpGolomb64(atoi(_argv[2]));
-        } else if (strcmp(_argv[1], "testeg64le") == 0) {
-            if (_argc < 3) {
-                fprintf(stderr, "Too few arguments\n");
-                usage();
-                return 1;
-            }
-            return testExpGolomb64le(atoi(_argv[2]));
-        } else {
-            fprintf(stderr, "Wrong arguments\n");
-            usage();
-            return 1;
-        }
-    } else {
-        fprintf(stderr, "Too few arguments\n");
-        usage();
-        return 1;
-    }
-    return 0;
-}
-
-FASTOS_MAIN(ExpGolombApp);
-
-
diff --git a/searchlib/src/apps/loadattribute/.gitignore b/searchlib/src/apps/loadattribute/.gitignore
deleted file mode 100644
index 4f008fbf84e..00000000000
--- a/searchlib/src/apps/loadattribute/.gitignore
+++ /dev/null
@@ -1,3 +0,0 @@
-.depend
-Makefile
-loadattribute
diff --git a/searchlib/src/apps/uniform/.gitignore b/searchlib/src/apps/uniform/.gitignore
index ff18dbaa7fd..d1fbbf62d24 100644
--- a/searchlib/src/apps/uniform/.gitignore
+++ b/searchlib/src/apps/uniform/.gitignore
@@ -1,3 +1,3 @@
 .depend
 Makefile
-uniform
+searchlib_uniform_app
diff --git a/searchlib/src/apps/uniform/CMakeLists.txt b/searchlib/src/apps/uniform/CMakeLists.txt
index 9f9c2139f42..7b835a64e8c 100644
--- a/searchlib/src/apps/uniform/CMakeLists.txt
+++ b/searchlib/src/apps/uniform/CMakeLists.txt
@@ -2,8 +2,6 @@
 vespa_add_executable(searchlib_uniform_app
     SOURCES
     uniform.cpp
-    OUTPUT_NAME uniform
-    INSTALL bin
     DEPENDS
     searchlib
 )
diff --git a/searchlib/src/apps/vespa-attribute-inspect/.gitignore b/searchlib/src/apps/vespa-attribute-inspect/.gitignore
new file mode 100644
index 00000000000..62bcd8ab443
--- /dev/null
+++ b/searchlib/src/apps/vespa-attribute-inspect/.gitignore
@@ -0,0 +1,3 @@
+.depend
+Makefile
+vespa-attribute-inspect
diff --git a/searchlib/src/apps/loadattribute/CMakeLists.txt b/searchlib/src/apps/vespa-attribute-inspect/CMakeLists.txt
index 6712519e59a..4d170b9f325 100644
--- a/searchlib/src/apps/loadattribute/CMakeLists.txt
+++ b/searchlib/src/apps/vespa-attribute-inspect/CMakeLists.txt
@@ -1,8 +1,8 @@
 # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-vespa_add_executable(searchlib_loadattribute_app
+vespa_add_executable(searchlib_vespa-attribute-inspect_app
     SOURCES
-    loadattribute.cpp
-    OUTPUT_NAME loadattribute
+    vespa-attribute-inspect.cpp
+    OUTPUT_NAME vespa-attribute-inspect
     INSTALL bin
     DEPENDS
     searchlib
diff --git a/searchlib/src/apps/loadattribute/loadattribute.rb b/searchlib/src/apps/vespa-attribute-inspect/loadattribute.rb
index d1fb5a5632c..d1fb5a5632c 100644
--- a/searchlib/src/apps/loadattribute/loadattribute.rb
+++ b/searchlib/src/apps/vespa-attribute-inspect/loadattribute.rb
diff --git a/searchlib/src/apps/loadattribute/loadattribute.cpp b/searchlib/src/apps/vespa-attribute-inspect/vespa-attribute-inspect.cpp
index e82ab3ac645..189074aa5d3 100644
--- a/searchlib/src/apps/loadattribute/loadattribute.cpp
+++ b/searchlib/src/apps/vespa-attribute-inspect/vespa-attribute-inspect.cpp
@@ -98,7 +98,7 @@ LoadAttribute::printContent(const AttributePtr & ptr, std::ostream & os)
 void
 LoadAttribute::usage()
 {
-    std::cout << "usage: loadattribute [-p (print content to <attribute>.out)]" << std::endl;
+    std::cout << "usage: vespa-attribute-inspect [-p (print content to <attribute>.out)]" << std::endl;
     std::cout << "                     [-a (apply a single update)]" << std::endl;
     std::cout << "                     [-s (save attribute to <attribute>.save.dat)]" << std::endl;
     std::cout << "                     <attribute>" << std::endl;
diff --git a/searchlib/src/tests/docstore/document_store_visitor/document_store_visitor_test.cpp b/searchlib/src/tests/docstore/document_store_visitor/document_store_visitor_test.cpp
index 6c628b761b4..df1fffa2a0e 100644
--- a/searchlib/src/tests/docstore/document_store_visitor/document_store_visitor_test.cpp
+++ b/searchlib/src/tests/docstore/document_store_visitor/document_store_visitor_test.cpp
@@ -7,6 +7,7 @@
 #include <vespa/searchlib/index/dummyfileheadercontext.h>
 #include <vespa/searchlib/common/bitvector.h>
 #include <vespa/document/repo/documenttyperepo.h>
+#include <vespa/document/datatype/documenttype.h>
 #include <vespa/document/repo/configbuilder.h>
 #include <vespa/vespalib/io/fileutil.h>
 #include <vespa/vespalib/stllike/asciistream.h>
diff --git a/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp b/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp
index fc69cf12d74..46e7333ebb7 100644
--- a/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp
+++ b/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp
@@ -3,6 +3,7 @@
 #include <vespa/vespalib/testkit/test_kit.h>
 #include <vespa/document/repo/configbuilder.h>
 #include <vespa/document/repo/documenttyperepo.h>
+#include <vespa/document/datatype/documenttype.h>
 #include <vespa/searchlib/docstore/chunkformats.h>
 #include <vespa/searchlib/docstore/logdocumentstore.h>
 #include <vespa/searchlib/docstore/storebybucket.h>
diff --git a/searchlib/src/tests/engine/transportserver/transportserver_test.cpp b/searchlib/src/tests/engine/transportserver/transportserver_test.cpp
index b7932ea2381..d8fd8a52b0e 100644
--- a/searchlib/src/tests/engine/transportserver/transportserver_test.cpp
+++ b/searchlib/src/tests/engine/transportserver/transportserver_test.cpp
@@ -4,7 +4,7 @@
 #include <vespa/document/base/documentid.h>
 #include <vespa/searchlib/common/packets.h>
 #include <vespa/searchlib/engine/transportserver.h>
-
+#include <vespa/vespalib/util/stringfmt.h>
 #include <vespa/fnet/fnet.h>
 #include <vespa/searchlib/engine/errorcodes.h>
 #include <vespa/log/log.h>
diff --git a/searchlib/src/tests/features/element_completeness/element_completeness_test.cpp b/searchlib/src/tests/features/element_completeness/element_completeness_test.cpp
index 44077d06d15..16670c0ff30 100644
--- a/searchlib/src/tests/features/element_completeness/element_completeness_test.cpp
+++ b/searchlib/src/tests/features/element_completeness/element_completeness_test.cpp
@@ -7,6 +7,7 @@
 #include <vespa/searchlib/features/element_completeness_feature.h>
 #include <vespa/searchlib/fef/test/ftlib.h>
 #include <vespa/searchlib/fef/test/dummy_dependency_handler.h>
+#include <vespa/vespalib/util/stringfmt.h>
 
 using namespace search::fef;
 using namespace search::fef::test;
diff --git a/searchlib/src/tests/features/element_similarity_feature/element_similarity_feature_test.cpp b/searchlib/src/tests/features/element_similarity_feature/element_similarity_feature_test.cpp
index d693062bb00..f9e1fc902c8 100644
--- a/searchlib/src/tests/features/element_similarity_feature/element_similarity_feature_test.cpp
+++ b/searchlib/src/tests/features/element_similarity_feature/element_similarity_feature_test.cpp
@@ -8,6 +8,7 @@
 #include <vespa/searchlib/fef/test/ftlib.h>
 #include <initializer_list>
 #include <vespa/searchlib/fef/test/dummy_dependency_handler.h>
+#include <vespa/vespalib/util/stringfmt.h>
 
 using namespace search::fef;
 using namespace search::fef::test;
diff --git a/searchlib/src/tests/features/item_raw_score/item_raw_score_test.cpp b/searchlib/src/tests/features/item_raw_score/item_raw_score_test.cpp
index 51405064ac6..44f4fd46765 100644
--- a/searchlib/src/tests/features/item_raw_score/item_raw_score_test.cpp
+++ b/searchlib/src/tests/features/item_raw_score/item_raw_score_test.cpp
@@ -9,6 +9,7 @@
 #include <vespa/searchlib/fef/fef.h>
 #include <vespa/searchlib/fef/test/dummy_dependency_handler.h>
 #include <vespa/vespalib/stllike/asciistream.h>
+#include <vespa/vespalib/util/stringfmt.h>
 
 using search::feature_t;
 using namespace search::fef;
diff --git a/searchlib/src/tests/features/native_dot_product/native_dot_product_test.cpp b/searchlib/src/tests/features/native_dot_product/native_dot_product_test.cpp
index 93313c69a3f..a84e8f42847 100644
--- a/searchlib/src/tests/features/native_dot_product/native_dot_product_test.cpp
+++ b/searchlib/src/tests/features/native_dot_product/native_dot_product_test.cpp
@@ -8,6 +8,7 @@
 #include <vespa/searchlib/fef/fef.h>
 #include <vespa/searchlib/query/weight.h>
 #include <vespa/searchlib/fef/test/dummy_dependency_handler.h>
+#include <vespa/vespalib/util/stringfmt.h>
 
 using search::feature_t;
 using namespace search::fef;
diff --git a/searchlib/src/tests/features/prod_features_fieldtermmatch.cpp b/searchlib/src/tests/features/prod_features_fieldtermmatch.cpp
index 7ae7ba8b26a..3a3dba039ff 100644
--- a/searchlib/src/tests/features/prod_features_fieldtermmatch.cpp
+++ b/searchlib/src/tests/features/prod_features_fieldtermmatch.cpp
@@ -1,9 +1,10 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-#include <vespa/log/log.h>
-LOG_SETUP(".prod_features_fieldtermmatch");
 
 #include "prod_features.h"
 #include <vespa/searchlib/features/fieldtermmatchfeature.h>
+#include <vespa/vespalib/util/stringfmt.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".prod_features_fieldtermmatch");
 
 using namespace search::features;
 using namespace search::fef;
diff --git a/searchlib/src/tests/features/raw_score/raw_score_test.cpp b/searchlib/src/tests/features/raw_score/raw_score_test.cpp
index 8b97a8edbcc..acf5066b2c3 100644
--- a/searchlib/src/tests/features/raw_score/raw_score_test.cpp
+++ b/searchlib/src/tests/features/raw_score/raw_score_test.cpp
@@ -7,6 +7,7 @@
 #include <vespa/searchlib/features/raw_score_feature.h>
 #include <vespa/searchlib/fef/fef.h>
 #include <vespa/searchlib/fef/test/dummy_dependency_handler.h>
+#include <vespa/vespalib/util/stringfmt.h>
 
 using search::feature_t;
 using namespace search::fef;
diff --git a/searchlib/src/tests/features/subqueries/subqueries_test.cpp b/searchlib/src/tests/features/subqueries/subqueries_test.cpp
index 233b3a65b9f..0dfe2858735 100644
--- a/searchlib/src/tests/features/subqueries/subqueries_test.cpp
+++ b/searchlib/src/tests/features/subqueries/subqueries_test.cpp
@@ -7,6 +7,7 @@
 #include <vespa/searchlib/features/subqueries_feature.h>
 #include <vespa/searchlib/fef/fef.h>
 #include <vespa/searchlib/fef/test/dummy_dependency_handler.h>
+#include <vespa/vespalib/util/stringfmt.h>
 
 using search::feature_t;
 using namespace search::fef;
diff --git a/searchlib/src/tests/features/text_similarity_feature/text_similarity_feature_test.cpp b/searchlib/src/tests/features/text_similarity_feature/text_similarity_feature_test.cpp
index 88b043cb2e8..bf5416291d7 100644
--- a/searchlib/src/tests/features/text_similarity_feature/text_similarity_feature_test.cpp
+++ b/searchlib/src/tests/features/text_similarity_feature/text_similarity_feature_test.cpp
@@ -8,6 +8,7 @@
 #include <vespa/searchlib/fef/test/ftlib.h>
 #include <initializer_list>
 #include <vespa/searchlib/fef/test/dummy_dependency_handler.h>
+#include <vespa/vespalib/util/stringfmt.h>
 
 using namespace search::fef;
 using namespace search::fef::test;
diff --git a/searchlib/src/tests/index/docbuilder/docbuilder_test.cpp b/searchlib/src/tests/index/docbuilder/docbuilder_test.cpp
index a4402736f2d..f5eb6d385d9 100644
--- a/searchlib/src/tests/index/docbuilder/docbuilder_test.cpp
+++ b/searchlib/src/tests/index/docbuilder/docbuilder_test.cpp
@@ -124,13 +124,13 @@ Test::testBuilder()
             endField();
         b.startIndexField("iu").
             startSubField("all").
-            addUrlTokenizedString("http://www.yahoo.com:81/fluke?ab=2#4").
+            addUrlTokenizedString("http://www.example.com:81/fluke?ab=2#4").
             endSubField().
             startSubField("scheme").
             addUrlTokenizedString("http").
             endSubField().
             startSubField("host").
-            addUrlTokenizedString("www.yahoo.com").
+            addUrlTokenizedString("www.example.com").
             endSubField().
             startSubField("port").
             addUrlTokenizedString("81").
@@ -148,13 +148,13 @@ Test::testBuilder()
         b.startIndexField("iau").
             startElement(1).
             startSubField("all").
-            addUrlTokenizedString("http://www.yahoo.com:82/fluke?ab=2#8").
+            addUrlTokenizedString("http://www.example.com:82/fluke?ab=2#8").
             endSubField().
             startSubField("scheme").
             addUrlTokenizedString("http").
             endSubField().
             startSubField("host").
-            addUrlTokenizedString("www.yahoo.com").
+            addUrlTokenizedString("www.example.com").
             endSubField().
             startSubField("port").
             addUrlTokenizedString("82").
@@ -196,13 +196,13 @@ Test::testBuilder()
         b.startIndexField("iwu").
             startElement(4).
             startSubField("all").
-            addUrlTokenizedString("http://www.yahoo.com:83/fluke?ab=2#12").
+            addUrlTokenizedString("http://www.example.com:83/fluke?ab=2#12").
             endSubField().
             startSubField("scheme").
             addUrlTokenizedString("http").
             endSubField().
             startSubField("host").
-            addUrlTokenizedString("www.yahoo.com").
+            addUrlTokenizedString("www.example.com").
             endSubField().
             startSubField("port").
             addUrlTokenizedString("83").
@@ -309,8 +309,8 @@ Test::testBuilder()
         EXPECT_EQUAL("<item weight=\"1\">foo</item>", *itr++);
         EXPECT_EQUAL("</ic>", *itr++);
         EXPECT_EQUAL("<iu>", *itr++);
-        EXPECT_EQUAL("<all>http://www.yahoo.com:81/fluke?ab=2#4</all>", *itr++);
-        EXPECT_EQUAL("<host>www.yahoo.com</host>", *itr++);
+        EXPECT_EQUAL("<all>http://www.example.com:81/fluke?ab=2#4</all>", *itr++);
+        EXPECT_EQUAL("<host>www.example.com</host>", *itr++);
         EXPECT_EQUAL("<scheme>http</scheme>", *itr++);
         EXPECT_EQUAL("<path>/fluke</path>", *itr++);
         EXPECT_EQUAL("<port>81</port>", *itr++);
@@ -319,8 +319,8 @@ Test::testBuilder()
         EXPECT_EQUAL("</iu>", *itr++);
         EXPECT_EQUAL("<iau>", *itr++);
         EXPECT_EQUAL("<item>", *itr++);
-        EXPECT_EQUAL("<all>http://www.yahoo.com:82/fluke?ab=2#8</all>", *itr++);
-        EXPECT_EQUAL("<host>www.yahoo.com</host>", *itr++);
+        EXPECT_EQUAL("<all>http://www.example.com:82/fluke?ab=2#8</all>", *itr++);
+        EXPECT_EQUAL("<host>www.example.com</host>", *itr++);
         EXPECT_EQUAL("<scheme>http</scheme>", *itr++);
         EXPECT_EQUAL("<path>/fluke</path>", *itr++);
         EXPECT_EQUAL("<port>82</port>", *itr++);
@@ -339,8 +339,8 @@ Test::testBuilder()
         EXPECT_EQUAL("</iau>", *itr++);
         EXPECT_EQUAL("<iwu>", *itr++);
         EXPECT_EQUAL("<item weight=\"4\">", *itr++);
-        EXPECT_EQUAL("<all>http://www.yahoo.com:83/fluke?ab=2#12</all>", *itr++);
-        EXPECT_EQUAL("<host>www.yahoo.com</host>", *itr++);
+        EXPECT_EQUAL("<all>http://www.example.com:83/fluke?ab=2#12</all>", *itr++);
+        EXPECT_EQUAL("<host>www.example.com</host>", *itr++);
         EXPECT_EQUAL("<scheme>http</scheme>", *itr++);
         EXPECT_EQUAL("<path>/fluke</path>", *itr++);
         EXPECT_EQUAL("<port>83</port>", *itr++);
diff --git a/searchlib/src/tests/memoryindex/dictionary/dictionary_test.cpp b/searchlib/src/tests/memoryindex/dictionary/dictionary_test.cpp
index 8ebab883613..3dbf4efc368 100644
--- a/searchlib/src/tests/memoryindex/dictionary/dictionary_test.cpp
+++ b/searchlib/src/tests/memoryindex/dictionary/dictionary_test.cpp
@@ -1178,13 +1178,13 @@ TEST_F("requireThatUriIndexingIsWorking", DictionaryFixture<UriFixture>)
     f._b.startDocument("doc::10");
     f._b.startIndexField("iu").
         startSubField("all").
-        addUrlTokenizedString("http://www.yahoo.com:81/fluke?ab=2#4").
+        addUrlTokenizedString("http://www.example.com:81/fluke?ab=2#4").
         endSubField().
         startSubField("scheme").
         addUrlTokenizedString("http").
         endSubField().
         startSubField("host").
-        addUrlTokenizedString("www.yahoo.com").
+        addUrlTokenizedString("www.example.com").
         endSubField().
         startSubField("port").
         addUrlTokenizedString("81").
@@ -1202,13 +1202,13 @@ TEST_F("requireThatUriIndexingIsWorking", DictionaryFixture<UriFixture>)
     f._b.startIndexField("iau").
         startElement(1).
         startSubField("all").
-        addUrlTokenizedString("http://www.yahoo.com:82/fluke?ab=2#8").
+        addUrlTokenizedString("http://www.example.com:82/fluke?ab=2#8").
         endSubField().
         startSubField("scheme").
         addUrlTokenizedString("http").
         endSubField().
         startSubField("host").
-        addUrlTokenizedString("www.yahoo.com").
+        addUrlTokenizedString("www.example.com").
         endSubField().
         startSubField("port").
         addUrlTokenizedString("82").
@@ -1250,13 +1250,13 @@ TEST_F("requireThatUriIndexingIsWorking", DictionaryFixture<UriFixture>)
     f._b.startIndexField("iwu").
         startElement(4).
         startSubField("all").
-        addUrlTokenizedString("http://www.yahoo.com:83/fluke?ab=2#12").
+        addUrlTokenizedString("http://www.example.com:83/fluke?ab=2#12").
         endSubField().
         startSubField("scheme").
         addUrlTokenizedString("http").
         endSubField().
         startSubField("host").
-        addUrlTokenizedString("www.yahoo.com").
+        addUrlTokenizedString("www.example.com").
         endSubField().
         startSubField("port").
         addUrlTokenizedString("83").
@@ -1315,7 +1315,7 @@ TEST_F("requireThatUriIndexingIsWorking", DictionaryFixture<UriFixture>)
     }
     {
         uint32_t fieldId = f.getSchema().getIndexFieldId("iu");
-        PostingIterator itr(f._d.findFrozen("yahoo", fieldId),
+        PostingIterator itr(f._d.findFrozen("example", fieldId),
                             featureStoreRef(f._d, fieldId),
                             fieldId, matchData);
         itr.initFullRange();
@@ -1327,7 +1327,7 @@ TEST_F("requireThatUriIndexingIsWorking", DictionaryFixture<UriFixture>)
     }
     {
         uint32_t fieldId = f.getSchema().getIndexFieldId("iau");
-        PostingIterator itr(f._d.findFrozen("yahoo", fieldId),
+        PostingIterator itr(f._d.findFrozen("example", fieldId),
                             featureStoreRef(f._d, fieldId),
                             fieldId, matchData);
         itr.initFullRange();
@@ -1340,7 +1340,7 @@ TEST_F("requireThatUriIndexingIsWorking", DictionaryFixture<UriFixture>)
     }
     {
         uint32_t fieldId = f.getSchema().getIndexFieldId("iwu");
-        PostingIterator itr(f._d.findFrozen("yahoo", fieldId),
+        PostingIterator itr(f._d.findFrozen("example", fieldId),
                             featureStoreRef(f._d, fieldId),
                             fieldId, matchData);
         itr.initFullRange();
diff --git a/searchlib/src/tests/memoryindex/urlfieldinverter/urlfieldinverter_test.cpp b/searchlib/src/tests/memoryindex/urlfieldinverter/urlfieldinverter_test.cpp
index 233c95b58c8..5011310f412 100644
--- a/searchlib/src/tests/memoryindex/urlfieldinverter/urlfieldinverter_test.cpp
+++ b/searchlib/src/tests/memoryindex/urlfieldinverter/urlfieldinverter_test.cpp
@@ -30,13 +30,13 @@ makeDoc10Single(DocBuilder &b)
     b.startDocument("doc::10");
     b.startIndexField("url").
         startSubField("all").
-        addUrlTokenizedString("http://www.yahoo.com:81/fluke?ab=2#4").
+        addUrlTokenizedString("http://www.example.com:81/fluke?ab=2#4").
         endSubField().
         startSubField("scheme").
         addUrlTokenizedString("http").
         endSubField().
         startSubField("host").
-        addUrlTokenizedString("www.yahoo.com").
+        addUrlTokenizedString("www.example.com").
         endSubField().
         startSubField("port").
         addUrlTokenizedString("81").
@@ -63,13 +63,13 @@ makeDoc10Array(DocBuilder &b)
     b.startIndexField("url").
         startElement(1).
         startSubField("all").
-        addUrlTokenizedString("http://www.yahoo.com:82/fluke?ab=2#8").
+        addUrlTokenizedString("http://www.example.com:82/fluke?ab=2#8").
         endSubField().
         startSubField("scheme").
         addUrlTokenizedString("http").
         endSubField().
         startSubField("host").
-        addUrlTokenizedString("www.yahoo.com").
+        addUrlTokenizedString("www.example.com").
         endSubField().
         startSubField("port").
         addUrlTokenizedString("82").
@@ -119,13 +119,13 @@ makeDoc10WeightedSet(DocBuilder &b)
     b.startIndexField("url").
         startElement(4).
         startSubField("all").
-        addUrlTokenizedString("http://www.yahoo.com:83/fluke?ab=2#12").
+        addUrlTokenizedString("http://www.example.com:83/fluke?ab=2#12").
         endSubField().
         startSubField("scheme").
         addUrlTokenizedString("http").
         endSubField().
         startSubField("host").
-        addUrlTokenizedString("www.yahoo.com").
+        addUrlTokenizedString("www.example.com").
         endSubField().
         startSubField("port").
         addUrlTokenizedString("83").
@@ -258,16 +258,16 @@ TEST_F("requireThatSingleUrlFieldWorks", Fixture(CollectionType::SINGLE))
                  "w=81,a=10,"
                  "w=ab,a=10,"
                  "w=com,a=10,"
+                 "w=example,a=10,"
                  "w=fluke,a=10,"
                  "w=http,a=10,"
                  "w=www,a=10,"
-                 "w=yahoo,a=10,"
                  "f=1,"
                  "w=http,a=10,"
                  "f=2,"
                  "w=com,a=10,"
+                 "w=example,a=10,"
                  "w=www,a=10,"
-                 "w=yahoo,a=10,"
                  "f=3,"
                  "w=81,a=10,"
                  "f=4,"
@@ -281,8 +281,8 @@ TEST_F("requireThatSingleUrlFieldWorks", Fixture(CollectionType::SINGLE))
                  "w=EnDhOsT,a=10,"
                  "w=StArThOsT,a=10,"
                  "w=com,a=10,"
-                 "w=www,a=10,"
-                 "w=yahoo,a=10",
+                 "w=example,a=10,"
+                 "w=www,a=10",
                  f._inserter.toStr());
 }
 
@@ -298,18 +298,18 @@ TEST_F("requireThatArrayUrlFieldWorks", Fixture(CollectionType::ARRAY))
                  "w=9,a=10,"
                  "w=ab,a=10,"
                  "w=com,a=10,"
+                 "w=example,a=10,"
                  "w=flickr,a=10,"
                  "w=fluke,a=10,"
                  "w=http,a=10,"
                  "w=www,a=10,"
-                 "w=yahoo,a=10,"
                  "f=1,"
                  "w=http,a=10,"
                  "f=2,"
                  "w=com,a=10,"
+                 "w=example,a=10,"
                  "w=flickr,a=10,"
                  "w=www,a=10,"
-                 "w=yahoo,a=10,"
                  "f=3,"
                  "w=82,a=10,"
                  "f=4,"
@@ -324,9 +324,9 @@ TEST_F("requireThatArrayUrlFieldWorks", Fixture(CollectionType::ARRAY))
                  "w=EnDhOsT,a=10,"
                  "w=StArThOsT,a=10,"
                  "w=com,a=10,"
+                 "w=example,a=10,"
                  "w=flickr,a=10,"
-                 "w=www,a=10,"
-                 "w=yahoo,a=10",
+                 "w=www,a=10",
                  f._inserter.toStr());
 }
 
@@ -342,18 +342,18 @@ TEST_F("requireThatWeightedSetFieldWorks", Fixture(CollectionType::WEIGHTEDSET))
                  "w=85,a=10,"
                  "w=ab,a=10,"
                  "w=com,a=10,"
+                 "w=example,a=10,"
                  "w=flickr,a=10,"
                  "w=fluke,a=10,"
                  "w=http,a=10,"
                  "w=www,a=10,"
-                 "w=yahoo,a=10,"
                  "f=1,"
                  "w=http,a=10,"
                  "f=2,"
                  "w=com,a=10,"
+                 "w=example,a=10,"
                  "w=flickr,a=10,"
                  "w=www,a=10,"
-                 "w=yahoo,a=10,"
                  "f=3,"
                  "w=83,a=10,"
                  "w=85,a=10,"
@@ -369,9 +369,9 @@ TEST_F("requireThatWeightedSetFieldWorks", Fixture(CollectionType::WEIGHTEDSET))
                  "w=EnDhOsT,a=10,"
                  "w=StArThOsT,a=10,"
                  "w=com,a=10,"
+                 "w=example,a=10,"
                  "w=flickr,a=10,"
-                 "w=www,a=10,"
-                 "w=yahoo,a=10",
+                 "w=www,a=10",
                  f._inserter.toStr());
 }
 
@@ -386,16 +386,16 @@ TEST_F("requireThatAnnotatedSingleUrlFieldWorks", Fixture(CollectionType::SINGLE
                  "w=81,a=10,"
                  "w=ab,a=10,"
                  "w=com,a=10,"
+                 "w=example,a=10,"
                  "w=fluke,a=10,"
                  "w=http,a=10,"
                  "w=www,a=10,"
-                 "w=yahoo,a=10,"
                  "f=1,"
                  "w=http,a=10,"
                  "f=2,"
                  "w=com,a=10,"
+                 "w=example,a=10,"
                  "w=www,a=10,"
-                 "w=yahoo,a=10,"
                  "f=3,"
                  "w=81,a=10,"
                  "f=4,"
@@ -410,8 +410,8 @@ TEST_F("requireThatAnnotatedSingleUrlFieldWorks", Fixture(CollectionType::SINGLE
                  "w=EnDhOsT,a=10,"
                  "w=StArThOsT,a=10,"
                  "w=com,a=10,"
-                 "w=www,a=10,"
-                 "w=yahoo,a=10",
+                 "w=example,a=10,"
+                 "w=www,a=10",
                  f._inserter.toStr());
 }
 
@@ -428,18 +428,18 @@ TEST_F("requireThatAnnotatedArrayUrlFieldWorks", Fixture(CollectionType::ARRAY))
                  "w=9,a=10,"
                  "w=ab,a=10,"
                  "w=com,a=10,"
+                 "w=example,a=10,"
                  "w=flickr,a=10,"
                  "w=fluke,a=10,"
                  "w=http,a=10,"
                  "w=www,a=10,"
-                 "w=yahoo,a=10,"
                  "f=1,"
                  "w=http,a=10,"
                  "f=2,"
                  "w=com,a=10,"
+                 "w=example,a=10,"
                  "w=flickr,a=10,"
                  "w=www,a=10,"
-                 "w=yahoo,a=10,"
                  "f=3,"
                  "w=82,a=10,"
                  "f=4,"
@@ -455,9 +455,9 @@ TEST_F("requireThatAnnotatedArrayUrlFieldWorks", Fixture(CollectionType::ARRAY))
                  "w=EnDhOsT,a=10,"
                  "w=StArThOsT,a=10,"
                  "w=com,a=10,"
+                 "w=example,a=10,"
                  "w=flickr,a=10,"
-                 "w=www,a=10,"
-                 "w=yahoo,a=10",
+                 "w=www,a=10",
                  f._inserter.toStr());
 }
 
@@ -476,18 +476,18 @@ TEST_F("requireThatAnnotatedWeightedSetFieldWorks",
                  "w=85,a=10(e=1,w=7,l=9[4]),"
                  "w=ab,a=10(e=0,w=4,l=9[6],e=1,w=7,l=9[6]),"
                  "w=com,a=10(e=0,w=4,l=9[3],e=1,w=7,l=9[3]),"
+                 "w=example,a=10(e=0,w=4,l=9[2]),"
                  "w=flickr,a=10(e=1,w=7,l=9[2]),"
                  "w=fluke,a=10(e=0,w=4,l=9[5],e=1,w=7,l=9[5]),"
                  "w=http,a=10(e=0,w=4,l=9[0],e=1,w=7,l=9[0]),"
                  "w=www,a=10(e=0,w=4,l=9[1],e=1,w=7,l=9[1]),"
-                 "w=yahoo,a=10(e=0,w=4,l=9[2]),"
                  "f=1,"
                  "w=http,a=10(e=0,w=4,l=1[0],e=1,w=7,l=1[0]),"
                  "f=2,"
                  "w=com,a=10(e=0,w=4,l=3[2],e=1,w=7,l=3[2]),"
+                 "w=example,a=10(e=0,w=4,l=3[1]),"
                  "w=flickr,a=10(e=1,w=7,l=3[1]),"
                  "w=www,a=10(e=0,w=4,l=3[0],e=1,w=7,l=3[0]),"
-                 "w=yahoo,a=10(e=0,w=4,l=3[1]),"
                  "f=3,"
                  "w=83,a=10(e=0,w=4,l=1[0]),"
                  "w=85,a=10(e=1,w=7,l=1[0]),"
@@ -504,9 +504,9 @@ TEST_F("requireThatAnnotatedWeightedSetFieldWorks",
                  "w=EnDhOsT,a=10(e=0,w=4,l=5[4],e=1,w=7,l=5[4]),"
                  "w=StArThOsT,a=10(e=0,w=4,l=5[0],e=1,w=7,l=5[0]),"
                  "w=com,a=10(e=0,w=4,l=5[3],e=1,w=7,l=5[3]),"
+                 "w=example,a=10(e=0,w=4,l=5[2]),"
                  "w=flickr,a=10(e=1,w=7,l=5[2]),"
-                 "w=www,a=10(e=0,w=4,l=5[1],e=1,w=7,l=5[1]),"
-                 "w=yahoo,a=10(e=0,w=4,l=5[2])",
+                 "w=www,a=10(e=0,w=4,l=5[1],e=1,w=7,l=5[1])",
                  f._inserter.toStr());
 }
 
diff --git a/searchlib/src/vespa/searchlib/docstore/chunkformat.cpp b/searchlib/src/vespa/searchlib/docstore/chunkformat.cpp
index cd44d34ea22..cddf8c96b2e 100644
--- a/searchlib/src/vespa/searchlib/docstore/chunkformat.cpp
+++ b/searchlib/src/vespa/searchlib/docstore/chunkformat.cpp
@@ -2,6 +2,7 @@
 
 #include "chunkformats.h"
 #include <vespa/document/util/compressor.h>
+#include <vespa/vespalib/util/stringfmt.h>
 
 namespace search {
 
diff --git a/searchlib/src/vespa/searchlib/docstore/chunkformats.cpp b/searchlib/src/vespa/searchlib/docstore/chunkformats.cpp
index c715f8a2129..0843d473121 100644
--- a/searchlib/src/vespa/searchlib/docstore/chunkformats.cpp
+++ b/searchlib/src/vespa/searchlib/docstore/chunkformats.cpp
@@ -3,6 +3,7 @@
 #include "chunkformats.h"
 #include <vespa/vespalib/util/crc.h>
 #include <vespa/vespalib/xxhash/xxhash.h>
+#include <vespa/vespalib/util/stringfmt.h>
 
 namespace search {
 
diff --git a/searchlib/src/vespa/searchlib/docstore/documentstore.cpp b/searchlib/src/vespa/searchlib/docstore/documentstore.cpp
index dd6a0f07197..fd7df29cd49 100644
--- a/searchlib/src/vespa/searchlib/docstore/documentstore.cpp
+++ b/searchlib/src/vespa/searchlib/docstore/documentstore.cpp
@@ -5,6 +5,8 @@
 #include "visitcache.h"
 #include "ibucketizer.h"
 #include <vespa/vespalib/stllike/cache.hpp>
+#include <vespa/vespalib/data/databuffer.h>
+#include <vespa/document/util/compressor.h>
 
 namespace search {
 
diff --git a/searchlib/src/vespa/searchlib/docstore/visitcache.cpp b/searchlib/src/vespa/searchlib/docstore/visitcache.cpp
index c424cb61d8c..8fac288a23a 100644
--- a/searchlib/src/vespa/searchlib/docstore/visitcache.cpp
+++ b/searchlib/src/vespa/searchlib/docstore/visitcache.cpp
@@ -4,9 +4,10 @@
 #include "ibucketizer.h"
 #include <vespa/vespalib/stllike/cache.hpp>
 #include <vespa/vespalib/stllike/hash_map.hpp>
+#include <vespa/vespalib/data/databuffer.h>
+#include <vespa/document/util/compressor.h>
 
-namespace search {
-namespace docstore {
+namespace search::docstore {
 
 using vespalib::ConstBufferRef;
 using vespalib::LockGuard;
@@ -241,5 +242,3 @@ VisitCache::Cache::onRemove(const K & key) {
 }
 
 }
-}
-
diff --git a/searchlib/src/vespa/searchlib/docstore/visitcache.h b/searchlib/src/vespa/searchlib/docstore/visitcache.h
index d47796d446a..6e5887349ee 100644
--- a/searchlib/src/vespa/searchlib/docstore/visitcache.h
+++ b/searchlib/src/vespa/searchlib/docstore/visitcache.h
@@ -6,11 +6,12 @@
 #include "cachestats.h"
 #include <vespa/vespalib/stllike/cache.h>
 #include <vespa/vespalib/stllike/hash_set.h>
+#include <vespa/vespalib/stllike/hash_map.h>
 #include <vespa/vespalib/util/alloc.h>
 #include <vespa/vespalib/objects/nbostream.h>
+#include <vespa/document/util/bytebuffer.h>
 
-namespace search {
-namespace docstore {
+namespace search::docstore {
 
 /**
  * Represents a unique set of keys that together acts as a key in a map.
@@ -160,4 +161,3 @@ private:
 };
 
 }
-}
diff --git a/searchlib/src/vespa/searchlib/expression/arrayatlookupfunctionnode.cpp b/searchlib/src/vespa/searchlib/expression/arrayatlookupfunctionnode.cpp
index 7d443219f29..eaa96587d6d 100644
--- a/searchlib/src/vespa/searchlib/expression/arrayatlookupfunctionnode.cpp
+++ b/searchlib/src/vespa/searchlib/expression/arrayatlookupfunctionnode.cpp
@@ -4,7 +4,7 @@
 #include "integerresultnode.h"
 #include "stringresultnode.h"
 #include <vespa/searchcommon/attribute/iattributecontext.h>
-
+#include <vespa/vespalib/util/stringfmt.h>
 
 namespace search {
 namespace expression {
diff --git a/searchlib/src/vespa/searchlib/expression/documentfieldnode.cpp b/searchlib/src/vespa/searchlib/expression/documentfieldnode.cpp
index 3a5375b672b..d9e4f6a3253 100644
--- a/searchlib/src/vespa/searchlib/expression/documentfieldnode.cpp
+++ b/searchlib/src/vespa/searchlib/expression/documentfieldnode.cpp
@@ -3,6 +3,7 @@
 #include "getdocidnamespacespecificfunctionnode.h"
 #include "getymumchecksumfunctionnode.h"
 #include <vespa/document/fieldvalue/fieldvalues.h>
+#include <vespa/document/datatype/documenttype.h>
 #include <vespa/vespalib/encoding/base64.h>
 
 #include <vespa/log/log.h>
diff --git a/searchlib/src/vespa/searchlib/expression/resultnode.cpp b/searchlib/src/vespa/searchlib/expression/resultnode.cpp
index 3306068efdf..bd8a5ff03d5 100644
--- a/searchlib/src/vespa/searchlib/expression/resultnode.cpp
+++ b/searchlib/src/vespa/searchlib/expression/resultnode.cpp
@@ -1,10 +1,18 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
 #include "resultnode.h"
-#include <stdexcept>
+#include <vespa/vespalib/util/stringfmt.h>
+#include <vespa/vespalib/util/exception.h>
 
 namespace search {
 namespace expression {
 
+int64_t
+ResultNode::onGetEnum(size_t index) const {
+    (void) index;
+    throw vespalib::Exception("search::expression::ResultNode onGetEnum is not implemented");
+}
+
 uint64_t ResultNode::radixAsc(const void * buf) const
 {
     (void) buf;
@@ -65,6 +73,11 @@ size_t ResultNode::getRawByteSize() const
     throw std::runtime_error("ResultNode::getRawByteSize() const must be overloaded by '" + vespalib::string(getClass().name()) + "'.");
 }
 
+const BucketResultNode&
+ResultNode::getNullBucket() const {
+    throw std::runtime_error(vespalib::make_string("No null bucket defined for this type"));
+}
+
 }
 }
 
diff --git a/searchlib/src/vespa/searchlib/expression/resultnode.h b/searchlib/src/vespa/searchlib/expression/resultnode.h
index 6343e90dc6b..26849d27ed3 100644
--- a/searchlib/src/vespa/searchlib/expression/resultnode.h
+++ b/searchlib/src/vespa/searchlib/expression/resultnode.h
@@ -3,7 +3,6 @@
 
 #include "expressionnode.h"
 #include "serializer.h"
-#include <vespa/vespalib/util/exception.h>
 #include <vespa/vespalib/util/buffer.h>
 
 namespace search {
@@ -49,10 +48,7 @@ public:
 
 private:
     virtual int64_t onGetInteger(size_t index) const = 0;
-    virtual int64_t onGetEnum(size_t index) const {
-        (void) index;
-        throw vespalib::Exception("search::expression::ResultNode onGetEnum is not implemented");
-    }
+    virtual int64_t onGetEnum(size_t index) const;
     virtual double    onGetFloat(size_t index) const = 0;
     virtual ConstBufferRef onGetString(size_t index, BufferRef buf) const = 0;
 
@@ -124,9 +120,8 @@ public:
     virtual ResultDeserializer & onDeserializeResult(ResultDeserializer & is);
     virtual size_t getRawByteSize() const;
     virtual bool isMultiValue() const { return false; }
-    virtual const BucketResultNode& getNullBucket() const { throw std::runtime_error(vespalib::make_string("No null bucket defined for this type")); }
+    virtual const BucketResultNode& getNullBucket() const;
 };
 
 }
 }
-
diff --git a/searchlib/src/vespa/searchlib/fef/test/ftlib.cpp b/searchlib/src/vespa/searchlib/fef/test/ftlib.cpp
index 1a359b4056e..08a0017c3e2 100644
--- a/searchlib/src/vespa/searchlib/fef/test/ftlib.cpp
+++ b/searchlib/src/vespa/searchlib/fef/test/ftlib.cpp
@@ -3,6 +3,7 @@
 #include "ftlib.h"
 #include "dummy_dependency_handler.h"
 #include <vespa/searchlib/features/utils.h>
+#include <vespa/vespalib/util/stringfmt.h>
 #include <boost/tokenizer.hpp>
 
 #include <vespa/log/log.h>
diff --git a/searchlib/src/vespa/searchlib/fef/test/matchdatabuilder.cpp b/searchlib/src/vespa/searchlib/fef/test/matchdatabuilder.cpp
index 1f59cfb52b8..02782565f80 100644
--- a/searchlib/src/vespa/searchlib/fef/test/matchdatabuilder.cpp
+++ b/searchlib/src/vespa/searchlib/fef/test/matchdatabuilder.cpp
@@ -4,6 +4,7 @@
 #include <vespa/searchlib/attribute/attributevector.h>
 #include <vespa/searchlib/attribute/attributevector.hpp>
 #include <vespa/searchlib/attribute/stringbase.h>
+#include <vespa/vespalib/util/stringfmt.h>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".fef.matchdatabuilder");
diff --git a/searchlib/src/vespa/searchlib/query/querynoderesultbase.h b/searchlib/src/vespa/searchlib/query/querynoderesultbase.h
index c0d47f0f11d..05f0c9033d5 100644
--- a/searchlib/src/vespa/searchlib/query/querynoderesultbase.h
+++ b/searchlib/src/vespa/searchlib/query/querynoderesultbase.h
@@ -2,6 +2,7 @@
 #pragma once
 
 #include <vespa/vespalib/objects/cloneable.h>
+#include <memory>
 
 namespace search {
 
diff --git a/searchlib/src/vespa/searchlib/test/imported_attribute_fixture.h b/searchlib/src/vespa/searchlib/test/imported_attribute_fixture.h
index 4b376df2bdd..eb278280156 100644
--- a/searchlib/src/vespa/searchlib/test/imported_attribute_fixture.h
+++ b/searchlib/src/vespa/searchlib/test/imported_attribute_fixture.h
@@ -15,6 +15,7 @@
 #include <vespa/searchlib/query/queryterm.h>
 #include <vespa/searchcommon/attribute/attributecontent.h>
 #include <vespa/vespalib/testkit/testapp.h>
+#include <vespa/vespalib/util/stringfmt.h>
 #include <algorithm>
 #include <future>
 #include <map>
diff --git a/slobrok/src/vespa/slobrok/server/rpc_server_manager.cpp b/slobrok/src/vespa/slobrok/server/rpc_server_manager.cpp
index 84ef6432f74..9d34d29fc0b 100644
--- a/slobrok/src/vespa/slobrok/server/rpc_server_manager.cpp
+++ b/slobrok/src/vespa/slobrok/server/rpc_server_manager.cpp
@@ -5,6 +5,7 @@
 #include "rpc_server_map.h"
 #include "remote_slobrok.h"
 #include "sbenv.h"
+#include <vespa/vespalib/util/stringfmt.h>
 #include <sstream>
 
 #include <vespa/log/log.h>
diff --git a/staging_vespalib/src/vespa/vespalib/encoding/base64.cpp b/staging_vespalib/src/vespa/vespalib/encoding/base64.cpp
index 10bc2af51ce..54ac8363bf6 100644
--- a/staging_vespalib/src/vespa/vespalib/encoding/base64.cpp
+++ b/staging_vespalib/src/vespa/vespalib/encoding/base64.cpp
@@ -8,7 +8,8 @@
 
 #include <vespa/vespalib/encoding/base64.h>
 #include <vespa/vespalib/util/exceptions.h>
-#include <assert.h>
+#include <vespa/vespalib/util/stringfmt.h>
+#include <cassert>
 
 namespace vespalib {
 
@@ -129,9 +130,8 @@ Base64::decode(const char* inBuffer, int inLen, char* outBuffer, int outLen)
             continue; // Some illegal chars will be skipped.
         } else if (curchar == -1) {
             // Other illegal characters will generate failure
-            throw vespalib::IllegalArgumentException(vespalib::make_string(
-                    "Illegal base64 character %u found.",
-                    (unsigned int) *thischar), VESPA_STRLOC);
+            throw IllegalArgumentException(make_string("Illegal base64 character %u found.",
+                                                       (unsigned int) *thischar), VESPA_STRLOC);
         } else {
 
             // Four bytes from input (eqals three bytes in output)
@@ -162,4 +162,3 @@ Base64::decode(const char* inBuffer, int inLen, char* outBuffer, int outLen)
 }
 
 } // namespace vespalib
-
diff --git a/staging_vespalib/src/vespa/vespalib/net/http_server.cpp b/staging_vespalib/src/vespa/vespalib/net/http_server.cpp
index 32b9caab687..2c67a3c48c9 100644
--- a/staging_vespalib/src/vespa/vespalib/net/http_server.cpp
+++ b/staging_vespalib/src/vespa/vespalib/net/http_server.cpp
@@ -3,7 +3,7 @@
 #include "http_server.h"
 #include <vespa/vespalib/util/exceptions.h>
 #include <vespa/vespalib/util/host_name.h>
-#include <algorithm>
+#include <vespa/vespalib/util/stringfmt.h>
 
 namespace vespalib {
 
diff --git a/staging_vespalib/src/vespa/vespalib/objects/cloneable.h b/staging_vespalib/src/vespa/vespalib/objects/cloneable.h
index c0b12272797..ea9c8002d6e 100644
--- a/staging_vespalib/src/vespa/vespalib/objects/cloneable.h
+++ b/staging_vespalib/src/vespa/vespalib/objects/cloneable.h
@@ -5,8 +5,6 @@
  * @brief Superclass for objects implementing clone() deep copy.
  */
 
-#include <memory>
-
 namespace vespalib {
 
 class Cloneable {
diff --git a/staging_vespalib/src/vespa/vespalib/util/document_runnable.cpp b/staging_vespalib/src/vespa/vespalib/util/document_runnable.cpp
index 7c581c89af0..576343245e4 100644
--- a/staging_vespalib/src/vespa/vespalib/util/document_runnable.cpp
+++ b/staging_vespalib/src/vespa/vespalib/util/document_runnable.cpp
@@ -2,6 +2,7 @@
 
 #include "document_runnable.h"
 #include <vespa/vespalib/util/exceptions.h>
+#include <vespa/vespalib/util/stringfmt.h>
 
 namespace document {
 
diff --git a/staging_vespalib/src/vespa/vespalib/util/jsonstream.cpp b/staging_vespalib/src/vespa/vespalib/util/jsonstream.cpp
index e8f242879d8..c7a7ac34ef3 100644
--- a/staging_vespalib/src/vespa/vespalib/util/jsonstream.cpp
+++ b/staging_vespalib/src/vespa/vespalib/util/jsonstream.cpp
@@ -4,6 +4,7 @@
 #include "jsonexception.h"
 #include <vespa/vespalib/util/exceptions.h>
 #include <vespa/vespalib/stllike/asciistream.h>
+#include <vespa/vespalib/util/stringfmt.h>
 
 namespace vespalib {
 
@@ -18,7 +19,7 @@ JsonStream::getStateName(const State& s) {
     throw IllegalStateException("Control should not reach this point", VESPA_STRLOC);
 }
 
-JsonStream::JsonStream(vespalib::asciistream& as, bool createIndents)
+JsonStream::JsonStream(asciistream& as, bool createIndents)
     : _writer(as)
 {
     if (createIndents) _writer.setPretty();
@@ -28,7 +29,7 @@ JsonStream::JsonStream(vespalib::asciistream& as, bool createIndents)
 JsonStream::~JsonStream() {}
 
 JsonStream&
-JsonStream::operator<<(vespalib::stringref value)
+JsonStream::operator<<(stringref value)
 {
     if (_state.empty()) {
         fail("Stream already finalized. Can't add a string value.");
@@ -314,7 +315,7 @@ JsonStream::finalize()
 string
 JsonStream::getStateString() const
 {
-    vespalib::asciistream as;
+    asciistream as;
     for (auto it(_state.begin()), mt(_state.end()); it != mt; it++) {
         switch (it->state) {
             case State::OBJECT_EXPECTING_KEY:
@@ -343,7 +344,7 @@ JsonStream::getStateString() const
     return as.str();
 }
 
-vespalib::string
+string
 JsonStream::getJsonStreamState() const
 {
     asciistream report;
@@ -360,4 +361,4 @@ JsonStream::fail(stringref error) const
     throw JsonStreamException(report.str(), "", VESPA_STRLOC);
 }
 
-} // vespalib
+}
diff --git a/staging_vespalib/src/vespa/vespalib/util/librarypool.cpp b/staging_vespalib/src/vespa/vespalib/util/librarypool.cpp
index 8dd6995a6c1..d992b1b6db8 100644
--- a/staging_vespalib/src/vespa/vespalib/util/librarypool.cpp
+++ b/staging_vespalib/src/vespa/vespalib/util/librarypool.cpp
@@ -1,6 +1,8 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
 #include <vespa/vespalib/util/librarypool.h>
 #include <vespa/vespalib/util/exceptions.h>
+#include <vespa/vespalib/util/stringfmt.h>
 
 namespace vespalib {
 
@@ -17,14 +19,14 @@ LibraryPool::~LibraryPool()
 }
 
 void
-LibraryPool::loadLibrary(const vespalib::stringref & libName)
+LibraryPool::loadLibrary(const stringref & libName)
 {
     LockGuard guard(_lock);
     if (_libraries.find(libName) == _libraries.end()) {
         DynamicLibrarySP lib(new FastOS_DynamicLibrary);
-        vespalib::string file(libName);
+        string file(libName);
         if (!lib->Open(file.c_str())) {
-            vespalib::string error = lib->GetLastErrorString();
+            string error = lib->GetLastErrorString();
             throw IllegalArgumentException(make_string("Failed loading dynamic library '%s' due to '%s'.",
                       file.c_str(), error.c_str()));
         } else {
@@ -34,7 +36,7 @@ LibraryPool::loadLibrary(const vespalib::stringref & libName)
 }
 
 FastOS_DynamicLibrary *
-LibraryPool::get(const vespalib::stringref & name)
+LibraryPool::get(const stringref & name)
 {
     LockGuard guard(_lock);
     LibraryMap::const_iterator found(_libraries.find(name));
@@ -44,7 +46,7 @@ LibraryPool::get(const vespalib::stringref & name)
 }
 
 const FastOS_DynamicLibrary *
-LibraryPool::get(const vespalib::stringref & name) const
+LibraryPool::get(const stringref & name) const
 {
     LockGuard guard(_lock);
     LibraryMap::const_iterator found(_libraries.find(name));
diff --git a/staging_vespalib/src/vespa/vespalib/util/programoptions.cpp b/staging_vespalib/src/vespa/vespalib/util/programoptions.cpp
index 41ff561011b..6573390c712 100644
--- a/staging_vespalib/src/vespa/vespalib/util/programoptions.cpp
+++ b/staging_vespalib/src/vespa/vespalib/util/programoptions.cpp
@@ -1,7 +1,6 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-#include <vespa/vespalib/util/programoptions.h>
-
-#include <iostream>
+#include "programoptions.h"
+#include <vespa/vespalib/util/stringfmt.h>
 #include <vespa/vespalib/util/exceptions.h>
 #include <boost/lexical_cast.hpp>
 
diff --git a/staging_vespalib/src/vespa/vespalib/util/xmlserializable.cpp b/staging_vespalib/src/vespa/vespalib/util/xmlserializable.cpp
index 58a5489647d..043cddc3259 100644
--- a/staging_vespalib/src/vespa/vespalib/util/xmlserializable.cpp
+++ b/staging_vespalib/src/vespa/vespalib/util/xmlserializable.cpp
@@ -1,9 +1,9 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
-#include <vespa/vespalib/util/xmlserializable.hpp>
+#include "xmlserializable.hpp"
 #include <vespa/vespalib/encoding/base64.h>
 #include <vespa/vespalib/stllike/asciistream.h>
-#include <vector>
+#include <vespa/vespalib/util/stringfmt.h>
 #include <cassert>
 
 namespace vespalib {
diff --git a/staging_vespalib/src/vespa/vespalib/util/xmlserializable.hpp b/staging_vespalib/src/vespa/vespalib/util/xmlserializable.hpp
index 7fcc964d8f1..0da684d6b28 100644
--- a/staging_vespalib/src/vespa/vespalib/util/xmlserializable.hpp
+++ b/staging_vespalib/src/vespa/vespalib/util/xmlserializable.hpp
@@ -3,8 +3,8 @@
 #pragma once
 
 #include "xmlserializable.h"
-#include <sstream>
 #include <vespa/vespalib/util/exceptions.h>
+#include <sstream>
 
 namespace vespalib {
 namespace xml {
diff --git a/storage/src/tests/bucketdb/bucketmanagertest.cpp b/storage/src/tests/bucketdb/bucketmanagertest.cpp
index f9ae4f57fa5..227dd40b34d 100644
--- a/storage/src/tests/bucketdb/bucketmanagertest.cpp
+++ b/storage/src/tests/bucketdb/bucketmanagertest.cpp
@@ -16,6 +16,7 @@
 #include <tests/common/testhelper.h>
 #include <vespa/vespalib/io/fileutil.h>
 #include <vespa/vespalib/testkit/testapp.h>
+#include <vespa/vespalib/util/stringfmt.h>
 #include <vespa/config/helper/configgetter.hpp>
 #include <future>
 
diff --git a/storage/src/tests/persistence/filestorage/filestormanagertest.cpp b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp
index e2f41aa5b3e..52b976586e8 100644
--- a/storage/src/tests/persistence/filestorage/filestormanagertest.cpp
+++ b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp
@@ -11,6 +11,7 @@
 #include <vespa/vdslib/container/mutabledocumentlist.h>
 #include <vespa/vdstestlib/cppunit/macros.h>
 #include <vespa/vespalib/io/fileutil.h>
+#include <vespa/vespalib/util/stringfmt.h>
 #include <vespa/storageapi/message/bucket.h>
 #include <vespa/storageapi/message/bucketsplitting.h>
 #include <vespa/storageapi/message/multioperation.h>
diff --git a/storage/src/tests/persistence/splitbitdetectortest.cpp b/storage/src/tests/persistence/splitbitdetectortest.cpp
index 3251dd9a9a8..47efaaa371e 100644
--- a/storage/src/tests/persistence/splitbitdetectortest.cpp
+++ b/storage/src/tests/persistence/splitbitdetectortest.cpp
@@ -1,7 +1,7 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include <vespa/vdstestlib/cppunit/macros.h>
-#include <fstream>
+#include <vespa/vespalib/util/stringfmt.h>
 #include <vespa/storage/persistence/splitbitdetector.h>
 #include <vespa/vespalib/io/fileutil.h>
 #include <vespa/persistence/dummyimpl/dummypersistence.h>
diff --git a/storage/src/tests/storageserver/documentapiconvertertest.cpp b/storage/src/tests/storageserver/documentapiconvertertest.cpp
index 1f1198efc3c..b7d3554e506 100644
--- a/storage/src/tests/storageserver/documentapiconvertertest.cpp
+++ b/storage/src/tests/storageserver/documentapiconvertertest.cpp
@@ -3,6 +3,7 @@
 #include <vespa/document/base/testdocrepo.h>
 #include <cppunit/extensions/HelperMacros.h>
 #include <vespa/document/fieldvalue/document.h>
+#include <vespa/document/datatype/documenttype.h>
 #include <vespa/documentapi/documentapi.h>
 #include <vespa/messagebus/emptyreply.h>
 #include <vespa/storage/storageserver/documentapiconverter.h>
diff --git a/storage/src/vespa/storage/bucketdb/bucketmanager.cpp b/storage/src/vespa/storage/bucketdb/bucketmanager.cpp
index 0570d0a3e27..18fc5efb4b5 100644
--- a/storage/src/vespa/storage/bucketdb/bucketmanager.cpp
+++ b/storage/src/vespa/storage/bucketdb/bucketmanager.cpp
@@ -15,6 +15,7 @@
 #include <vespa/storageapi/message/state.h>
 #include <vespa/storageapi/message/bucketsplitting.h>
 #include <vespa/storageapi/message/stat.h>
+#include <vespa/vespalib/util/stringfmt.h>
 #include <vespa/vespalib/stllike/hash_map.hpp>
 #include <vespa/config/config.h>
 #include <unordered_map>
diff --git a/storage/src/vespa/storage/bucketmover/bucketmover.cpp b/storage/src/vespa/storage/bucketmover/bucketmover.cpp
index e4ddc3cb5a7..bda0a394802 100644
--- a/storage/src/vespa/storage/bucketmover/bucketmover.cpp
+++ b/storage/src/vespa/storage/bucketmover/bucketmover.cpp
@@ -6,6 +6,7 @@
 #include <vespa/storage/common/bucketmessages.h>
 #include <vespa/storage/common/nodestateupdater.h>
 #include <vespa/storage/storageutil/log.h>
+#include <vespa/vespalib/util/stringfmt.h>
 
 LOG_SETUP(".bucketmover");
 
diff --git a/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp b/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp
index 5b907d21fcd..54ef503a772 100644
--- a/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp
+++ b/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp
@@ -5,6 +5,7 @@
 #include "putoperation.h"
 #include "updateoperation.h"
 #include <vespa/document/fieldvalue/document.h>
+#include <vespa/document/datatype/documenttype.h>
 #include <vespa/document/select/parser.h>
 #include <vespa/storageapi/message/persistence.h>
 #include <vespa/storageapi/message/batch.h>
diff --git a/storage/src/vespa/storage/distributor/operations/operation.cpp b/storage/src/vespa/storage/distributor/operations/operation.cpp
index 37e997d6080..8f6acc3258d 100644
--- a/storage/src/vespa/storage/distributor/operations/operation.cpp
+++ b/storage/src/vespa/storage/distributor/operations/operation.cpp
@@ -5,6 +5,7 @@
 #include <vespa/storageapi/messageapi/storagemessage.h>
 #include <vespa/storageapi/messageapi/storagecommand.h>
 #include <vespa/storageapi/messageapi/storagereply.h>
+#include <vespa/vespalib/util/stringfmt.h>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".distributor.callback");
diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp
index a3c8fc3d671..43d5de4a5a0 100644
--- a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp
+++ b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp
@@ -17,6 +17,7 @@
 #include <vespa/storage/common/bucketoperationlogger.h>
 #include <vespa/storage/bucketdb/lockablemap.hpp>
 #include <vespa/vespalib/stllike/hash_map.hpp>
+#include <vespa/vespalib/util/stringfmt.h>
 
 LOG_SETUP(".persistence.filestor.manager");
 
diff --git a/storage/src/vespa/storage/storageserver/mergethrottler.cpp b/storage/src/vespa/storage/storageserver/mergethrottler.cpp
index 577fedb58e1..ede7be3b9ad 100644
--- a/storage/src/vespa/storage/storageserver/mergethrottler.cpp
+++ b/storage/src/vespa/storage/storageserver/mergethrottler.cpp
@@ -2,9 +2,9 @@
 
 #include "mergethrottler.h"
 #include "storagemetricsset.h"
-#include <iostream>
 #include <sstream>
 #include <vespa/vespalib/stllike/asciistream.h>
+#include <vespa/vespalib/util/stringfmt.h>
 #include <vespa/storage/common/nodestateupdater.h>
 #include <vespa/storage/persistence/messages.h>
 #include <vespa/log/log.h>
diff --git a/storage/src/vespa/storage/storageserver/messagesink.cpp b/storage/src/vespa/storage/storageserver/messagesink.cpp
index 0df24ef3536..4960cd2085e 100644
--- a/storage/src/vespa/storage/storageserver/messagesink.cpp
+++ b/storage/src/vespa/storage/storageserver/messagesink.cpp
@@ -2,6 +2,7 @@
 
 #include "messagesink.h"
 #include <vespa/storageapi/message/persistence.h>
+#include <ostream>
 
 using std::shared_ptr;
 
diff --git a/storage/src/vespa/storage/tools/generatedistributionbits.cpp b/storage/src/vespa/storage/tools/generatedistributionbits.cpp
index 53c7c7cf1a1..3f5b3f4ae3d 100644
--- a/storage/src/vespa/storage/tools/generatedistributionbits.cpp
+++ b/storage/src/vespa/storage/tools/generatedistributionbits.cpp
@@ -1,9 +1,11 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
 #include <vespa/document/bucket/bucketidfactory.h>
 #include <vespa/vespalib/util/programoptions.h>
 #include <vespa/vdslib/distribution/distribution.h>
 #include <vespa/vdslib/state/clusterstate.h>
 #include <vespa/storage/bucketdb/judyarray.h>
+#include <vespa/vespalib/util/stringfmt.h>
 #include <iomanip>
 #include <iostream>
 #include <vespa/config-stor-distribution.h>
diff --git a/storage/src/vespa/storage/visiting/countvisitor.cpp b/storage/src/vespa/storage/visiting/countvisitor.cpp
index 282e2f32873..ca6678c1b3d 100644
--- a/storage/src/vespa/storage/visiting/countvisitor.cpp
+++ b/storage/src/vespa/storage/visiting/countvisitor.cpp
@@ -3,6 +3,7 @@
 #include "countvisitor.h"
 #include <vespa/document/fieldvalue/document.h>
 #include <vespa/documentapi/messagebus/messages/visitor.h>
+#include <vespa/vespalib/util/stringfmt.h>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".visitor.instance.countvisitor");
diff --git a/storage/src/vespa/storage/visiting/visitor.cpp b/storage/src/vespa/storage/visiting/visitor.cpp
index 704b523a3b6..6940706e211 100644
--- a/storage/src/vespa/storage/visiting/visitor.cpp
+++ b/storage/src/vespa/storage/visiting/visitor.cpp
@@ -10,7 +10,9 @@
 #include <vespa/document/select/node.h>
 #include <vespa/vespalib/stllike/hash_map.hpp>
 #include <vespa/vespalib/stllike/asciistream.h>
+#include <vespa/vespalib/util/stringfmt.h>
 #include <unordered_map>
+
 #include <vespa/log/log.h>
 LOG_SETUP(".visitor.instance");
 
diff --git a/storage/src/vespa/storage/visiting/visitormanager.cpp b/storage/src/vespa/storage/visiting/visitormanager.cpp
index 7b8ce974ca9..0df82dad80f 100644
--- a/storage/src/vespa/storage/visiting/visitormanager.cpp
+++ b/storage/src/vespa/storage/visiting/visitormanager.cpp
@@ -8,10 +8,9 @@
 #include "testvisitor.h"
 #include "recoveryvisitor.h"
 #include <vespa/storageframework/generic/memory/memorymanagerinterface.h>
-#include <vespa/storageapi/message/visitor.h>
 #include <vespa/storage/common/statusmessages.h>
-#include <vespa/storage/storageserver/storagemetricsset.h>
 #include <vespa/documentapi/loadtypes/loadtypeset.h>
+#include <vespa/vespalib/util/stringfmt.h>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".visitor.manager");
diff --git a/storageapi/src/vespa/storageapi/message/batch.cpp b/storageapi/src/vespa/storageapi/message/batch.cpp
index 405d5b4c9d2..ee7bbbcbdf6 100644
--- a/storageapi/src/vespa/storageapi/message/batch.cpp
+++ b/storageapi/src/vespa/storageapi/message/batch.cpp
@@ -2,7 +2,7 @@
 //
 #include "batch.h"
 #include <vespa/document/bucket/bucketidfactory.h>
-#include <algorithm>
+#include <ostream>
 
 using namespace storage::api;
 
diff --git a/storageapi/src/vespa/storageapi/message/bucket.cpp b/storageapi/src/vespa/storageapi/message/bucket.cpp
index 3400e0d6399..e4e6f796ac2 100644
--- a/storageapi/src/vespa/storageapi/message/bucket.cpp
+++ b/storageapi/src/vespa/storageapi/message/bucket.cpp
@@ -4,9 +4,10 @@
 #include <vespa/document/fieldvalue/document.h>
 #include <vespa/vespalib/stllike/asciistream.h>
 #include <vespa/vespalib/util/array.hpp>
+#include <ostream>
+#include <iterator>
 
-namespace storage {
-namespace api {
+namespace storage::api {
 
 IMPLEMENT_COMMAND(CreateBucketCommand, CreateBucketReply)
 IMPLEMENT_REPLY(CreateBucketReply)
@@ -625,7 +626,6 @@ SetBucketStateReply::print(std::ostream& out,
     }
 }
 
-} // api
-} // storage
+}
 
 template class vespalib::Array<storage::api::RequestBucketInfoReply::Entry>;
diff --git a/storageapi/src/vespa/storageapi/message/persistence.cpp b/storageapi/src/vespa/storageapi/message/persistence.cpp
index 6de53d9db82..ce1adcfa2ca 100644
--- a/storageapi/src/vespa/storageapi/message/persistence.cpp
+++ b/storageapi/src/vespa/storageapi/message/persistence.cpp
@@ -3,6 +3,7 @@
 #include "persistence.h"
 #include <vespa/vespalib/util/exceptions.h>
 #include <vespa/vespalib/stllike/asciistream.h>
+#include <ostream>
 
 namespace storage {
 namespace api {
diff --git a/streamingvisitors/src/tests/hitcollector/hitcollector.cpp b/streamingvisitors/src/tests/hitcollector/hitcollector.cpp
index dc10ea168c0..4e1ac516a7b 100644
--- a/streamingvisitors/src/tests/hitcollector/hitcollector.cpp
+++ b/streamingvisitors/src/tests/hitcollector/hitcollector.cpp
@@ -2,6 +2,7 @@
 #include <vespa/vespalib/testkit/testapp.h>
 
 #include <vespa/document/fieldvalue/fieldvalues.h>
+#include <vespa/document/datatype/documenttype.h>
 #include <vespa/searchlib/fef/matchdata.h>
 #include <vespa/searchlib/fef/feature_resolver.h>
 #include <vespa/searchvisitor/hitcollector.h>
diff --git a/streamingvisitors/src/vespa/searchvisitor/hitcollector.cpp b/streamingvisitors/src/vespa/searchvisitor/hitcollector.cpp
index 5ca14fd617d..074070d2ccd 100644
--- a/streamingvisitors/src/vespa/searchvisitor/hitcollector.cpp
+++ b/streamingvisitors/src/vespa/searchvisitor/hitcollector.cpp
@@ -2,6 +2,7 @@
 
 #include "hitcollector.h"
 #include <vespa/searchlib/fef/feature_resolver.h>
+#include <vespa/vespalib/util/stringfmt.h>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".searchvisitor.hitcollector");
diff --git a/streamingvisitors/src/vespa/searchvisitor/rankmanager.cpp b/streamingvisitors/src/vespa/searchvisitor/rankmanager.cpp
index d25154aba7f..4befe163377 100644
--- a/streamingvisitors/src/vespa/searchvisitor/rankmanager.cpp
+++ b/streamingvisitors/src/vespa/searchvisitor/rankmanager.cpp
@@ -3,7 +3,7 @@
 #include "rankmanager.h"
 #include <vespa/searchlib/features/setup.h>
 #include <vespa/searchlib/fef/functiontablefactory.h>
-#include <vespa/vespalib/util/vstringfmt.h>
+#include <vespa/vespalib/util/stringfmt.h>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".searchvisitor.rankmanager");
@@ -106,7 +106,7 @@ RankManager::Snapshot::initRankSetup(const BlueprintFactory & factory)
     LOG(debug, "Number of index environments and rank setups: %u", (uint32_t)_indexEnv.size());
     LOG_ASSERT(_properties.size() == _rankSetup.size());
     for (uint32_t i = 0; i < _properties.size(); ++i) {
-        vespalib::string number = vespalib::make_vespa_string("%u", i);
+        vespalib::string number = vespalib::make_string("%u", i);
         _rpmap[number] = i;
     }
     for (uint32_t i = 0; i < _properties.size(); ++i) {
diff --git a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp
index 616da034c54..56c95e7fbc7 100644
--- a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp
+++ b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp
@@ -4,6 +4,7 @@
 #include "searchenvironment.h"
 #include "searchvisitor.h"
 #include <vespa/document/datatype/positiondatatype.h>
+#include <vespa/document/datatype/documenttype.h>
 #include <vespa/searchlib/aggregation/modifiers.h>
 #include <vespa/searchlib/common/packets.h>
 #include <vespa/searchlib/uca/ucaconverter.h>
diff --git a/vdslib/src/tests/container/documentlisttest.cpp b/vdslib/src/tests/container/documentlisttest.cpp
index 5120c4efa05..02e2d0fd935 100644
--- a/vdslib/src/tests/container/documentlisttest.cpp
+++ b/vdslib/src/tests/container/documentlisttest.cpp
@@ -6,6 +6,7 @@
 #include <vespa/document/update/documentupdate.h>
 #include <vespa/vespalib/objects/nbostream.h>
 #include <vespa/vespalib/util/random.h>
+#include <vespa/vespalib/util/stringfmt.h>
 #include <vespa/vespalib/testkit/testapp.h>
 #include <cppunit/extensions/HelperMacros.h>
 
diff --git a/vdslib/src/vespa/vdslib/container/documentlist.cpp b/vdslib/src/vespa/vdslib/container/documentlist.cpp
index 2b105a77408..71bee5cf624 100644
--- a/vdslib/src/vespa/vdslib/container/documentlist.cpp
+++ b/vdslib/src/vespa/vdslib/container/documentlist.cpp
@@ -5,8 +5,9 @@
 #include <vespa/document/util/stringutil.h>
 #include <vespa/document/util/serializableexceptions.h>
 #include <vespa/document/update/documentupdate.h>
-#include <vespa/log/log.h>
+#include <vespa/document/datatype/documenttype.h>
 
+#include <vespa/log/log.h>
 LOG_SETUP(".vdslib.container.documentlist");
 
 using document::ByteBuffer;
diff --git a/vdslib/src/vespa/vdslib/container/mutabledocumentlist.cpp b/vdslib/src/vespa/vdslib/container/mutabledocumentlist.cpp
index 1c80e3ba8ee..f265a6c947f 100644
--- a/vdslib/src/vespa/vdslib/container/mutabledocumentlist.cpp
+++ b/vdslib/src/vespa/vdslib/container/mutabledocumentlist.cpp
@@ -2,6 +2,7 @@
 
 #include "mutabledocumentlist.h"
 #include <vespa/document/update/documentupdate.h>
+#include <vespa/document/datatype/documenttype.h>
 #include <vespa/vespalib/objects/nbostream.h>
 
 using vespalib::nbostream;
diff --git a/vdslib/src/vespa/vdslib/container/operationlist.cpp b/vdslib/src/vespa/vdslib/container/operationlist.cpp
index 6af338a4c0e..8c4515f15ff 100644
--- a/vdslib/src/vespa/vdslib/container/operationlist.cpp
+++ b/vdslib/src/vespa/vdslib/container/operationlist.cpp
@@ -3,6 +3,7 @@
 #include "operationlist.h"
 #include "documentlist.h"
 #include <vespa/document/update/documentupdate.h>
+#include <vespa/document/datatype/documenttype.h>
 #include <vespa/vespalib/objects/nbostream.h>
 
 namespace vdslib {
@@ -22,8 +23,7 @@ int OperationList::getRequiredBufferSize() const {
         switch(_operations[i].opt) {
         case OperationList::Operation::REMOVE:
         {
-            document::Document doc(*document::DataType::DOCUMENT,
-                                   _operations[i].docId);
+            document::Document doc(*document::DataType::DOCUMENT, _operations[i].docId);
             doc.serializeHeader(stream);
             break;
         }
diff --git a/vdstestlib/src/vespa/vdstestlib/cppunit/dirconfig.cpp b/vdstestlib/src/vespa/vdstestlib/cppunit/dirconfig.cpp
index 64ff1377128..5e1c7c6b1b4 100644
--- a/vdstestlib/src/vespa/vdstestlib/cppunit/dirconfig.cpp
+++ b/vdstestlib/src/vespa/vdstestlib/cppunit/dirconfig.cpp
@@ -4,6 +4,7 @@
 
 #include <vespa/vespalib/io/fileutil.h>
 #include <vespa/vespalib/util/exceptions.h>
+#include <vespa/vespalib/util/stringfmt.h>
 #include <fstream>
 #include <atomic>
 
diff --git a/vespabase/CMakeLists.txt b/vespabase/CMakeLists.txt
index b4ed5c69044..583bbf8c8b8 100644
--- a/vespabase/CMakeLists.txt
+++ b/vespabase/CMakeLists.txt
@@ -6,7 +6,7 @@ vespa_install_script(src/start-cbinaries.sh vdsgetnodestate bin)
 vespa_install_script(src/start-cbinaries.sh vdsgetsystemstate bin)
 vespa_install_script(src/start-cbinaries.sh vdssetnodestate bin)
 vespa_install_script(src/start-cbinaries.sh vdsstat bin)
-vespa_install_script(src/start-cbinaries.sh verify_ranksetup bin)
+vespa_install_script(src/start-cbinaries.sh vespa-verify-ranksetup bin)
 vespa_install_script(src/start-cbinaries.sh vespa-config-status bin)
 vespa_install_script(src/start-cbinaries.sh vespadoclocator bin)
 vespa_install_script(src/start-cbinaries.sh vespaget bin)
@@ -18,12 +18,11 @@ vespa_install_script(src/start-cbinaries.sh vespa-transactionlog-inspect bin)
 vespa_install_script(src/start-cbinaries.sh vespavisit bin)
 vespa_install_script(src/start-cbinaries.sh vespavisittarget bin)
 vespa_install_script(src/start-cbinaries.sh distributord sbin)
-vespa_install_script(src/start-cbinaries.sh fdispatch sbin)
+vespa_install_script(src/start-cbinaries.sh vespa-fdispatch sbin)
 vespa_install_script(src/start-cbinaries.sh filedistributor sbin)
-vespa_install_script(src/start-cbinaries.sh proton sbin)
+vespa_install_script(src/start-cbinaries.sh vespa-proton sbin)
 vespa_install_script(src/start-cbinaries.sh storaged sbin)
 
-vespa_install_script(src/vespa-allow-downgrade-from-6-to-5.sh vespa-allow-downgrade-from-6-to-5 bin)
 vespa_install_script(src/start-tool.sh vespa-start-tool.sh bin)
 
 vespa_install_script(src/rhel-prestart.sh vespa-prestart.sh bin)
diff --git a/vespabase/src/vespa-allow-downgrade-from-6-to-5.sh b/vespabase/src/vespa-allow-downgrade-from-6-to-5.sh
deleted file mode 100755
index c01333bdd79..00000000000
--- a/vespabase/src/vespa-allow-downgrade-from-6-to-5.sh
+++ /dev/null
@@ -1,8 +0,0 @@
-#!/bin/sh
-# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-for f in /home/y/var/db/vespa/search/*/*/documents/*/config/config-*
-do
-    echo $f
-    touch $f/smartsummary.cfg
-done
diff --git a/vespaclient-java/OWNERS b/vespaclient-java/OWNERS
new file mode 100644
index 00000000000..569bf1cc3a1
--- /dev/null
+++ b/vespaclient-java/OWNERS
@@ -0,0 +1 @@
+bjorncs
diff --git a/vespaclient-java/pom.xml b/vespaclient-java/pom.xml
new file mode 100644
index 00000000000..fb82220cb16
--- /dev/null
+++ b/vespaclient-java/pom.xml
@@ -0,0 +1,82 @@
+<?xml version="1.0"?>
+<!-- Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<!-- TODO: Remove this module on Vespa 7 -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>com.yahoo.vespa</groupId>
+        <artifactId>parent</artifactId>
+        <version>6-SNAPSHOT</version>
+    </parent>
+    <artifactId>vespaclient-java</artifactId>
+    <version>6-SNAPSHOT</version>
+    <dependencies>
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.mockito</groupId>
+            <artifactId>mockito-core</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.yahoo.vespa</groupId>
+            <artifactId>vespaclient-core</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.yahoo.vespa</groupId>
+            <artifactId>defaults</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.yahoo.vespa</groupId>
+            <artifactId>documentapi</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.yahoo.vespa</groupId>
+            <artifactId>container-dev</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>commons-cli</groupId>
+            <artifactId>commons-cli</artifactId>
+        </dependency>
+    </dependencies>
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <configuration>
+                    <compilerArgs>
+                        <arg>-Xlint:all</arg>
+                        <arg>-Xlint:-serial</arg>
+                        <arg>-Werror</arg>
+                    </compilerArgs>
+                </configuration>
+            </plugin>
+            <plugin>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <configuration>
+                    <descriptorRefs>
+                        <descriptorRef>jar-with-dependencies</descriptorRef>
+                    </descriptorRefs>
+                </configuration>
+                <executions>
+                    <execution>
+                        <id>make-assembly</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>single</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+</project>
diff --git a/vespaclient-java/src/main/java/com/yahoo/dummyreceiver/DummyReceiver.java b/vespaclient-java/src/main/java/com/yahoo/dummyreceiver/DummyReceiver.java
new file mode 100755
index 00000000000..7b104455f15
--- /dev/null
+++ b/vespaclient-java/src/main/java/com/yahoo/dummyreceiver/DummyReceiver.java
@@ -0,0 +1,196 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.dummyreceiver;
+
+import com.yahoo.concurrent.DaemonThreadFactory;
+import com.yahoo.documentapi.ThroughputLimitQueue;
+import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess;
+import com.yahoo.documentapi.messagebus.MessageBusParams;
+import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet;
+import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage;
+import com.yahoo.documentapi.messagebus.protocol.RemoveDocumentMessage;
+import com.yahoo.documentapi.messagebus.protocol.UpdateDocumentMessage;
+import com.yahoo.log.LogSetup;
+import com.yahoo.messagebus.DestinationSession;
+import com.yahoo.messagebus.EmptyReply;
+import com.yahoo.messagebus.Error;
+import com.yahoo.messagebus.ErrorCode;
+import com.yahoo.messagebus.Message;
+import com.yahoo.messagebus.MessageHandler;
+import com.yahoo.messagebus.Reply;
+import com.yahoo.messagebus.network.Identity;
+import com.yahoo.messagebus.network.rpc.RPCNetworkParams;
+
+import java.util.LinkedList;
+import java.util.List;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingDeque;
+import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+import static java.lang.System.out;
+
+public class DummyReceiver implements MessageHandler {
+    String name = null;
+    DestinationSession session;
+    MessageBusDocumentAccess da;
+    long sleepTime = 0;
+    long messageCount = 0;
+    int maxPendingCount = 0;
+    long silentNum = 0;
+    boolean instant = false;
+    ThreadPoolExecutor executor = null;
+    int threads = 10;
+    long maxQueueTime = -1;
+    BlockingQueue<Runnable> queue;
+    boolean verbose = false;
+
+    DummyReceiver() {
+    }
+
+    public class Task implements Runnable {
+        Reply reply;
+
+        public Task(Reply reply) {
+            this.reply = reply;
+        }
+
+        public void run() {
+            if (sleepTime > 0) {
+                try {
+                    Thread.sleep(sleepTime);
+                } catch (InterruptedException e) {
+                    e.printStackTrace();
+                }
+            }
+            session.reply(reply);
+        }
+    }
+
+    public void init() {
+        MessageBusParams params = new MessageBusParams(new LoadTypeSet());
+        params.setRPCNetworkParams(new RPCNetworkParams().setIdentity(new Identity(name)));
+        params.setDocumentManagerConfigId("client");
+        params.getMessageBusParams().setMaxPendingCount(maxPendingCount);
+        params.getMessageBusParams().setMaxPendingSize(0);
+        da = new MessageBusDocumentAccess(params);
+        queue = (maxQueueTime < 0) ? new LinkedBlockingDeque<>() : new ThroughputLimitQueue<>(maxQueueTime);
+        session = da.getMessageBus().createDestinationSession("default", true, this);
+        executor = new ThreadPoolExecutor(threads, threads, 5, TimeUnit.SECONDS, queue, new DaemonThreadFactory());
+        System.out.println("Registered listener at " + name + "/default with " + maxPendingCount + " max pending and sleep time of " + sleepTime);
+    }
+
+    public void handleMessage(Message message) {
+        messageCount++;
+        if ( silentNum == 0 ) {
+            System.out.println("Received message " + message + ". Received " + messageCount + " messages so far. In queue size " + queue.size());
+
+            if (verbose) {
+                if (message instanceof PutDocumentMessage) {
+                    System.out.println("  Document:\n" + ((PutDocumentMessage) message).getDocumentPut().getDocument().toXML("  "));
+                } else if (message instanceof RemoveDocumentMessage) {
+                    System.out.println("  Document id: " + ((RemoveDocumentMessage) message).getDocumentId());
+                } else if (message instanceof UpdateDocumentMessage) {
+                    System.out.println("  Update:\n  " + ((UpdateDocumentMessage) message).getDocumentUpdate().toString());
+                }
+            }
+        } else {
+            if ((messageCount % silentNum) == 0) {
+                System.out.println("Received " + messageCount + " messages so far. In queue size " + queue.size());
+            }
+        }
+
+        EmptyReply reply = new EmptyReply();
+        message.swapState(reply);
+
+        if ( ! instant ) {
+            try {
+                executor.execute(new Task(reply));
+            } catch (RejectedExecutionException e) {
+                reply.addError(new Error(ErrorCode.SESSION_BUSY, "Session " + name + "/default is busy"));
+                session.reply(reply);
+            }
+        } else {
+            session.reply(reply);
+        }
+    }
+
+    String getParam(List<String> args, String arg) throws IllegalArgumentException {
+        try {
+            return args.remove(0);
+        } catch (Exception e) {
+            System.err.println("--" + arg + " requires an argument");
+            throw new IllegalArgumentException(arg);
+        }
+    }
+
+    public void help() {
+        out.println("Simple receiver for messagebus messages. Prints the messages received to stdout.\n" +
+                    "\n" +
+                    "The options are:\n" +
+                    "  --instant          Reply in message thread." +
+                    "  --name arg         Slobrok name to register\n" +
+                    "  --maxqueuetime arg Adjust the in queue size to have a maximum queue wait period of this many ms (default -1 = unlimited)\n" +
+                    "  --silent #nummsg   Do not dump anything, but progress every #nummsg\n" +
+                    "  --sleeptime arg    The number of milliseconds to sleep per message, to simulate processing time\n" +
+                    "  --threads arg      The number of threads to process the incoming data\n" +
+                    "  --verbose          If set, dump the contents of certain messages to stdout");
+    }
+
+    boolean parseArgs(List<String> args) {
+        try {
+            while (!args.isEmpty()) {
+                String arg = args.remove(0);
+
+                if (arg.equals("-h") || arg.equals("--help")) {
+                    help();
+                    return false;
+                } else if ("--name".equals(arg)) {
+                    name = getParam(args, arg);
+                } else if ("--sleeptime".equals(arg)) {
+                    sleepTime = Long.parseLong(getParam(args, arg));
+                } else if ("--instant".equals(arg)) {
+                    instant = true;
+                } else if ("--silent".equals(arg)) {
+                    silentNum = Long.parseLong(getParam(args, arg));
+                } else if ("--maxqueuetime".equals(arg)) {
+                    maxQueueTime = Long.parseLong(getParam(args, arg));
+                } else if ("--threads".equals(arg)) {
+                    threads = Integer.parseInt(getParam(args, arg));
+                } else if ("--verbose".equals(arg)) {
+                    verbose = true;
+                } else {
+                    help();
+                    return false;
+                }
+            }
+
+            return true;
+        } catch (IllegalArgumentException e) {
+            return false;
+        }
+    }
+
+
+    public static void main(String[] args) {
+        LogSetup.initVespaLogging("dummyreceiver");
+        DummyReceiver rcv = new DummyReceiver();
+
+        List<String> l = new LinkedList<>();
+        for (String arg : args) {
+            l.add(arg);
+        }
+        if (!rcv.parseArgs(l)) {
+            System.exit(1);
+        }
+
+        rcv.init();
+        while (true) {
+            try {
+                Thread.sleep(100);
+            } catch (InterruptedException e) {
+                e.printStackTrace();
+            }
+        }
+    }
+}
diff --git a/vespaclient-java/src/main/java/com/yahoo/vespafeeder/Arguments.java b/vespaclient-java/src/main/java/com/yahoo/vespafeeder/Arguments.java
new file mode 100644
index 00000000000..249ed1fd70c
--- /dev/null
+++ b/vespaclient-java/src/main/java/com/yahoo/vespafeeder/Arguments.java
@@ -0,0 +1,191 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespafeeder;
+
+import com.yahoo.vespa.config.content.LoadTypeConfig;
+import com.yahoo.feedapi.DummySessionFactory;
+import com.yahoo.feedapi.MessageBusSessionFactory;
+import com.yahoo.feedapi.MessagePropertyProcessor;
+import com.yahoo.feedapi.SessionFactory;
+import com.yahoo.vespaclient.config.FeederConfig;
+
+import java.io.BufferedOutputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.LinkedList;
+import java.util.List;
+
+import static java.lang.System.out;
+
+/**
+ * Argument parsing class for the vespa feeder.
+ */
+public class Arguments {
+    public FeederConfig getFeederConfig() {
+        return new FeederConfig(feederConfigBuilder);
+    }
+
+    public List<String> getFiles() {
+        return files;
+    }
+
+    public String getMode() {
+        return mode;
+    }
+
+    public boolean isVerbose() {
+        return verbose;
+    }
+
+    private FeederConfig.Builder feederConfigBuilder = new FeederConfig.Builder();
+    private List<String> files = new ArrayList<String>();
+    private String dumpDocumentsFile = null;
+    private String mode = "standard";
+    private boolean validateOnly = false;
+    private boolean verbose = false;
+    SessionFactory sessionFactory = null;
+    MessagePropertyProcessor propertyProcessor = null;
+    private String priority = null;
+
+    public MessagePropertyProcessor getPropertyProcessor() {
+        return propertyProcessor;
+    }
+
+    public void help() {
+        out.println("This is a tool for feeding xml (deprecated) or json data to a Vespa application.\n" +
+                "\n" +
+                "The options are:\n" +
+                "  --abortondataerror arg (true) Whether or not to abort if the xml input has \n" +
+                "                                errors (true|false).\n" +
+                "  --abortonsenderror arg (true) Whether or not to abort if an error occured while\n" +
+                "                                sending operations to Vespa (true|false).\n" +
+                "  --file arg                    The name of the input files to read. These can \n" +
+                "                                also be passed as arguments without the option \n" +
+                "                                prefix. If none is given, this tool parses \n" +
+                "                                identifiers from standard in.\n" +
+                "  -h [ --help ]                 Shows this help page.\n" +
+                "  --maxpending arg              The maximum number of operations that are allowed\n" +
+                "                                to be pending at any given time. NOTE: This disables dynamic throttling. Use with care.\n" +
+                "  --maxpendingsize arg          The maximum size (in bytes) of operations that \n" +
+                "                                are allowed to be pending at any given time. \n" +
+                "  --maxfeedrate arg             Limits the feed rate to the given number (operations/second). You may still want to increase\n" +
+                "                                the max pending size if your feed rate doesn't reach the desired number.\n" +
+                "  --mode arg (=standard)        The mode to run vespafeeder in (standard | benchmark).\n" +
+                "  --noretry                     Turns off retries of recoverable failures.\n" +
+                "  --retrydelay arg (=1)         The time (in seconds) to wait between retries of \n" +
+                "                                a failed operation.\n" +
+                "  --route arg (=default)        The route to send the data to.\n" +
+                "  --timeout arg (=180)          The time (in seconds) allowed for sending \n" +
+                "                                operations.\n" +
+                "  --trace arg (=0)              The trace level of network traffic.\n" +
+                "  --validate                    Run validation tool on input files instead of \n" +
+                "                                feeding them.\n" +
+                "  --dumpDocuments <filename>    Specify a file where documents in the put are serialized.\n" +
+                "  --priority arg                Specify priority of sent messages (see documentation for priority values)\n" +
+                "  --create-if-non-existent      Enable setting of create-if-non-existent to true on all document updates in the given xml feed.\n" +
+                "  -v [ --verbose ]              Enable verbose output of progress.\n");
+    }
+
+    public class HelpShownException extends Exception {
+
+    }
+
+    public Arguments(String[] argList, SessionFactory factory) throws HelpShownException, FileNotFoundException {
+        parse(argList);
+
+        if (factory != null) {
+            sessionFactory = factory;
+        } else if (validateOnly) {
+            if (dumpDocumentsFile != null) {
+                BufferedOutputStream out = new BufferedOutputStream(new FileOutputStream(dumpDocumentsFile));
+                sessionFactory = new DummySessionFactory(null, out);
+            } else {
+                sessionFactory = new DummySessionFactory(null, null);
+            }
+        } else {
+            sessionFactory = new MessageBusSessionFactory(propertyProcessor);
+        }
+    }
+
+    void parse(String[] argList) throws HelpShownException {
+        List<String> args = new LinkedList<String>();
+        args.addAll(Arrays.asList(argList));
+
+        while (!args.isEmpty()) {
+            String arg = args.remove(0);
+
+            if (arg.equals("-h") || arg.equals("--help")) {
+                help();
+                throw new HelpShownException();
+            } else if ("--abortondataerror".equals(arg)) {
+                feederConfigBuilder.abortondocumenterror(getBoolean(getParam(args, arg)));
+            } else if ("--abortonsenderror".equals(arg)) {
+                feederConfigBuilder.abortonsenderror(getBoolean(getParam(args, arg)));
+            } else if ("--file".equals(arg)) {
+                files.add(getParam(args, arg));
+            } else if ("--maxpending".equals(arg)) {
+                feederConfigBuilder.maxpendingdocs(Integer.parseInt(getParam(args, arg)));
+            } else if ("--maxpendingsize".equals(arg)) {
+                feederConfigBuilder.maxpendingbytes(Integer.parseInt(getParam(args, arg)));
+            } else if ("--mode".equals(arg)) {
+                mode = getParam(args, arg);
+            } else if ("--noretry".equals(arg)) {
+                feederConfigBuilder.retryenabled(false);
+            } else if ("--retrydelay".equals(arg)) {
+                feederConfigBuilder.retrydelay(Integer.parseInt(getParam(args, arg)));
+            } else if ("--route".equals(arg)) {
+                feederConfigBuilder.route(getParam(args, arg));
+            } else if ("--timeout".equals(arg)) {
+                feederConfigBuilder.timeout(Double.parseDouble(getParam(args, arg)));
+            } else if ("--trace".equals(arg)) {
+                feederConfigBuilder.tracelevel(Integer.parseInt(getParam(args, arg)));
+            } else if ("--validate".equals(arg)) {
+                validateOnly = true;
+            } else if ("--dumpDocuments".equals(arg)) {
+                dumpDocumentsFile = getParam(args, arg);
+            } else if ("--maxfeedrate".equals(arg)) {
+                feederConfigBuilder.maxfeedrate(Double.parseDouble(getParam(args, arg)));
+            } else if ("--create-if-non-existent".equals(arg)) {
+                feederConfigBuilder.createifnonexistent(true);
+            } else if ("-v".equals(arg) || "--verbose".equals(arg)) {
+                verbose = true;
+            } else if ("--priority".equals(arg)) {
+                priority = getParam(args, arg);
+            } else {
+                files.add(arg);
+            }
+        }
+
+        propertyProcessor = new MessagePropertyProcessor(getFeederConfig(), new LoadTypeConfig(new LoadTypeConfig.Builder()));
+    }
+
+    private String getParam(List<String> args, String arg) throws IllegalArgumentException {
+        try {
+            return args.remove(0);
+        } catch (Exception e) {
+            System.err.println("--" + arg + " requires an argument");
+            throw new IllegalArgumentException(arg);
+        }
+    }
+
+    private Boolean getBoolean(String arg) {
+        if (arg.equalsIgnoreCase("yes")) {
+            return true;
+        } else if (arg.equalsIgnoreCase("no")) {
+            return false;
+        } else {
+            return Boolean.parseBoolean(arg);
+        }
+    }
+
+    public String getPriority() {
+        return priority;
+    }
+
+    public SessionFactory getSessionFactory() {
+        return sessionFactory;
+    }
+
+
+}
diff --git a/vespaclient-java/src/main/java/com/yahoo/vespafeeder/BenchmarkProgressPrinter.java b/vespaclient-java/src/main/java/com/yahoo/vespafeeder/BenchmarkProgressPrinter.java
new file mode 100644
index 00000000000..cc0d8f8c780
--- /dev/null
+++ b/vespaclient-java/src/main/java/com/yahoo/vespafeeder/BenchmarkProgressPrinter.java
@@ -0,0 +1,76 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespafeeder;
+
+import com.yahoo.clientmetrics.MessageTypeMetricSet;
+import com.yahoo.clientmetrics.RouteMetricSet;
+import com.yahoo.concurrent.Timer;
+import com.yahoo.metrics.Metric;
+import com.yahoo.metrics.MetricSet;
+import com.yahoo.metrics.MetricVisitor;
+
+import java.io.PrintStream;
+
+/**
+ * Class that takes progress from the feed and prints to a stream.
+ */
+public class BenchmarkProgressPrinter implements RouteMetricSet.ProgressCallback {
+    private final long startTime;
+    private final Timer timer;
+    private final PrintStream output;
+
+    public BenchmarkProgressPrinter(Timer timer, PrintStream output) {
+        this.timer = timer;
+        this.output = output;
+        this.startTime = timer.milliTime();
+    }
+
+    class PrintVisitor extends MetricVisitor {
+        private final PrintStream out;
+
+        PrintVisitor(PrintStream out) {
+            this.out = out;
+        }
+
+        @Override
+        public boolean visitMetricSet(MetricSet set, boolean autoGenerated) {
+            if (set instanceof MessageTypeMetricSet && set.getName().equals("total")) {
+                Metric m = set.getMetric("latency");
+                Metric count = set.getMetric("count");
+                Metric err = set.getMetric("errors.total");
+
+                long okCount = 0, errCount = 0, minLatency = 0, maxLatency = 0, avgLatency = 0;
+
+                if (m != null) {
+                    minLatency = m.getLongValue("min");
+                    maxLatency = m.getLongValue("max");
+                    avgLatency = m.getLongValue("average");
+                }
+                if (count != null) {
+                    okCount = count.getLongValue("count");
+                }
+
+                if (err != null) {
+                    errCount = err.getLongValue("count");
+                }
+                long timeUsed = timer.milliTime() - startTime;
+                out.println(timeUsed + ", " + okCount + ", " + errCount + ", " + minLatency + ", " + maxLatency + ", " + avgLatency);
+            }
+            return true;
+        }
+    }
+
+    @Override
+    public void onProgress(RouteMetricSet metrics) {
+	//metrics.visit(new PrintVisitor(output), false);
+    }
+
+    @Override
+    public void done(RouteMetricSet metrics) {
+        try {
+		output.println("# Time used, num ok, num error, min latency, max latency, average latency");
+            metrics.visit(new PrintVisitor(output), false);
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+}
diff --git a/vespaclient-java/src/main/java/com/yahoo/vespafeeder/FileRequest.java b/vespaclient-java/src/main/java/com/yahoo/vespafeeder/FileRequest.java
new file mode 100755
index 00000000000..3479221257d
--- /dev/null
+++ b/vespaclient-java/src/main/java/com/yahoo/vespafeeder/FileRequest.java
@@ -0,0 +1,14 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespafeeder;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+
+public class FileRequest extends InputStreamRequest {
+
+    FileRequest(File f) throws FileNotFoundException {
+        super(new FileInputStream(f));
+    }
+
+}
diff --git a/vespaclient-java/src/main/java/com/yahoo/vespafeeder/InputStreamRequest.java b/vespaclient-java/src/main/java/com/yahoo/vespafeeder/InputStreamRequest.java
new file mode 100644
index 00000000000..e69eb6727b0
--- /dev/null
+++ b/vespaclient-java/src/main/java/com/yahoo/vespafeeder/InputStreamRequest.java
@@ -0,0 +1,38 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespafeeder;
+
+import com.yahoo.container.jdisc.HttpRequest;
+
+import java.io.InputStream;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * This is needed because whoever wrote this library moronically decided to pass in-process communication through
+ * the HTTP layer. As the feeded is being phased out in favor of the standalone HTTP client we don't bother to clean
+ * it up properly.
+ *
+ * @author bratseth
+ */
+public class InputStreamRequest {
+
+    private InputStream input;
+    private Map<String, String> properties = new HashMap<>();
+
+    protected InputStreamRequest(InputStream input) {
+        this.input = input;
+    }
+
+    public void setProperty(String key, String value) {
+        properties.put(key, value);
+    }
+
+    public String getProperty(String key) {
+        return properties.get(key);
+    }
+
+    public HttpRequest toRequest() {
+        return HttpRequest.createTestRequest("", com.yahoo.jdisc.http.HttpRequest.Method.POST, input, properties);
+    }
+
+}
diff --git a/vespaclient-java/src/main/java/com/yahoo/vespafeeder/ProgressPrinter.java b/vespaclient-java/src/main/java/com/yahoo/vespafeeder/ProgressPrinter.java
new file mode 100644
index 00000000000..52087d33a47
--- /dev/null
+++ b/vespaclient-java/src/main/java/com/yahoo/vespafeeder/ProgressPrinter.java
@@ -0,0 +1,149 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespafeeder;
+
+import com.yahoo.clientmetrics.MessageTypeMetricSet;
+import com.yahoo.clientmetrics.RouteMetricSet;
+import com.yahoo.concurrent.Timer;
+import com.yahoo.metrics.Metric;
+import com.yahoo.metrics.MetricSet;
+import com.yahoo.metrics.MetricVisitor;
+import com.yahoo.metrics.SumMetric;
+
+import java.io.IOException;
+import java.io.PrintStream;
+import java.math.RoundingMode;
+import java.text.NumberFormat;
+import java.util.Locale;
+
+/**
+ * Class that takes progress from the feed and prints to a stream.
+ */
+public class ProgressPrinter implements RouteMetricSet.ProgressCallback {
+    private long startTime = 0;
+    private long lastProgressTime = 0;
+    private long lastVerboseProgress = 0;
+    final Timer timer;
+    final PrintStream output;
+
+    public ProgressPrinter(Timer timer, PrintStream output) {
+        this.timer = timer;
+        this.output = output;
+
+        startTime = timer.milliTime();
+        lastProgressTime = startTime;
+        lastVerboseProgress = startTime;
+    }
+
+    class PrintVisitor extends MetricVisitor {
+        final PrintStream out;
+        final NumberFormat format;
+
+        PrintVisitor(PrintStream out) {
+            this.out = out;
+            format = NumberFormat.getNumberInstance(Locale.US);
+            format.setMaximumFractionDigits(2);
+            format.setMinimumFractionDigits(2);
+            format.setMinimumIntegerDigits(1);
+            format.setParseIntegerOnly(false);
+            format.setRoundingMode(RoundingMode.HALF_UP);
+            format.setGroupingUsed(false);
+        }
+
+        @Override
+        public boolean visitMetricSet(MetricSet set, boolean autoGenerated) {
+            if (set instanceof MessageTypeMetricSet && !set.getName().equals("total")) {
+                Metric m = set.getMetric("latency");
+                Metric count = set.getMetric("count");
+                Metric err = set.getMetric("errors.total");
+
+                long okCount = 0, errCount = 0, ignored = 0;
+                long minLatency = 0, maxLatency = 0, avgLatency = 0;
+
+                if (m != null) {
+                    minLatency = m.getLongValue("min");
+                    maxLatency = m.getLongValue("max");
+                    avgLatency = m.getLongValue("average");
+                }
+                if (count != null) {
+                    okCount = count.getLongValue("count");
+                }
+                Metric ignoredMetric = set.getMetric("ignored");
+                if (ignoredMetric != null) {
+                    ignored = ignoredMetric.getLongValue("count");
+                }
+
+                if (err != null) {
+                    errCount = err.getLongValue("count");
+                }
+
+                long timeSinceStart = timer.milliTime() - startTime;
+
+                out.println(((MessageTypeMetricSet)set).getMessageName() + ":\t" +
+                        "ok: " + okCount +
+                        " msgs/sec: " + format.format((double)okCount * 1000 / timeSinceStart) +
+                        " failed: " + errCount +
+                        " ignored: " + ignored +
+                        " latency(min, max, avg): " + minLatency + ", " + maxLatency + ", " + avgLatency);
+            }
+            return true;
+        }
+    }
+
+    public static String getDashes(int count) {
+        String dashes = "";
+        for (int i = 0; i < count; i++) {
+            dashes += "-";
+        }
+
+        return dashes;
+    }
+
+    public synchronized void renderStatusText(RouteMetricSet metrics, PrintStream stream) throws IOException {
+        String headline = "Messages sent to vespa (route " + metrics.getName() + ") :";
+        stream.println(headline);
+        stream.println(getDashes(headline.length()));
+        metrics.visit(new PrintVisitor(stream), false);
+    }
+
+    public long getOkMessageCount(RouteMetricSet metrics) {
+        SumMetric sum = (SumMetric)metrics.getMetric("total");
+
+        MetricSet ms = (MetricSet)sum.generateSum();
+        if (ms != null) {
+            Metric latency = ms.getMetric("latency");
+            if (latency != null) {
+                return latency.getLongValue("count");
+            }
+        }
+
+        return 0;
+    }
+
+    @Override
+    public void onProgress(RouteMetricSet metrics) {
+        try {
+            long timeNow = timer.milliTime();
+
+            if (timeNow - lastVerboseProgress > 30000) {
+                output.println("\n");
+                renderStatusText(metrics, output);
+                lastVerboseProgress = timeNow;
+            } else if (timeNow - lastProgressTime > 1000) {
+                output.print("\rSuccessfully sent " + getOkMessageCount(metrics) + " messages so far");
+                lastProgressTime = timeNow;
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+
+    @Override
+    public void done(RouteMetricSet metrics) {
+        try {
+            output.println("\n");
+            renderStatusText(metrics, output);
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+}
diff --git a/vespaclient-java/src/main/java/com/yahoo/vespafeeder/VespaFeeder.java b/vespaclient-java/src/main/java/com/yahoo/vespafeeder/VespaFeeder.java
new file mode 100755
index 00000000000..a6ede66c43d
--- /dev/null
+++ b/vespaclient-java/src/main/java/com/yahoo/vespafeeder/VespaFeeder.java
@@ -0,0 +1,171 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespafeeder;
+
+import com.yahoo.clientmetrics.RouteMetricSet;
+import com.yahoo.concurrent.ThreadFactoryFactory;
+import com.yahoo.document.DocumentTypeManager;
+import com.yahoo.document.DocumentTypeManagerConfigurer;
+import com.yahoo.feedapi.FeedContext;
+import com.yahoo.feedhandler.FeedResponse;
+import com.yahoo.feedhandler.NullFeedMetric;
+import com.yahoo.feedhandler.VespaFeedHandler;
+import com.yahoo.log.LogSetup;
+import com.yahoo.concurrent.SystemTimer;
+import com.yahoo.vespaclient.ClusterList;
+
+import java.io.*;
+import java.util.*;
+import java.util.concurrent.Executor;
+import java.util.concurrent.Executors;
+
+public class VespaFeeder {
+
+    Arguments args;
+    DocumentTypeManager manager;
+    Executor threadPool = Executors.newCachedThreadPool(ThreadFactoryFactory.getThreadFactory("vespafeeder"));
+
+    public VespaFeeder(Arguments args, DocumentTypeManager manager) {
+        this.args = args;
+        this.manager = manager;
+    }
+
+    public static class FeedErrorException extends Exception {
+        String message;
+
+        public FeedErrorException(String message) {
+            this.message = message;
+        }
+
+        @Override
+        public String getMessage() {
+            return message;
+        }
+
+    }
+
+    static FeedErrorException renderErrors(List<String> errors) {
+        StringBuilder buffer = new StringBuilder();
+
+        if (!errors.isEmpty()) {
+            String headline = (errors.size() > 10) ? "First 10 errors (of " + errors.size() + "):" : "Errors:";
+            buffer.append(headline).append("\n");
+            for (int i = 0; i < headline.length(); ++i) {
+                buffer.append("-");
+            }
+            buffer.append("\n");
+            for (int i = 0; i < errors.size() && i < 10; ++i) {
+                buffer.append("    ").append(errors.get(i)).append("\n");
+            }
+        }
+
+        return new FeedErrorException(buffer.toString());
+    }
+
+    public RouteMetricSet.ProgressCallback createProgressCallback(PrintStream output) {
+        if ("benchmark".equals(args.getMode())) {
+            return new BenchmarkProgressPrinter(SystemTimer.INSTANCE, output);
+        } else {
+            return new ProgressPrinter(SystemTimer.INSTANCE, output);
+        }
+    }
+
+    void parseFiles(InputStream stdin, PrintStream output) throws Exception {
+        FeedContext context = new FeedContext(
+                args.getPropertyProcessor(),
+                args.getSessionFactory(),
+                manager,
+                new ClusterList(), new NullFeedMetric());
+
+        final BufferedInputStream input = new BufferedInputStream(stdin);
+        VespaFeedHandler handler = VespaFeedHandler.createFromContext(context, threadPool);
+
+        if (args.getFiles().isEmpty()) {
+            InputStreamRequest req = new InputStreamRequest(input);
+            setProperties(req, input);
+            FeedResponse response = (FeedResponse)handler.handle(req.toRequest(), createProgressCallback(output));
+            if ( ! response.isSuccess()) {
+                throw renderErrors(response.getErrorList());
+            }
+        } else {
+            if (args.isVerbose()) {
+                for (String fileName : args.getFiles()) {
+                    long thisSize = new File(fileName).length();
+                    output.println("Size of file '" + fileName + "' is " + thisSize + " B.");
+                }
+            }
+
+            for (String fileName : args.getFiles()) {
+                File f = new File(fileName);
+                FileRequest req = new FileRequest(f);
+                final BufferedInputStream inputSnooper = new BufferedInputStream(new FileInputStream(fileName));
+                setProperties(req, inputSnooper);
+                inputSnooper.close();
+                FeedResponse response = (FeedResponse)handler.handle(req.toRequest(), createProgressCallback(output));
+                if (!response.isSuccess()) {
+                    throw renderErrors(response.getErrorList());
+                }
+            }
+        }
+    }
+
+    // use BufferedInputStream to enforce the input.markSupported() == true
+    private void setProperties(InputStreamRequest req, BufferedInputStream input) throws IOException {
+        setPriority(req);
+        setCreateIfNonExistent(req);
+        setJsonInput(req, input);
+    }
+
+    private void setPriority(InputStreamRequest req) {
+        if (args.getPriority() != null) {
+            req.setProperty("priority", args.getPriority());
+        }
+    }
+
+    private void setCreateIfNonExistent(InputStreamRequest req) {
+        if (args.getFeederConfig().createifnonexistent()) {
+            req.setProperty("createifnonexistent", "true");
+        }
+    }
+
+    // package access for easy testing
+    static void setJsonInput(InputStreamRequest req, BufferedInputStream input) throws IOException {
+        input.mark(4);
+        int b = input.read();
+        input.reset();
+        // A valid JSON feed will always start with '['
+        if (b == '[') {
+            req.setProperty(VespaFeedHandler.JSON_INPUT, Boolean.TRUE.toString());
+        } else {
+            req.setProperty(VespaFeedHandler.JSON_INPUT, Boolean.FALSE.toString());
+        }
+    }
+
+    public static void main(String[] args) {
+        LogSetup.initVespaLogging("vespafeeder");
+
+        try {
+            Arguments arguments = new Arguments(args, null);
+
+            DocumentTypeManager manager = new DocumentTypeManager();
+            DocumentTypeManagerConfigurer.configure(manager, "client").close();
+
+            VespaFeeder feeder = new VespaFeeder(arguments, manager);
+            feeder.parseFiles(System.in, System.out);
+            System.exit(0);
+        } catch (Arguments.HelpShownException e) {
+            System.exit(0);
+        } catch (IllegalArgumentException e) {
+            System.exit(1);
+        } catch (FileNotFoundException e) {
+            System.err.println("Could not open file " + e.getMessage());
+            System.exit(1);
+        } catch (FeedErrorException e) {
+            System.err.println("\n" + e.getMessage());
+            System.exit(1);
+        } catch (Exception e) {
+            System.err.println("Got exception " + e.getMessage() + ", aborting feed.");
+            System.exit(1);
+        }
+    }
+
+}
diff --git a/vespaclient-java/src/main/java/com/yahoo/vespaget/ClientParameters.java b/vespaclient-java/src/main/java/com/yahoo/vespaget/ClientParameters.java
new file mode 100644
index 00000000000..b57a9f7bf85
--- /dev/null
+++ b/vespaclient-java/src/main/java/com/yahoo/vespaget/ClientParameters.java
@@ -0,0 +1,160 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespaget;
+
+import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol;
+
+import java.util.Iterator;
+/**
+ * This class contains the the program parameters.
+ *
+ * @author bjorncs
+ */
+public class ClientParameters {
+
+    // Determines if the help page should be presented
+    public final boolean help;
+    // Contains the document ids. Is backed by either a list iterator if the ids were given as CLI arguments or Scanner(System.in) if ids are provided by standard input.
+    public final Iterator<String> documentIds;
+    // Print only the id for retrieved documents
+    public final boolean printIdsOnly;
+    // Determines which document fields to retrieve. Default is '[all]'.
+    public final String fieldSet;
+    // The Vespa route
+    public final String route;
+    // Alternative way to specify the route using cluster name.
+    public final String cluster;
+    // The configuration id for message bus. Default "client".
+    public final String configId;
+    // Determines if the serialized document size should be printed
+    public final boolean showDocSize;
+    // Document request timeout
+    public final double timeout;
+    // Determines whether or not the document request can be resent
+    public final boolean noRetry;
+    // Vespa trace level
+    public final int traceLevel;
+    // Document request priority
+    public final DocumentProtocol.Priority priority;
+    // Determines the Vespa load type
+    public final String loadTypeName;
+    // If full documents are printed, they will be printed as JSON (instead of XML)
+    public final boolean jsonOutput;
+
+
+    private ClientParameters(
+            boolean help, Iterator<String> documentIds, boolean printIdsOnly,
+            String fieldSet, String route, String cluster, String configId,
+            boolean showDocSize, double timeout, boolean noRetry, int traceLevel,
+            DocumentProtocol.Priority priority, String loadTypeName, boolean jsonOutput) {
+
+        this.help = help;
+        this.documentIds = documentIds;
+        this.printIdsOnly = printIdsOnly;
+        this.fieldSet = fieldSet;
+        this.route = route;
+        this.cluster = cluster;
+        this.configId = configId;
+        this.showDocSize = showDocSize;
+        this.timeout = timeout;
+        this.noRetry = noRetry;
+        this.traceLevel = traceLevel;
+        this.priority = priority;
+        this.loadTypeName = loadTypeName;
+        this.jsonOutput = jsonOutput;
+    }
+
+    public static class Builder {
+        private boolean help;
+        private Iterator<String> documentIds;
+        private boolean printIdsOnly;
+        private String fieldSet;
+        private String route;
+        private String cluster;
+        private String configId;
+        private boolean showDocSize;
+        private double timeout;
+        private boolean noRetry;
+        private int traceLevel;
+        private DocumentProtocol.Priority priority;
+        private String loadTypeName;
+        private boolean jsonOutput;
+
+        public Builder setHelp(boolean help) {
+            this.help = help;
+            return this;
+        }
+
+        public Builder setDocumentIds(Iterator<String> documentIds) {
+            this.documentIds = documentIds;
+            return this;
+        }
+
+        public Builder setPrintIdsOnly(boolean printIdsOnly) {
+            this.printIdsOnly = printIdsOnly;
+            return this;
+        }
+
+        public Builder setFieldSet(String fieldSet) {
+            this.fieldSet = fieldSet;
+            return this;
+        }
+
+        public Builder setRoute(String route) {
+            this.route = route;
+            return this;
+        }
+
+        public Builder setCluster(String cluster) {
+            this.cluster = cluster;
+            return this;
+        }
+
+        public Builder setConfigId(String configId) {
+            this.configId = configId;
+            return this;
+        }
+
+        public Builder setShowDocSize(boolean showDocSize) {
+            this.showDocSize = showDocSize;
+            return this;
+        }
+
+        public Builder setTimeout(double timeout) {
+            this.timeout = timeout;
+            return this;
+        }
+
+        public Builder setNoRetry(boolean noRetry) {
+            this.noRetry = noRetry;
+            return this;
+        }
+
+        public Builder setTraceLevel(int traceLevel) {
+            this.traceLevel = traceLevel;
+            return this;
+        }
+
+        public Builder setPriority(DocumentProtocol.Priority priority) {
+            this.priority = priority;
+            return this;
+        }
+
+        public Builder setLoadTypeName(String loadTypeName) {
+            this.loadTypeName = loadTypeName;
+            return this;
+        }
+
+        public Builder setJsonOutput(boolean jsonOutput) {
+            this.jsonOutput = jsonOutput;
+            return this;
+        }
+
+        public ClientParameters build() {
+            return new ClientParameters(
+                    help, documentIds, printIdsOnly, fieldSet, route, cluster, configId,
+                    showDocSize, timeout, noRetry, traceLevel, priority, loadTypeName, jsonOutput);
+        }
+    }
+
+
+}
diff --git a/vespaclient-java/src/main/java/com/yahoo/vespaget/CommandLineOptions.java b/vespaclient-java/src/main/java/com/yahoo/vespaget/CommandLineOptions.java
new file mode 100644
index 00000000000..cbaef17a70a
--- /dev/null
+++ b/vespaclient-java/src/main/java/com/yahoo/vespaget/CommandLineOptions.java
@@ -0,0 +1,263 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespaget;
+
+import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.DefaultParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+
+import java.io.InputStream;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Scanner;
+
+/**
+ * This class is responsible for parsing the command line arguments and print the help page.
+ *
+ * @author bjorncs
+ */
+public class CommandLineOptions {
+
+    public static final String HELP_OPTION = "help";
+    public static final String PRINTIDS_OPTION = "printids";
+    public static final String HEADERSONLY_OPTION = "headersonly";
+    public static final String FIELDSET_OPTION = "fieldset";
+    public static final String CLUSTER_OPTION = "cluster";
+    public static final String ROUTE_OPTION = "route";
+    public static final String CONFIGID_OPTION = "configid";
+    public static final String SHOWDOCSIZE_OPTION = "showdocsize";
+    public static final String TIMEOUT_OPTION = "timeout";
+    public static final String NORETRY_OPTION = "noretry";
+    public static final String TRACE_OPTION = "trace";
+    public static final String PRIORITY_OPTION = "priority";
+    public static final String LOADTYPE_OPTION = "loadtype";
+    public static final String JSONOUTPUT_OPTION = "jsonoutput";
+
+    private final Options options = createOptions();
+    private final InputStream stdIn;
+
+    public CommandLineOptions(InputStream stdIn) {
+        this.stdIn = stdIn;
+    }
+
+    public CommandLineOptions() {
+        this(System.in);
+    }
+
+    @SuppressWarnings("AccessStaticViaInstance")
+    private static Options createOptions() {
+        Options options = new Options();
+
+        options.addOption(Option.builder("h")
+                .hasArg(false)
+                .desc("Show this syntax page.")
+                .longOpt(HELP_OPTION)
+                .build());
+
+        options.addOption(Option.builder("i")
+                .hasArg(false)
+                .desc("Show only identifiers of retrieved documents.")
+                .longOpt(PRINTIDS_OPTION)
+                .build());
+
+        options.addOption(Option.builder("e")
+                .hasArg(false)
+                .desc("Retrieve header fields only. [Deprecated].")
+                .longOpt(HEADERSONLY_OPTION).build());
+
+        options.addOption(Option.builder("f")
+                .hasArg(true)
+                .desc("Retrieve the specified fields only (see http://vespa.corp.yahoo.com/5/documentation/reference/fieldsets.html) (default '[all]')")
+                .longOpt(FIELDSET_OPTION)
+                .argName("fieldset").build());
+
+        options.addOption(Option.builder("u")
+                .hasArg(true)
+                .desc("Send request to the given content cluster.")
+                .longOpt(CLUSTER_OPTION)
+                .argName("cluster").build());
+
+        options.addOption(Option.builder("r")
+                .hasArg(true)
+                .desc("Send request to the given messagebus route.")
+                .longOpt(ROUTE_OPTION)
+                .argName("route").build());
+
+        options.addOption(Option.builder("c")
+                .hasArg(true)
+                .desc("Use the specified config id for messagebus configuration.")
+                .longOpt(CONFIGID_OPTION)
+                .argName("configid").build());
+
+        options.addOption(Option.builder("s")
+                .hasArg(false)
+                .desc("Show binary size of document.")
+                .longOpt(SHOWDOCSIZE_OPTION).build());
+
+        options.addOption(Option.builder("t")
+                .hasArg(true)
+                .desc("Set timeout for the request in seconds (default 0).")
+                .longOpt(TIMEOUT_OPTION)
+                .argName("timeout")
+                .type(Number.class).build());
+
+        options.addOption(Option.builder("n")
+                .hasArg(false)
+                .desc("Do not retry operation on transient errors, as is default.")
+                .longOpt(NORETRY_OPTION).build());
+
+        options.addOption(Option.builder("a")
+                .hasArg(true)
+                .desc("Trace level to use (default 0).")
+                .longOpt(TRACE_OPTION)
+                .argName("trace")
+                .type(Number.class).build());
+
+        options.addOption(Option.builder("p")
+                .hasArg(true)
+                .desc("Priority (default 6).")
+                .longOpt(PRIORITY_OPTION)
+                .argName("priority")
+                .type(Number.class).build());
+
+        options.addOption(Option.builder("l")
+                .hasArg(true)
+                .desc("Load type (default \"\").")
+                .longOpt(LOADTYPE_OPTION)
+                .argName("loadtype").build());
+
+        options.addOption(Option.builder("j")
+                .hasArg(false)
+                .desc("JSON output")
+                .longOpt(JSONOUTPUT_OPTION).build());
+
+        return options;
+    }
+
+    public void printHelp() {
+        HelpFormatter formatter = new HelpFormatter();
+
+        formatter.printHelp(
+                "vespaget <options> [documentid...]", "Fetch a document from a Vespa Content cluster.", options,
+                "If one or more document identifier are specified, these documents will be " +
+                        "retrieved. Otherwise, document identifiers (separated with line break) will be read from standard in.\n",
+                false);
+    }
+
+    public ClientParameters parseCommandLineArguments(String[] args) throws IllegalArgumentException {
+        try {
+            CommandLineParser clp = new DefaultParser();
+            CommandLine cl = clp.parse(options, args);
+
+            boolean printIdsOnly = cl.hasOption(PRINTIDS_OPTION);
+            boolean headersOnly = cl.hasOption(HEADERSONLY_OPTION);
+            String fieldSet = cl.getOptionValue(FIELDSET_OPTION, "");
+            String cluster = cl.getOptionValue(CLUSTER_OPTION, "");
+            String route = cl.getOptionValue(ROUTE_OPTION, "");
+            String configId = cl.getOptionValue(CONFIGID_OPTION, "");
+            boolean help = cl.hasOption(HELP_OPTION);
+            String loadtype = cl.getOptionValue(LOADTYPE_OPTION, "");
+            boolean noRetry = cl.hasOption(NORETRY_OPTION);
+            boolean showDocSize = cl.hasOption(SHOWDOCSIZE_OPTION);
+            boolean jsonOutput = cl.hasOption(JSONOUTPUT_OPTION);
+            int trace = getTrace(cl);
+            DocumentProtocol.Priority priority = getPriority(cl);
+            double timeout = getTimeout(cl);
+            Iterator<String> documentIds = getDocumentIds(cl);
+
+            if (printIdsOnly && headersOnly) {
+                throw new IllegalArgumentException("Print ids and headers only options are mutually exclusive.");
+            }
+            if ((printIdsOnly || headersOnly) && !fieldSet.isEmpty()) {
+                throw new IllegalArgumentException("Field set option can not be used in combination with print ids or headers only options.");
+            }
+
+            if (printIdsOnly) {
+                fieldSet = "[id]";
+            } else if (headersOnly) {
+                fieldSet = "[header]";
+            } else if (fieldSet.isEmpty()) {
+                fieldSet = "[all]";
+            }
+
+            if (!cluster.isEmpty() && !route.isEmpty()) {
+                throw new IllegalArgumentException("Cluster and route options are mutually exclusive.");
+            }
+
+            if (route.isEmpty() && cluster.isEmpty()) {
+                route = "default";
+            }
+
+            if (trace < 0 || trace > 9) {
+                throw new IllegalArgumentException("Invalid tracelevel: " + trace);
+            }
+
+            if (configId.isEmpty()) {
+                configId = "client";
+            }
+
+            ClientParameters.Builder paramsBuilder = new ClientParameters.Builder();
+            return paramsBuilder
+                    .setDocumentIds(documentIds)
+                    .setConfigId(configId)
+                    .setFieldSet(fieldSet)
+                    .setHelp(help)
+                    .setPrintIdsOnly(printIdsOnly)
+                    .setLoadTypeName(loadtype)
+                    .setNoRetry(noRetry)
+                    .setCluster(cluster)
+                    .setRoute(route)
+                    .setShowDocSize(showDocSize)
+                    .setTraceLevel(trace)
+                    .setPriority(priority)
+                    .setTimeout(timeout)
+                    .setJsonOutput(jsonOutput)
+                    .build();
+        } catch (ParseException pe) {
+            throw new IllegalArgumentException(pe.getMessage());
+        }
+    }
+
+    private Iterator<String> getDocumentIds(CommandLine cl) {
+        // Fetch document ids from stdin if no ids are passed in as command line arguments
+        List<String> documentIds = Arrays.asList(cl.getArgs());
+        // WARNING: CommandLine.getArgs may return a single empty string as the only element
+        if (documentIds.isEmpty() ||
+                documentIds.size() == 1 && documentIds.get(0).isEmpty()) {
+            return new Scanner(stdIn);
+        } else {
+            return documentIds.iterator();
+        }
+    }
+
+    private static double getTimeout(CommandLine cl) throws ParseException {
+        Number timeoutObj = (Number) cl.getParsedOptionValue(TIMEOUT_OPTION);
+        return timeoutObj != null ? timeoutObj.doubleValue() : 0;
+    }
+
+    private static int getTrace(CommandLine cl) throws ParseException {
+        Number traceObj = (Number) cl.getParsedOptionValue(TRACE_OPTION);
+        return traceObj != null ? traceObj.intValue() : 0;
+    }
+
+    private static DocumentProtocol.Priority getPriority(CommandLine cl) throws ParseException {
+        Number priorityObj = (Number) cl.getParsedOptionValue(PRIORITY_OPTION);
+        int priorityNumber = priorityObj != null ? priorityObj.intValue() : DocumentProtocol.Priority.NORMAL_2.getValue();
+        return parsePriority(priorityNumber);
+    }
+
+    private static DocumentProtocol.Priority parsePriority(int n) {
+        for (DocumentProtocol.Priority priority : DocumentProtocol.Priority.values()) {
+            if (priority.getValue() == n) {
+                return priority;
+            }
+        }
+        throw new IllegalArgumentException("Invalid priority: " + n);
+    }
+
+}
diff --git a/vespaclient-java/src/main/java/com/yahoo/vespaget/DocumentAccessFactory.java b/vespaclient-java/src/main/java/com/yahoo/vespaget/DocumentAccessFactory.java
new file mode 100644
index 00000000000..6836f033c11
--- /dev/null
+++ b/vespaclient-java/src/main/java/com/yahoo/vespaget/DocumentAccessFactory.java
@@ -0,0 +1,17 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespaget;
+
+import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess;
+import com.yahoo.documentapi.messagebus.MessageBusParams;
+
+/**
+ * Factory class for {@link com.yahoo.documentapi.messagebus.MessageBusDocumentAccess}.
+ *
+ * @author bjorncs
+ */
+public class DocumentAccessFactory {
+
+    public MessageBusDocumentAccess createDocumentAccess(MessageBusParams messageBusParams) {
+        return new MessageBusDocumentAccess(messageBusParams);
+    }
+}
diff --git a/vespaclient-java/src/main/java/com/yahoo/vespaget/DocumentRetriever.java b/vespaclient-java/src/main/java/com/yahoo/vespaget/DocumentRetriever.java
new file mode 100644
index 00000000000..6e52e89c580
--- /dev/null
+++ b/vespaclient-java/src/main/java/com/yahoo/vespaget/DocumentRetriever.java
@@ -0,0 +1,207 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespaget;
+
+import com.yahoo.document.Document;
+import com.yahoo.document.DocumentId;
+import com.yahoo.document.json.JsonWriter;
+import com.yahoo.documentapi.SyncParameters;
+import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess;
+import com.yahoo.documentapi.messagebus.MessageBusParams;
+import com.yahoo.documentapi.messagebus.MessageBusSyncSession;
+import com.yahoo.documentapi.messagebus.loadtypes.LoadType;
+import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet;
+import com.yahoo.documentapi.messagebus.protocol.GetDocumentMessage;
+import com.yahoo.documentapi.messagebus.protocol.GetDocumentReply;
+import com.yahoo.messagebus.Message;
+import com.yahoo.messagebus.Reply;
+import com.yahoo.messagebus.Trace;
+import com.yahoo.text.Utf8;
+import com.yahoo.vespaclient.ClusterDef;
+import com.yahoo.vespaclient.ClusterList;
+
+import java.util.Iterator;
+import java.util.Map;
+
+/**
+ * The document retriever is responsible for retrieving documents using the Document API and printing the result to standard out.
+ *
+ * @author bjorncs
+ */
+public class DocumentRetriever {
+
+    private final ClusterList clusterList;
+    private final DocumentAccessFactory documentAccessFactory;
+    private final ClientParameters params;
+    private final LoadTypeSet loadTypeSet;
+
+    private MessageBusSyncSession session;
+    private MessageBusDocumentAccess documentAccess;
+
+    public DocumentRetriever(ClusterList clusterList,
+                             DocumentAccessFactory documentAccessFactory,
+                             LoadTypeSet loadTypeSet,
+                             ClientParameters params) {
+        this.clusterList = clusterList;
+        this.documentAccessFactory = documentAccessFactory;
+        this.loadTypeSet = loadTypeSet;
+        this.params = params;
+    }
+
+    public void shutdown() {
+        try {
+            if (session != null) {
+                session.destroy();
+            }
+        } catch (IllegalStateException e) {
+            // Ignore exception on shutdown
+        }
+        try {
+            if (documentAccess != null) {
+                documentAccess.shutdown();
+            }
+        } catch (IllegalStateException e) {
+            // Ignore exception on shutdown
+        }
+    }
+
+    public void retrieveDocuments() throws DocumentRetrieverException {
+        boolean first = true;
+        String route = params.cluster.isEmpty() ? params.route : resolveClusterRoute(params.cluster);
+        LoadType loadType = params.loadTypeName.isEmpty() ? null : resolveLoadType(params.loadTypeName);
+
+        MessageBusParams messageBusParams = createMessageBusParams(params.configId, params.timeout, route);
+        documentAccess = documentAccessFactory.createDocumentAccess(messageBusParams);
+        session = documentAccess.createSyncSession(new SyncParameters());
+        int trace = params.traceLevel;
+        if (trace > 0) {
+            session.setTraceLevel(trace);
+        }
+
+        Iterator<String> iter = params.documentIds;
+        if (params.jsonOutput && !params.printIdsOnly) {
+            System.out.println('[');
+        }
+        while (iter.hasNext()) {
+            if (params.jsonOutput && !params.printIdsOnly) {
+                if (!first) {
+                    System.out.println(',');
+                } else {
+                    first = false;
+                }
+            }
+            String docid = iter.next();
+            Message msg = createDocumentRequest(docid, loadType);
+            Reply reply = session.syncSend(msg);
+            printReply(reply);
+        }
+        if (params.jsonOutput && !params.printIdsOnly) {
+            System.out.println(']');
+        }
+    }
+
+    private String resolveClusterRoute(String clusterName) throws DocumentRetrieverException {
+        if (clusterList.getStorageClusters().isEmpty()) {
+            throw new DocumentRetrieverException("The Vespa cluster does not have any content clusters declared.");
+        }
+
+        ClusterDef clusterDef = null;
+        for (ClusterDef c : clusterList.getStorageClusters()) {
+            if (c.getName().equals(clusterName)) {
+                clusterDef = c;
+            }
+        }
+        if (clusterDef == null) {
+            String names = createClusterNamesString();
+            throw new DocumentRetrieverException(String.format(
+                    "The Vespa cluster contains the content clusters %s, not %s. Please select a valid vespa cluster.",
+                    names, clusterName));
+        }
+        return String.format("[Storage:cluster=%s;clusterconfigid=%s]", clusterDef.getName(), clusterDef.getConfigId());
+    }
+
+    private LoadType resolveLoadType(String loadTypeName) throws DocumentRetrieverException {
+        Map<String, LoadType> loadTypesNameMap = loadTypeSet.getNameMap();
+        if (!loadTypesNameMap.containsKey(loadTypeName)) {
+            throw new DocumentRetrieverException(String.format("Loadtype with name '%s' does not exist.\n", loadTypeName));
+        } else {
+            return loadTypesNameMap.get(loadTypeName);
+        }
+    }
+
+    private MessageBusParams createMessageBusParams(String configId, double timeout, String route) {
+        MessageBusParams messageBusParams = new MessageBusParams(loadTypeSet);
+        messageBusParams.setRoute(route);
+        messageBusParams.setProtocolConfigId(configId);
+        messageBusParams.setRoutingConfigId(configId);
+        messageBusParams.setDocumentManagerConfigId(configId);
+
+        if (timeout > 0) {
+            messageBusParams.getSourceSessionParams().setTimeout(timeout);
+        }
+        return messageBusParams;
+    }
+
+    private Message createDocumentRequest(String docid, LoadType loadType) {
+        GetDocumentMessage msg = new GetDocumentMessage(new DocumentId(docid), params.fieldSet);
+        msg.setPriority(params.priority);
+        msg.setRetryEnabled(!params.noRetry);
+
+        if (loadType != null) {
+            msg.setLoadType(loadType);
+        }
+        return msg;
+    }
+
+    private void printReply(Reply reply) {
+        Trace trace = reply.getTrace();
+        if (!trace.getRoot().isEmpty()) {
+            System.out.println(trace);
+        }
+
+        if (reply.hasErrors()) {
+            System.err.print("Request failed: ");
+            for (int i = 0; i < reply.getNumErrors(); i++) {
+                System.err.printf("\n  %s", reply.getError(i));
+            }
+            System.err.println();
+            return;
+        }
+
+        if (!(reply instanceof GetDocumentReply)) {
+            System.err.printf("Unexpected reply %s: '%s'\n", reply.getType(), reply.toString());
+            return;
+        }
+
+        GetDocumentReply documentReply = (GetDocumentReply) reply;
+        Document document = documentReply.getDocument();
+
+        if (document == null) {
+            System.out.println("Document not found.");
+            return;
+        }
+
+        if (params.showDocSize) {
+            System.out.printf("Document size: %d bytes.\n", document.getSerializedSize());
+        }
+        if (params.printIdsOnly) {
+            System.out.println(document.getId());
+        } else {
+            if (params.jsonOutput) {
+                System.out.print(Utf8.toString(JsonWriter.toByteArray(document)));
+            } else {
+                System.out.print(document.toXML("  "));
+            }
+        }
+    }
+
+    private String createClusterNamesString() {
+        StringBuilder names = new StringBuilder();
+        for (ClusterDef c : clusterList.getStorageClusters()) {
+            if (names.length() > 0) {
+                names.append(", ");
+            }
+            names.append(c.getName());
+        }
+        return names.toString();
+    }
+}
diff --git a/vespaclient-java/src/main/java/com/yahoo/vespaget/DocumentRetrieverException.java b/vespaclient-java/src/main/java/com/yahoo/vespaget/DocumentRetrieverException.java
new file mode 100644
index 00000000000..4cf0e9885a3
--- /dev/null
+++ b/vespaclient-java/src/main/java/com/yahoo/vespaget/DocumentRetrieverException.java
@@ -0,0 +1,14 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespaget;
+
+/**
+ * Exception thrown by {@link DocumentRetriever}.
+ *
+ * @author bjorncs
+ */
+public class DocumentRetrieverException extends Exception {
+
+    public DocumentRetrieverException(String message) {
+        super(message);
+    }
+}
diff --git a/vespaclient-java/src/main/java/com/yahoo/vespaget/Main.java b/vespaclient-java/src/main/java/com/yahoo/vespaget/Main.java
new file mode 100644
index 00000000000..324107d8909
--- /dev/null
+++ b/vespaclient-java/src/main/java/com/yahoo/vespaget/Main.java
@@ -0,0 +1,46 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespaget;
+
+
+import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet;
+import com.yahoo.vespaclient.ClusterList;
+
+/**
+ * The vespaget tool retrieves documents from a Vespa Document Storage cluster, and prints them to stdout as XML.
+ *
+ * @author bjorncs
+ */
+public class Main {
+
+    public static void main(String[] args) {
+        try {
+            CommandLineOptions options = new CommandLineOptions();
+            ClientParameters params = options.parseCommandLineArguments(args);
+
+            if (params.help) {
+                options.printHelp();
+            } else {
+                DocumentRetriever documentRetriever = createDocumentRetriever(params);
+                addShutdownHook(documentRetriever);
+                documentRetriever.retrieveDocuments();
+            }
+        } catch (IllegalArgumentException e) {
+            System.err.printf("Failed to parse command line arguments: %s.\n", e.getMessage());
+        } catch (DocumentRetrieverException e) {
+            System.err.printf("Failed to retrieve documents: %s\n", e.getMessage());
+        }
+    }
+
+    private static void addShutdownHook(DocumentRetriever documentRetriever) {
+        Runtime.getRuntime().addShutdownHook(new Thread(documentRetriever::shutdown));
+    }
+
+    private static DocumentRetriever createDocumentRetriever(ClientParameters params) {
+        return new DocumentRetriever(
+                new ClusterList("client"),
+                new DocumentAccessFactory(),
+                new LoadTypeSet(params.configId),
+                params
+        );
+    }
+}
diff --git a/vespaclient-java/src/main/java/com/yahoo/vespastat/BucketStatsException.java b/vespaclient-java/src/main/java/com/yahoo/vespastat/BucketStatsException.java
new file mode 100644
index 00000000000..a6f471e7b5e
--- /dev/null
+++ b/vespaclient-java/src/main/java/com/yahoo/vespastat/BucketStatsException.java
@@ -0,0 +1,18 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespastat;
+
+/**
+ * Exception class used by {@link com.yahoo.vespastat.BucketStatsRetriever}.
+ *
+ * @author bjorncs
+ */
+public class BucketStatsException extends Exception {
+    public BucketStatsException(String message) {
+        super(message);
+    }
+
+    public BucketStatsException(String message, Throwable cause) {
+        super(message, cause);
+    }
+
+}
diff --git a/vespaclient-java/src/main/java/com/yahoo/vespastat/BucketStatsPrinter.java b/vespaclient-java/src/main/java/com/yahoo/vespastat/BucketStatsPrinter.java
new file mode 100644
index 00000000000..a4a263188b5
--- /dev/null
+++ b/vespaclient-java/src/main/java/com/yahoo/vespastat/BucketStatsPrinter.java
@@ -0,0 +1,59 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespastat;
+
+import com.yahoo.document.BucketId;
+import com.yahoo.documentapi.messagebus.protocol.GetBucketListReply;
+
+import java.io.PrintStream;
+import java.util.List;
+
+/**
+ * The class is responsible for printing bucket information to a printstream.
+ *
+ * @author bjorncs
+ */
+public class BucketStatsPrinter {
+    private final BucketStatsRetriever retriever;
+    private final PrintStream out;
+
+    public BucketStatsPrinter(
+            BucketStatsRetriever retriever,
+            PrintStream out) {
+        this.retriever = retriever;
+        this.out = out;
+    }
+
+    public void retrieveAndPrintBucketStats(ClientParameters.SelectionType type, String id, boolean dumpData) throws BucketStatsException {
+        BucketId bucketId = retriever.getBucketIdForType(type, id);
+        if (type == ClientParameters.SelectionType.GROUP || type == ClientParameters.SelectionType.USER) {
+            out.printf("Generated 32-bit bucket id: %s\n", bucketId);
+        }
+
+        List<GetBucketListReply.BucketInfo> bucketList = retriever.retrieveBucketList(bucketId);
+        printBucketList(bucketList);
+
+        if (dumpData) {
+            for (GetBucketListReply.BucketInfo bucketInfo : bucketList) {
+                BucketId bucket = bucketInfo.getBucketId();
+                String bucketStats = retriever.retrieveBucketStats(type, id, bucket);
+                printBucketStats(bucket, bucketStats);
+            }
+        }
+    }
+
+    private void printBucketList(List<GetBucketListReply.BucketInfo> bucketList) {
+        if (bucketList.isEmpty()) {
+            out.println("No actual files were stored for this bucket.");
+        } else {
+            out.println("Bucket maps to the following actual files:");
+            for (GetBucketListReply.BucketInfo bucketInfo : bucketList) {
+                out.printf("\t%s\n", bucketInfo);
+            }
+        }
+    }
+
+    private void printBucketStats(BucketId bucket, String stats) {
+        out.printf("\nDetails for %s:\n%s", bucket, stats);
+    }
+
+}
diff --git a/vespaclient-java/src/main/java/com/yahoo/vespastat/BucketStatsRetriever.java b/vespaclient-java/src/main/java/com/yahoo/vespastat/BucketStatsRetriever.java
new file mode 100644
index 00000000000..84e89349f9f
--- /dev/null
+++ b/vespaclient-java/src/main/java/com/yahoo/vespastat/BucketStatsRetriever.java
@@ -0,0 +1,176 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespastat;
+
+import com.yahoo.document.BucketId;
+import com.yahoo.document.BucketIdFactory;
+import com.yahoo.document.DocumentId;
+import com.yahoo.document.GlobalId;
+import com.yahoo.document.select.BucketSelector;
+import com.yahoo.document.select.BucketSet;
+import com.yahoo.document.select.parser.ParseException;
+import com.yahoo.documentapi.SyncParameters;
+import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess;
+import com.yahoo.documentapi.messagebus.MessageBusSyncSession;
+import com.yahoo.documentapi.messagebus.protocol.DocumentMessage;
+import com.yahoo.documentapi.messagebus.protocol.GetBucketListMessage;
+import com.yahoo.documentapi.messagebus.protocol.GetBucketListReply;
+import com.yahoo.documentapi.messagebus.protocol.StatBucketMessage;
+import com.yahoo.documentapi.messagebus.protocol.StatBucketReply;
+import com.yahoo.messagebus.Reply;
+import com.yahoo.messagebus.routing.Route;
+
+import java.util.List;
+
+/**
+ * This class fetches bucket information from Vespa
+ *
+ * @author bjorncs
+ */
+public class BucketStatsRetriever {
+
+    private final BucketIdFactory bucketIdFactory = new BucketIdFactory();
+    private final BucketSelector selector = new BucketSelector(bucketIdFactory);
+
+    private final MessageBusSyncSession session;
+    private final MessageBusDocumentAccess documentAccess;
+    private final String route;
+
+    public BucketStatsRetriever(
+            DocumentAccessFactory documentAccessFactory,
+            String route,
+            ShutdownHookRegistrar registrar) {
+        registerShutdownHook(registrar);
+        this.documentAccess = documentAccessFactory.createDocumentAccess();
+        this.session = documentAccess.createSyncSession(new SyncParameters());
+        this.route = route;
+    }
+
+    private void registerShutdownHook(ShutdownHookRegistrar registrar) {
+        registrar.registerShutdownHook(() -> {
+            try {
+                session.destroy();
+            } catch (Exception e) {
+                // Ignore exception on shutdown
+            }
+            try {
+                documentAccess.shutdown();
+            } catch (Exception e) {
+                // Ignore exception on shutdown
+            }
+        });
+    }
+
+    public BucketId getBucketIdForType(ClientParameters.SelectionType type, String id) throws BucketStatsException {
+        switch (type) {
+            case DOCUMENT:
+                return bucketIdFactory.getBucketId(new DocumentId(id));
+            case BUCKET:
+                // The internal parser of BucketID is used since the Java Long.decode cannot handle unsigned longs.
+                return new BucketId(String.format("BucketId(%s)", id));
+            case GID:
+                return convertGidToBucketId(id);
+            case USER:
+            case GROUP:
+                try {
+                    BucketSet bucketList = selector.getBucketList(createDocumentSelection(type, id));
+                    if (bucketList.size() != 1) {
+                        String message = String.format("Document selection must map to only one location. " +
+                                "Specified selection matches %d locations.", bucketList.size());
+                        throw new BucketStatsException(message);
+                    }
+                    return bucketList.iterator().next();
+                } catch (ParseException e) {
+                    throw new BucketStatsException(String.format("Invalid id: %s (%s).", id, e.getMessage()), e);
+                }
+            default:
+                throw new RuntimeException("Unreachable code");
+        }
+    }
+
+    public String retrieveBucketStats(ClientParameters.SelectionType type, String id, BucketId bucketId) throws BucketStatsException {
+        String documentSelection = createDocumentSelection(type, id);
+        StatBucketMessage msg = new StatBucketMessage(bucketId, documentSelection);
+        StatBucketReply statBucketReply = sendMessage(msg, StatBucketReply.class);
+        return statBucketReply.getResults();
+    }
+
+    public List<GetBucketListReply.BucketInfo> retrieveBucketList(BucketId bucketId) throws BucketStatsException {
+        GetBucketListMessage msg = new GetBucketListMessage(bucketId);
+        GetBucketListReply bucketListReply = sendMessage(msg, GetBucketListReply.class);
+        return bucketListReply.getBuckets();
+    }
+
+
+    private <T extends Reply> T sendMessage(DocumentMessage msg, Class<T> expectedReply) throws BucketStatsException {
+        setRoute(msg, route);
+        Reply reply = session.syncSend(msg);
+        return validateReply(reply, expectedReply);
+    }
+
+    private static void setRoute(DocumentMessage msg, String route) throws BucketStatsException {
+        try {
+            msg.setRoute(Route.parse(route));
+        } catch (Exception e) {
+            throw new BucketStatsException(String.format("Invalid route: '%s'.", route));
+        }
+    }
+
+    private static <T extends Reply> T validateReply(Reply reply, Class<T> type) throws BucketStatsException {
+        if (reply.hasErrors()) {
+            throw new BucketStatsException(makeErrorMessage(reply));
+        }
+        if (!type.isInstance(reply)) {
+            throw new BucketStatsException(String.format("Unexpected reply %s: '%s'", reply.getType(), reply.toString()));
+        }
+        return type.cast(reply);
+    }
+
+    private static String makeErrorMessage(Reply reply) {
+        StringBuilder b = new StringBuilder();
+        b.append("Request failed: \n");
+        for (int i = 0; i < reply.getNumErrors(); i++) {
+            b.append(String.format("\t %s\n", reply.getError(i)));
+        }
+        return b.toString();
+    }
+
+    private static String createDocumentSelection(ClientParameters.SelectionType type, String id) {
+        switch (type) {
+            case BUCKET:
+                return "true";
+            case DOCUMENT:
+                return String.format("id=\"%s\"", id);
+            case GID:
+                return String.format("id.gid=\"gid(%s)\"", id);
+            case USER:
+                return String.format("id.user=%s", id);
+            case GROUP:
+                return String.format("id.group=\"%s\"", id);
+            default:
+                throw new RuntimeException("Unreachable code");
+        }
+    }
+
+    private static BucketId convertGidToBucketId(String id) throws BucketStatsException {
+        if (!id.matches("0x\\p{XDigit}{24}")) {
+            throw new BucketStatsException("Invalid gid: " + id);
+        }
+        String hexWithoutPrefix = id.substring(2);
+        return new GlobalId(convertHexStringToByteArray(hexWithoutPrefix)).toBucketId();
+    }
+
+    private static byte[] convertHexStringToByteArray(String s) throws BucketStatsException {
+        int len = s.length();
+        byte[] data = new byte[len / 2];
+        for (int i = 0; i < len; i += 2) {
+            int digit1 = Character.digit(s.charAt(i), 16);
+            int digit2 = Character.digit(s.charAt(i + 1), 16);
+            data[i / 2] = (byte) ((digit1 << 4) + digit2);
+        }
+        return data;
+    }
+
+    public interface ShutdownHookRegistrar {
+        void registerShutdownHook(Runnable runnable);
+    }
+}
diff --git a/vespaclient-java/src/main/java/com/yahoo/vespastat/ClientParameters.java b/vespaclient-java/src/main/java/com/yahoo/vespastat/ClientParameters.java
new file mode 100644
index 00000000000..bba0d9803ed
--- /dev/null
+++ b/vespaclient-java/src/main/java/com/yahoo/vespastat/ClientParameters.java
@@ -0,0 +1,73 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespastat;
+
+/**
+ * This class contains the program parameters.
+ *
+ * @author bjorncs
+ */
+public class ClientParameters {
+    // Show help page if true
+    public final boolean help;
+    // Dump list of documents for all buckets matching the selection if true
+    public final boolean dumpData;
+    // The message bus route
+    public final String route;
+    // The selection type
+    public final SelectionType selectionType;
+    // The selection id
+    public final String id;
+
+    public ClientParameters(
+            boolean help,
+            boolean dumpData,
+            String route,
+            SelectionType selectionType,
+            String id) {
+        this.help = help;
+        this.dumpData = dumpData;
+        this.route = route;
+        this.selectionType = selectionType;
+        this.id = id;
+    }
+
+    public enum SelectionType {USER, GROUP, BUCKET, GID, DOCUMENT}
+
+    public static class Builder {
+        private boolean help;
+        private boolean dumpData;
+        private String route;
+        private SelectionType selectionType;
+        private String id;
+
+        public Builder setHelp(boolean help) {
+            this.help = help;
+            return this;
+        }
+
+        public Builder setDumpData(boolean dumpData) {
+            this.dumpData = dumpData;
+            return this;
+        }
+
+        public Builder setRoute(String route) {
+            this.route = route;
+            return this;
+        }
+
+        public Builder setSelectionType(SelectionType selectionType) {
+            this.selectionType = selectionType;
+            return this;
+        }
+
+        public Builder setId(String id) {
+            this.id = id;
+            return this;
+        }
+
+        public ClientParameters build() {
+            return new ClientParameters(help, dumpData, route, selectionType, id);
+        }
+    }
+
+}
diff --git a/vespaclient-java/src/main/java/com/yahoo/vespastat/CommandLineOptions.java b/vespaclient-java/src/main/java/com/yahoo/vespastat/CommandLineOptions.java
new file mode 100644
index 00000000000..b0b6246a262
--- /dev/null
+++ b/vespaclient-java/src/main/java/com/yahoo/vespastat/CommandLineOptions.java
@@ -0,0 +1,139 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespastat;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.DefaultParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionGroup;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+
+/**
+ * Responsible for parsing the command line arguments and presenting the help page
+ *
+ * @author bjorncs
+ */
+public class CommandLineOptions {
+
+    private static final String HELP_OPTION = "help";
+    private static final String DUMP_OPTION = "dump";
+    private static final String ROUTE_OPTION = "route";
+    private static final String USER_OPTION = "user";
+    private static final String GROUP_OPTION = "group";
+    private static final String BUCKET_OPTION = "bucket";
+    private static final String GID_OPTION = "gid";
+    private static final String DOCUMENT_OPTION = "document";
+
+    private final Options options = createOptions();
+
+    @SuppressWarnings("AccessStaticViaInstance")
+    private static Options createOptions() {
+        Options options = new Options();
+
+        options.addOption(Option.builder("h")
+                .hasArg(false)
+                .desc("Show this syntax page.")
+                .longOpt(HELP_OPTION)
+                .build());
+
+        options.addOption(Option.builder("d")
+                .hasArg(false)
+                .desc("Dump list of documents for all buckets matching the selection command.")
+                .longOpt(DUMP_OPTION)
+                .build());
+
+        options.addOption(Option.builder("r")
+                .hasArg(true)
+                .desc("Route to send the messages to, usually the name of the storage cluster.")
+                .argName("route")
+                .longOpt(ROUTE_OPTION)
+                .build());
+
+        // A group of mutually exclusive options for user, group, bucket, gid and document.
+        OptionGroup optionGroup = new OptionGroup();
+        optionGroup.setRequired(false);
+
+        optionGroup.addOption(Option.builder("u")
+                .hasArg(true)
+                .desc("Dump list of buckets that can contain the given user.")
+                .argName("userid")
+                .longOpt(USER_OPTION)
+                .build());
+
+        optionGroup.addOption(Option.builder("g")
+                .hasArg(true)
+                .desc("Dump list of buckets that can contain the given group.")
+                .argName("groupid")
+                .longOpt(GROUP_OPTION)
+                .build());
+
+        optionGroup.addOption(Option.builder("b")
+                .hasArg(true)
+                .desc("Dump list of buckets that are contained in the given bucket, or that contain it.")
+                .argName("bucketid")
+                .longOpt(BUCKET_OPTION)
+                .build());
+
+        optionGroup.addOption(Option.builder("l")
+                .hasArg(true)
+                .desc("Dump information about one specific document, as given by the GID (implies --dump).")
+                .argName("globalid")
+                .longOpt(GID_OPTION)
+                .build());
+
+        optionGroup.addOption(Option.builder("o")
+                .hasArg(true)
+                .desc("Dump information about one specific document (implies --dump).")
+                .argName("docid")
+                .longOpt(DOCUMENT_OPTION)
+                .build());
+
+        options.addOptionGroup(optionGroup);
+        return options;
+    }
+
+    public void printHelp() {
+        HelpFormatter formatter = new HelpFormatter();
+        formatter.printHelp("vdsstat [options]",
+                "Fetch statistics about a specific user, group, bucket, gid or document.", options, "", false);
+    }
+
+    public ClientParameters parseCommandLineArguments(String[] args) {
+        try {
+            CommandLineParser clp = new DefaultParser();
+            CommandLine cl = clp.parse(options, args);
+            ClientParameters.Builder builder = new ClientParameters.Builder();
+
+            builder.setHelp(cl.hasOption(HELP_OPTION));
+            builder.setDumpData(cl.hasOption(DUMP_OPTION));
+            builder.setRoute(cl.getOptionValue(ROUTE_OPTION, "default"));
+
+            if (cl.hasOption(USER_OPTION)) {
+                builder.setSelectionType(ClientParameters.SelectionType.USER);
+                builder.setId(cl.getOptionValue(USER_OPTION));
+            } else if (cl.hasOption(GROUP_OPTION)) {
+                builder.setSelectionType(ClientParameters.SelectionType.GROUP);
+                builder.setId(cl.getOptionValue(GROUP_OPTION));
+            } else if (cl.hasOption(BUCKET_OPTION)) {
+                builder.setSelectionType(ClientParameters.SelectionType.BUCKET);
+                builder.setId(cl.getOptionValue(BUCKET_OPTION));
+            } else if (cl.hasOption(GID_OPTION)) {
+                builder.setSelectionType(ClientParameters.SelectionType.GID);
+                builder.setId(cl.getOptionValue(GID_OPTION));
+                builder.setDumpData(true);
+            } else if (cl.hasOption(DOCUMENT_OPTION)) {
+                builder.setSelectionType(ClientParameters.SelectionType.DOCUMENT);
+                builder.setId(cl.getOptionValue(DOCUMENT_OPTION));
+                builder.setDumpData(true);
+            } else if (!cl.hasOption(HELP_OPTION)) {
+                throw new IllegalArgumentException("Must specify one of 'user', 'group', 'bucket', 'document' or 'gid'.");
+            }
+
+            return builder.build();
+        } catch (ParseException e) {
+            throw new IllegalArgumentException(e.getMessage(), e);
+        }
+    }
+}
diff --git a/vespaclient-java/src/main/java/com/yahoo/vespastat/DocumentAccessFactory.java b/vespaclient-java/src/main/java/com/yahoo/vespastat/DocumentAccessFactory.java
new file mode 100644
index 00000000000..55a31f30a2b
--- /dev/null
+++ b/vespaclient-java/src/main/java/com/yahoo/vespastat/DocumentAccessFactory.java
@@ -0,0 +1,15 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespastat;
+
+import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess;
+
+/**
+ * Factory class for {@link com.yahoo.documentapi.messagebus.MessageBusDocumentAccess}.
+ *
+ * @author bjorncs
+ */
+public class DocumentAccessFactory {
+    public MessageBusDocumentAccess createDocumentAccess() {
+        return new MessageBusDocumentAccess();
+    }
+}
diff --git a/vespaclient-java/src/main/java/com/yahoo/vespastat/Main.java b/vespaclient-java/src/main/java/com/yahoo/vespastat/Main.java
new file mode 100644
index 00000000000..9d87a6f68f4
--- /dev/null
+++ b/vespaclient-java/src/main/java/com/yahoo/vespastat/Main.java
@@ -0,0 +1,38 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespastat;
+
+/**
+ * Main application class
+ *
+ * @author bjorncs
+ */
+public class Main {
+
+    private Main() {
+    }
+
+    public static void main(String[] args) {
+        CommandLineOptions options = new CommandLineOptions();
+        try {
+            ClientParameters params = options.parseCommandLineArguments(args);
+            if (params.help) {
+                options.printHelp();
+                return;
+            }
+            BucketStatsRetriever retriever = new BucketStatsRetriever(
+                    new DocumentAccessFactory(),
+                    params.route,
+                    createShutdownHookRegistrar());
+            BucketStatsPrinter printer = new BucketStatsPrinter(retriever, System.out);
+            printer.retrieveAndPrintBucketStats(params.selectionType, params.id, params.dumpData);
+        } catch (IllegalArgumentException e) {
+            System.err.printf("Failed to parse command line arguments: %s.\n", e.getMessage());
+        } catch (BucketStatsException e) {
+            System.err.println(e.getMessage());
+        }
+    }
+
+    private static BucketStatsRetriever.ShutdownHookRegistrar createShutdownHookRegistrar() {
+        return runnable -> Runtime.getRuntime().addShutdownHook(new Thread(runnable));
+    }
+}
diff --git a/vespaclient-java/src/main/java/com/yahoo/vespasummarybenchmark/VespaSummaryBenchmark.java b/vespaclient-java/src/main/java/com/yahoo/vespasummarybenchmark/VespaSummaryBenchmark.java
new file mode 100644
index 00000000000..803445d16f5
--- /dev/null
+++ b/vespaclient-java/src/main/java/com/yahoo/vespasummarybenchmark/VespaSummaryBenchmark.java
@@ -0,0 +1,162 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespasummarybenchmark;
+
+import com.yahoo.compress.CompressionType;
+import com.yahoo.document.GlobalId;
+import com.yahoo.document.idstring.IdString;
+import com.yahoo.document.serialization.DeserializationException;
+import com.yahoo.jrt.*;
+import com.yahoo.log.LogSetup;
+import com.yahoo.slime.*;
+import net.jpountz.lz4.LZ4Factory;
+import net.jpountz.lz4.LZ4FastDecompressor;
+
+import java.io.*;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ *
+ * This is used for testing and benchmarking rpc docsum interface.
+ * time vespa-summary-benchmark file-containing-docids connectionspec summary-class repetitions threads
+ * fx ' time vespa-summary-benchmark feed.xml tcp/localhost:19115 keyvaluesummary 10000 32'
+ *
+ * @author baldersheim
+ */
+public class VespaSummaryBenchmark {
+
+    private final Supervisor supervisor = new Supervisor(new Transport());
+
+    private VespaSummaryBenchmark() { }
+
+    private static List<String> getDocIds(String fileName) {
+        try {
+            FileInputStream fstream = new FileInputStream(fileName);
+            DataInputStream in = new DataInputStream(fstream);
+            BufferedReader br = new BufferedReader(new InputStreamReader(in));
+            String strLine;
+
+            List<String> docIds = new ArrayList<>();
+            while ((strLine = br.readLine()) != null) {
+                docIds.add(strLine);
+            }
+            in.close();
+            return docIds;
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    private List<Target> getTargets(String connectionSpec, int numTargets) {
+        List<Target> targets = new ArrayList<>(numTargets);
+        for ( int i=0; i < numTargets; i++) {
+            targets.add(supervisor.connect(new Spec(connectionSpec)));
+        }
+        return targets;
+    }
+
+    private static Slime createDocsumRequest(String summaryClass, List<GlobalId> gids) {
+        Slime docsumRequest = new Slime();
+        Cursor root = docsumRequest.setObject();
+        root.setString("class", summaryClass);
+        Cursor gidCursor = root.setArray("gids");
+        for (GlobalId gid : gids) {
+            gidCursor.addData(gid.getRawId());
+        }
+        return docsumRequest;
+    }
+
+    private static class Waiter implements RequestWaiter {
+
+        int waitingFor;
+        boolean dump;
+
+        Waiter(int expect, boolean dump) {
+            waitingFor = expect;
+            this.dump = dump;
+        }
+
+        private void print(Request request) {
+            Values ret = request.returnValues();
+            CompressionType type = CompressionType.valueOf(ret.get(0).asInt8());
+            int uncompressedSize = ret.get(1).asInt32();
+            byte [] blob = ret.get(2).asData();
+            if (type == CompressionType.LZ4) {
+                LZ4Factory factory = LZ4Factory.fastestInstance();
+                LZ4FastDecompressor decompressor = factory.fastDecompressor();
+                byte [] uncompressed = new byte [uncompressedSize];
+                int compressedLength = decompressor.decompress(blob, 0, uncompressed, 0, uncompressedSize);
+                if (compressedLength != blob.length) {
+                    throw new DeserializationException("LZ4 decompression failed. compressed size does not match. Expected " + blob.length + ". Got " + compressedLength);
+                }
+                blob = uncompressed;
+            }
+            Slime slime = BinaryFormat.decode(blob);
+            try {
+                new JsonFormat(true).encode(System.out, slime);
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+        }
+
+        @Override
+        public void handleRequestDone(Request request) {
+            synchronized (this) {
+                if (dump) {
+                    print(request);
+                    dump = false;
+                }
+                waitingFor--;
+                if (waitingFor == 0) {
+                    this.notifyAll();
+                }
+            }
+        }
+        void waitForReplies() throws InterruptedException {
+            synchronized (this) {
+                while (waitingFor > 0) {
+                    this.wait();
+                }
+            }
+        }
+    }
+
+    private static void fetchDocIds(String summaryClass, List<Target> targets, List<GlobalId> gids, boolean dump) {
+        Slime docsumRequest = createDocsumRequest(summaryClass, gids);
+        byte [] blob = BinaryFormat.encode(docsumRequest);
+        Waiter waiter = new Waiter(targets.size(), dump);
+        for (Target target : targets) {
+            Request r = new Request("proton.getDocsums");
+            r.parameters().add(new Int8Value(CompressionType.NONE.getCode()));
+            r.parameters().add(new Int32Value(blob.length));
+            r.parameters().add(new DataValue(blob));
+            target.invokeAsync(r, 100.0, waiter);
+        }
+        try {
+            waiter.waitForReplies();
+        } catch (InterruptedException e) {
+            throw new RuntimeException(e);
+        }
+
+    }
+
+    public static void main(String[] args) {
+        LogSetup.initVespaLogging("vespasummarybenchmark");
+        String docidFileName = args[0];
+        String connectionSpec = args[1];
+        String summaryClass = args[2];
+        int numRuns = Integer.parseInt(args[3]);
+        int numTargets = Integer.parseInt(args[4]);
+        VespaSummaryBenchmark benchmark = new VespaSummaryBenchmark();
+        List<String> docIds = getDocIds(docidFileName);
+        List<GlobalId> gids = new ArrayList<>(docIds.size());
+        for (String docid : docIds) {
+            GlobalId gid = new GlobalId(IdString.createIdString(docid));
+            gids.add(gid);
+        }
+        List<Target> targets = benchmark.getTargets(connectionSpec, numTargets);
+        for (int i = 0; i < numRuns; i++) {
+            fetchDocIds(summaryClass, targets, gids, i==0);
+        }
+    }
+}
diff --git a/vespaclient-java/src/main/java/com/yahoo/vespavisit/StdOutVisitorHandler.java b/vespaclient-java/src/main/java/com/yahoo/vespavisit/StdOutVisitorHandler.java
new file mode 100644
index 00000000000..c849bc1741c
--- /dev/null
+++ b/vespaclient-java/src/main/java/com/yahoo/vespavisit/StdOutVisitorHandler.java
@@ -0,0 +1,292 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespavisit;
+
+import com.yahoo.document.BucketId;
+import com.yahoo.document.Document;
+import com.yahoo.document.DocumentId;
+import com.yahoo.document.json.JsonWriter;
+import com.yahoo.document.serialization.XmlStream;
+import com.yahoo.documentapi.AckToken;
+import com.yahoo.documentapi.DumpVisitorDataHandler;
+import com.yahoo.documentapi.ProgressToken;
+import com.yahoo.documentapi.VisitorControlHandler;
+import com.yahoo.documentapi.VisitorDataHandler;
+import com.yahoo.documentapi.messagebus.protocol.DocumentListEntry;
+import com.yahoo.documentapi.messagebus.protocol.DocumentListMessage;
+import com.yahoo.documentapi.messagebus.protocol.EmptyBucketsMessage;
+import com.yahoo.documentapi.messagebus.protocol.MapVisitorMessage;
+import com.yahoo.log.LogLevel;
+import com.yahoo.messagebus.Message;
+
+import java.io.IOException;
+import java.io.PrintStream;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.logging.Logger;
+
+/**
+ * A visitor data and progress handler that writes to STDOUT.
+ *
+ * Due to java not being able to inherit two classes, and neither being an
+ * interface this had to be implemented by creating a wrapper class.
+ *
+ * @author <a href="mailto:thomasg@yahoo-inc.com">Thomas Gundersen</a>
+ */
+public class StdOutVisitorHandler extends VdsVisitHandler {
+    private static final Logger log = Logger.getLogger(
+                                        StdOutVisitorHandler.class.getName());
+    private boolean printIds;
+    private boolean indentXml;
+    private int processTimeMilliSecs;
+    private PrintStream out;
+    private final boolean jsonOutput;
+
+    private VisitorDataHandler dataHandler;
+
+    public StdOutVisitorHandler(boolean printIds, boolean indentXml,
+                                boolean showProgress, boolean showStatistics, boolean doStatistics,
+                                boolean abortOnClusterDown, int processtime, boolean jsonOutput)
+    {
+        super(showProgress, showStatistics, abortOnClusterDown);
+
+        this.printIds = printIds;
+        this.indentXml = indentXml;
+        this.processTimeMilliSecs = processtime;
+        this.jsonOutput = jsonOutput;
+        String charset = "UTF-8";
+        try {
+            out = new PrintStream(System.out, true, charset);
+        } catch (java.io.UnsupportedEncodingException e) {
+            System.out.println(charset + " is an unsupported encoding, " +
+                               "using default instead.");
+            out = System.out;
+        }
+
+        dataHandler = new DataHandler(doStatistics);
+    }
+
+    @Override
+    public void onDone() {
+    }
+
+    public VisitorDataHandler getDataHandler() { return dataHandler; }
+
+    class StatisticsMap extends LinkedHashMap<String, Integer> {
+        int maxSize;
+
+        StatisticsMap(int maxSize) {
+            super(100, (float)0.75, true);
+            this.maxSize = maxSize;
+        }
+
+        protected boolean removeEldestEntry(Map.Entry<String, Integer> eldest) {
+            if (size() > maxSize) {
+                dump(eldest);
+                return true;
+            }
+
+            return false;
+        }
+
+        private void dump(Map.Entry<String, Integer> e) {
+            out.println(e.getKey() + ":" + e.getValue());
+        }
+
+        public void dumpAll() {
+            for (Map.Entry<String, Integer> e : entrySet()) {
+                dump(e);
+            }
+            clear();
+        }
+    }
+
+    class DataHandler extends DumpVisitorDataHandler {
+        boolean doStatistics;
+        StatisticsMap statisticsMap = new StatisticsMap(10000);
+        private volatile boolean first = true;
+
+        public DataHandler(boolean doStatistics) {
+            this.doStatistics = doStatistics;
+        }
+
+        @Override
+        public void onMessage(Message m, AckToken token) {
+            if (processTimeMilliSecs > 0) {
+                try {
+                    Thread.sleep(processTimeMilliSecs);
+                } catch (InterruptedException e) {}
+            }
+
+            synchronized (printLock) {
+                if (m instanceof MapVisitorMessage) {
+                    onMapVisitorData(((MapVisitorMessage)m).getData());
+                    ack(token);
+                } else if (m instanceof DocumentListMessage) {
+                    DocumentListMessage dlm = (DocumentListMessage)m;
+                    onDocumentList(dlm.getBucketId(), dlm.getDocuments());
+                    ack(token);
+                } else if (m instanceof EmptyBucketsMessage) {
+                    onEmptyBuckets(((EmptyBucketsMessage)m).getBucketIds());
+                    ack(token);
+                } else {
+                    super.onMessage(m, token);
+                }
+            }
+        }
+
+        @Override
+        public void onDocument(Document doc, long timestamp) {
+            try {
+                if (lastLineIsProgress) {
+                    System.err.print('\r');
+                }
+
+                if (printIds) {
+                    out.print(doc.getId());
+                    out.print(" (Last modified at ");
+                    out.println(timestamp + ")");
+                } else {
+                    if (jsonOutput) {
+                        writeJsonDocument(doc);
+                    } else {
+                        out.print(doc.toXML(
+                                indentXml ? "  " : ""));
+                    }
+                }
+            } catch (Exception e) {
+                System.err.println("Failed to output document: "
+                        + e.getMessage());
+                getControlHandler().abort();
+            }
+        }
+
+        private void writeJsonDocument(Document doc) throws IOException {
+            writeFeedStartOrRecordSeparator();
+            out.write(JsonWriter.toByteArray(doc));
+        }
+
+        @Override
+        public void onRemove(DocumentId docId) {
+            try {
+                if (lastLineIsProgress) {
+                    System.err.print('\r');
+                }
+
+                if (printIds) {
+                    out.println(docId + " (Removed)");
+                } else {
+                    if (jsonOutput) {
+                        writeJsonDocumentRemove(docId);
+                    } else {
+                        XmlStream stream = new XmlStream();
+                        stream.beginTag("remove");
+                        stream.addAttribute("documentid", docId);
+                        stream.endTag();
+                        assert(stream.isFinalized());
+                        out.print(stream);
+                    }
+                }
+            } catch (Exception e) {
+                System.err.println("Failed to output document: "
+                        + e.getMessage());
+                getControlHandler().abort();
+            }
+        }
+
+        private void writeJsonDocumentRemove(DocumentId docId)
+                throws IOException {
+            writeFeedStartOrRecordSeparator();
+            out.write(JsonWriter.documentRemove(docId));
+        }
+
+        private void writeFeedStartOrRecordSeparator() {
+            if (first) {
+                out.println("[");
+                first = false;
+            } else {
+                out.println(",");
+            }
+        }
+
+        private void writeFeedEnd() {
+            out.println("]");
+        }
+
+        public void onMapVisitorData(Map<String, String> data) {
+            for (String key : data.keySet()) {
+                if (doStatistics) {
+                    Integer i = statisticsMap.get(key);
+                    if (i != null) {
+                        statisticsMap.put(key, Integer.parseInt(data.get(key)) + i);
+                    } else {
+                        statisticsMap.put(key, Integer.parseInt(data.get(key)));
+                    }
+                } else {
+                    out.println(key + ":" + data.get(key));
+                }
+            }
+        }
+
+        public void onDocumentList(BucketId bucketId, List<DocumentListEntry> documents) {
+            out.println("Got document list of bucket " + bucketId.toString());
+            for (DocumentListEntry entry : documents) {
+                entry.getDocument().setLastModified(entry.getTimestamp());
+                onDocument(entry.getDocument(), entry.getTimestamp());
+            }
+        }
+
+        public void onEmptyBuckets(List<BucketId> bucketIds) {
+            StringBuilder buckets = new StringBuilder();
+            for(BucketId bid : bucketIds) {
+                buckets.append(" ");
+                buckets.append(bid.toString());
+            }
+            log.log(LogLevel.INFO, "Got EmptyBuckets: " + buckets);
+        }
+
+        public synchronized void onDone() {
+            if (jsonOutput) {
+                writeFeedEnd();
+            }
+            statisticsMap.dumpAll();
+            super.onDone();
+        }
+    }
+
+    class ControlHandler extends VisitorControlHandler {
+        public void onProgress(ProgressToken token) {
+            if (showProgress) {
+                synchronized (printLock) {
+                    if (lastLineIsProgress) {
+                        System.err.print('\r');
+                    }
+                    System.err.format("%.1f %% finished.",
+                                      token.percentFinished());
+                    lastLineIsProgress = true;
+                }
+            }
+            super.onProgress(token);
+        }
+
+        public void onDone(CompletionCode code, String message) {
+            if (lastLineIsProgress) {
+                System.err.print('\n');
+                lastLineIsProgress = false;
+            }
+            if (code != CompletionCode.SUCCESS) {
+                if (code == CompletionCode.ABORTED) {
+                    System.err.println("Visitor aborted: " + message);
+                } else if (code == CompletionCode.TIMEOUT) {
+                    System.err.println("Visitor timed out: " + message);
+                } else {
+                    System.err.println("Visitor aborted due to unknown issue "
+                                     + code + ": " + message);
+                }
+            } else if (showProgress) {
+                System.err.println("Completed visiting.");
+            }
+            super.onDone(code, message);
+        }
+    }
+}
diff --git a/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisit.java b/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisit.java
new file mode 100644
index 00000000000..ff072b845de
--- /dev/null
+++ b/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisit.java
@@ -0,0 +1,789 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespavisit;
+
+import com.yahoo.document.select.parser.ParseException;
+import com.yahoo.documentapi.ProgressToken;
+import com.yahoo.documentapi.VisitorControlHandler;
+import com.yahoo.documentapi.VisitorParameters;
+import com.yahoo.documentapi.VisitorSession;
+import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess;
+import com.yahoo.documentapi.messagebus.MessageBusParams;
+import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet;
+import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol;
+import com.yahoo.log.LogSetup;
+import com.yahoo.document.select.OrderingSpecification;
+import com.yahoo.messagebus.StaticThrottlePolicy;
+import com.yahoo.vespaclient.ClusterDef;
+import com.yahoo.vespaclient.ClusterList;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.DefaultParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+
+import java.io.*;
+import java.nio.charset.Charset;
+import java.util.Map;
+
+/**
+ * Example client using visiting
+ *
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>, based on work by <a href="mailto:humbe@yahoo-inc.com">H&aring;kon Humberset</a>
+ */
+public class VdsVisit {
+    private VdsVisitParameters params;
+    private MessageBusParams mbparams = new MessageBusParams(new LoadTypeSet());
+    private VisitorSession session;
+
+    private final VisitorSessionAccessorFactory sessionAccessorFactory;
+    private VisitorSessionAccessor sessionAccessor;
+    private ShutdownHookRegistrar shutdownHookRegistrar;
+
+    public interface ShutdownHookRegistrar {
+        public void registerShutdownHook(Thread thread);
+    }
+
+    public interface VisitorSessionAccessor {
+        public VisitorSession createVisitorSession(VisitorParameters params) throws ParseException;
+        public void shutdown();
+    }
+
+    public interface VisitorSessionAccessorFactory {
+        public VisitorSessionAccessor createVisitorSessionAccessor();
+    }
+
+    private static class MessageBusVisitorSessionAccessor implements VisitorSessionAccessor {
+        private MessageBusDocumentAccess access;
+
+        private MessageBusVisitorSessionAccessor(MessageBusParams mbparams) {
+            access = new MessageBusDocumentAccess(mbparams);
+        }
+        @Override
+        public VisitorSession createVisitorSession(VisitorParameters params) throws ParseException {
+            return access.createVisitorSession(params);
+        }
+
+        @Override
+        public void shutdown() {
+            access.shutdown();
+        }
+    }
+
+    private static class MessageBusVisitorSessionAccessorFactory implements VisitorSessionAccessorFactory {
+        MessageBusParams mbparams;
+
+        private MessageBusVisitorSessionAccessorFactory(MessageBusParams mbparams) {
+            this.mbparams = mbparams;
+        }
+
+        @Override
+        public VisitorSessionAccessor createVisitorSessionAccessor() {
+            return new MessageBusVisitorSessionAccessor(mbparams);
+        }
+    }
+
+    private static class JvmRuntimeShutdownHookRegistrar implements ShutdownHookRegistrar {
+        @Override
+        public void registerShutdownHook(Thread thread) {
+            Runtime.getRuntime().addShutdownHook(thread);
+        }
+    }
+
+    public VdsVisit() {
+        this.sessionAccessorFactory = new MessageBusVisitorSessionAccessorFactory(mbparams);
+        this.shutdownHookRegistrar = new JvmRuntimeShutdownHookRegistrar();
+    }
+
+    public VdsVisit(VisitorSessionAccessorFactory sessionAccessorFactory,
+                    ShutdownHookRegistrar shutdownHookRegistrar)
+    {
+        this.sessionAccessorFactory = sessionAccessorFactory;
+        this.shutdownHookRegistrar = shutdownHookRegistrar;
+    }
+
+    public static void main(String args[]) {
+        LogSetup.initVespaLogging("vespavisit");
+        VdsVisit vdsVisit = new VdsVisit();
+
+        Options options = createOptions();
+
+        try {
+            ArgumentParser parser = new ArgumentParser(options);
+            vdsVisit.params = parser.parse(args);
+            if (vdsVisit.params == null) {
+                vdsVisit.printSyntax(options);
+                System.exit(0);
+            }
+            ClusterList clusterList = new ClusterList("client");
+            vdsVisit.params.getVisitorParameters().setRoute(
+                    resolveClusterRoute(clusterList, vdsVisit.params.getCluster()));
+        } catch (org.apache.commons.cli.ParseException e) {
+            System.err.println("Failed to parse arguments. Try --help for syntax. " + e.getMessage());
+            System.exit(1);
+        } catch (IllegalArgumentException e) {
+            System.err.println(e.getMessage());
+            System.exit(1);
+        }
+
+        if (vdsVisit.params.isVerbose()) {
+            verbosePrintParameters(vdsVisit.params, System.err);
+        }
+
+        try {
+            vdsVisit.run();
+        } catch (Exception e) {
+            e.printStackTrace();
+            System.exit(1);
+        }
+    }
+
+    private void printSyntax(Options options) {
+        HelpFormatter formatter = new HelpFormatter();
+        formatter.printHelp("vespavisit <options>", "Visit documents from VDS", options , "");
+    }
+
+    @SuppressWarnings("AccessStaticViaInstance")
+    protected static Options createOptions() {
+        Options options = new Options();
+        options.addOption("h", "help", false, "Show this syntax page.");
+
+        options.addOption(Option.builder("d")
+                .longOpt("datahandler")
+                .hasArg(true)
+                .argName("target")
+                .desc("Send results to the given target.")
+                .build());
+
+        options.addOption(Option.builder("s")
+                .longOpt("selection")
+                .hasArg(true)
+                .argName("selection")
+                .desc("What documents to visit.")
+                .build());
+
+        options.addOption(Option.builder("f")
+                .longOpt("from")
+                .hasArg(true)
+                .argName("timestamp")
+                .desc("Only visit from the given timestamp (microseconds).")
+                .type(Number.class)
+                .build());
+
+        options.addOption(Option.builder("t")
+                .longOpt("to")
+                .hasArg(true)
+                .argName("timestamp")
+                .desc("Only visit up to the given timestamp (microseconds).")
+                .type(Number.class).build());
+
+        options.addOption("e", "headersonly", false, "Only visit headers of documents.[Deprecated]");
+
+        options.addOption(Option.builder("l")
+                .longOpt("fieldset")
+                .hasArg(true)
+                .argName("fieldset")
+                .desc("Retrieve the specified fields only (see http://vespa.corp.yahoo.com/5/documentation/reference/fieldsets.html). Default is [all].")
+                .build());
+
+        options.addOption(Option.builder()
+                .longOpt("visitinconsistentbuckets")
+                .hasArg(false)
+                .desc("Don't wait for inconsistent buckets to become consistent.")
+                .build());
+
+        options.addOption(Option.builder("m")
+                .longOpt("maxpending")
+                .hasArg(true)
+                .argName("num")
+                .desc("Maximum pending messages to data handlers per storage visitor.")
+                .type(Number.class)
+                .build());
+
+        options.addOption(Option.builder()
+                .longOpt("maxpendingsuperbuckets")
+                .hasArg(true)
+                .argName("num")
+                .desc("Maximum pending visitor messages from the vespavisit client. If set, dynamic throttling of visitors will be disabled!")
+                .type(Number.class)
+                .build());
+
+        options.addOption(Option.builder("b")
+                .longOpt("maxbuckets")
+                .hasArg(true)
+                .argName("num")
+                .desc("Maximum buckets per visitor.")
+                .type(Number.class)
+                .build());
+
+        options.addOption("i", "printids", false, "Display only document identifiers.");
+
+        options.addOption(Option.builder("p")
+                .longOpt("progress")
+                .hasArg(true)
+                .argName("file")
+                .desc("Use given file to track progress.")
+                .build());
+
+        options.addOption(Option.builder("o")
+                .longOpt("timeout")
+                .hasArg(true)
+                .argName("milliseconds")
+                .desc("Time out visitor after given time.")
+                .type(Number.class)
+                .build());
+
+        options.addOption(Option.builder("u")
+                .longOpt("buckettimeout")
+                .hasArg(true)
+                .argName("milliseconds")
+                .desc("Fail visitor if visiting a single bucket takes longer than this (default same as timeout)")
+                .type(Number.class)
+                .build());
+
+        options.addOption(Option.builder()
+                .longOpt("visitlibrary")
+                .hasArg(true)
+                .argName("string")
+                .desc("Use the given visitor library.")
+                .build());
+
+        options.addOption(Option.builder()
+                .longOpt("libraryparam")
+                .numberOfArgs(2)
+                .argName("key> <val")
+                .desc("Give the following parameter to the visitor.")
+                .build());
+
+        options.addOption("r", "visitremoves", false, "Include information of removed documents.");
+
+        options.addOption(Option.builder("c")
+                .longOpt("cluster")
+                .hasArg(true)
+                .argName("cluster")
+                .desc("Visit the given VDS cluster.")
+                .build());
+
+        options.addOption("v", "verbose", false, "Indent XML, show progress and info on STDERR.");
+
+        options.addOption(Option.builder()
+                .longOpt("statistics")
+                .hasArg(true)
+                .argName("args")
+                .desc("Use CountVisitor for document statistics. Use comma-separated arguments.")
+                .build());
+
+        options.addOption(Option.builder()
+                .longOpt("abortonclusterdown")
+                .hasArg(false)
+                .desc("Abort if cluster is down.")
+                .build());
+
+        options.addOption(Option.builder()
+                .longOpt("maxhits")
+                .hasArg(true)
+                .argName("num")
+                .desc("Abort visiting when we have received this many \"first pass\" documents. Only appropriate for visiting involving id.order. This is only an approximate number, all pending work will be completed and those documents will also be returned.")
+                .type(Number.class)
+                .build());
+
+        options.addOption(Option.builder()
+                .longOpt("maxtotalhits")
+                .hasArg(true)
+                .argName("num")
+                .desc("Abort visiting when we have received this many total documents. This is only an approximate number, all pending work will be completed and those documents will also be returned.")
+                .type(Number.class)
+                .build());
+
+        options.addOption(Option.builder()
+                .longOpt("processtime")
+                .hasArg(true)
+                .argName("num")
+                .desc("Sleep this amount of millisecs before processing message. (Debug option for pretending to be slow client)")
+                .type(Number.class)
+                .build());
+        options.addOption(Option.builder()
+                .longOpt("priority")
+                .hasArg(true)
+                .argName("name")
+                .desc("Priority used for each visitor. Defaults to NORMAL_3. " +
+                        "Use with care to avoid starving lower prioritized traffic in the cluster")
+                .build());
+
+        options.addOption(Option.builder()
+                .longOpt("ordering")
+                .hasArg(true)
+                .argName("order")
+                .desc("Order to visit documents in. Only makes sense in conjunction with a document selection involving id.order. Legal values are \"ascending\" and \"descending\"")
+                .build());
+
+        options.addOption(Option.builder()
+                .longOpt("tracelevel")
+                .hasArg(true)
+                .argName("level")
+                .desc("Tracelevel ([0-9]) to use for debugging purposes")
+                .type(Number.class)
+                .build());
+
+        options.addOption(Option.builder()
+                .longOpt("skipbucketsonfatalerrors")
+                .hasArg(false)
+                .desc("Skip visiting super buckets with fatal error codes.")
+                .build());
+
+        options.addOption(Option.builder()
+                .longOpt("jsonoutput")
+                .desc("Output documents as JSON")
+                .hasArg(false)
+                .build());
+        return options;
+    }
+
+    public static class VdsVisitParameters {
+        private VisitorParameters visitorParameters;
+        /** If not specified in options, will get form cluster list */
+        private String cluster = null;
+        private boolean verbose = false;
+        private boolean printIdsOnly = false;
+        private String statisticsParts = null;
+        private boolean abortOnClusterDown = false;
+        private int processTime = 0;
+        private int fullTimeout = 7 * 24 * 60 * 60 * 1000;
+        private boolean jsonOutput = false;
+
+        public VisitorParameters getVisitorParameters() {
+            return visitorParameters;
+        }
+
+        public void setVisitorParameters(VisitorParameters visitorParameters) {
+            this.visitorParameters = visitorParameters;
+        }
+
+        public String getCluster() {
+            return cluster;
+        }
+
+        public void setCluster(String cluster) {
+            this.cluster = cluster;
+        }
+
+        public boolean isVerbose() {
+            return verbose;
+        }
+
+        public void setVerbose(boolean verbose) {
+            this.verbose = verbose;
+        }
+
+        public boolean isPrintIdsOnly() {
+            return printIdsOnly;
+        }
+
+        public void setPrintIdsOnly(boolean printIdsOnly) {
+            this.printIdsOnly = printIdsOnly;
+        }
+
+        public String getStatisticsParts() {
+            return statisticsParts;
+        }
+
+        public void setStatisticsParts(String statisticsParts) {
+            this.statisticsParts = statisticsParts;
+        }
+
+        public boolean getAbortOnClusterDown() {
+            return abortOnClusterDown;
+        }
+
+        public void setAbortOnClusterDown(boolean abortOnClusterDown) {
+            this.abortOnClusterDown = abortOnClusterDown;
+        }
+
+        public int getFullTimeout() {
+            return fullTimeout;
+        }
+
+        public void setFullTimeout(int fullTimeout) {
+            this.fullTimeout = fullTimeout;
+        }
+
+        public int getProcessTime() {
+            return processTime;
+        }
+
+        public void setProcessTime(int processTime) {
+            this.processTime = processTime;
+        }
+
+        public void setJsonOutput(boolean jsonOutput) {
+            this.jsonOutput = jsonOutput;
+        }
+    }
+
+    protected static class ArgumentParser {
+        private Options options;
+
+        public ArgumentParser(Options options) {
+            this.options = options;
+        }
+
+        public VdsVisitParameters parse(String args[]) throws org.apache.commons.cli.ParseException {
+            VdsVisitParameters allParams = new VdsVisitParameters();
+            VisitorParameters params = new VisitorParameters("");
+            CommandLineParser parser = new DefaultParser();
+            CommandLine line = parser.parse(options, args);
+
+            if (line.hasOption("h")) {
+                return null;
+            }
+            if (line.hasOption("d")) {
+                params.setRemoteDataHandler(line.getOptionValue("d"));
+            }
+            if (line.hasOption("s")) {
+                params.setDocumentSelection(line.getOptionValue("s"));
+            }
+            if (line.hasOption("f")) {
+                params.setFromTimestamp(((Number) line.getParsedOptionValue("f")).longValue());
+            }
+            if (line.hasOption("t")) {
+                params.setToTimestamp(((Number) line.getParsedOptionValue("t")).longValue());
+            }
+            if (line.hasOption("e")) {
+                params.fieldSet("[header]");
+            }
+            if (line.hasOption("l")) {
+                params.fieldSet(line.getOptionValue("l"));
+            }
+            if (line.hasOption("visitinconsistentbuckets")) {
+                params.visitInconsistentBuckets(true);
+            }
+            if (line.hasOption("m")) {
+                params.setMaxPending(((Number) line.getParsedOptionValue("m")).intValue());
+            }
+            if (line.hasOption("b")) {
+                params.setMaxBucketsPerVisitor(((Number) line.getParsedOptionValue("b")).intValue());
+            }
+            if (line.hasOption("i")) {
+                allParams.setPrintIdsOnly(true);
+                params.fieldSet("[id]");
+            }
+            if (line.hasOption("p")) {
+                params.setResumeFileName(line.getOptionValue("p"));
+            }
+            if (line.hasOption("o")) {
+                allParams.setFullTimeout(((Number) line.getParsedOptionValue("o")).intValue());
+                params.setTimeoutMs(allParams.getFullTimeout());
+            }
+            if (line.hasOption("u")) {
+                params.setTimeoutMs(((Number) line.getParsedOptionValue("u")).intValue());
+            }
+            if (line.hasOption("visitlibrary")) {
+                params.setVisitorLibrary(line.getOptionValue("visitlibrary"));
+            }
+            if (line.hasOption("libraryparam")) {
+                String key = line.getOptionValues("libraryparam")[0];
+                String value = line.getOptionValues("libraryparam")[1];
+                params.setLibraryParameter(key, value);
+            }
+            if (line.hasOption("r")) {
+                params.visitRemoves(true);
+            }
+            if (line.hasOption("c")) {
+                allParams.setCluster(line.getOptionValue("c"));
+            }
+
+            if (line.hasOption("v")) {
+                allParams.setVerbose(true);
+            }
+
+            if (line.hasOption("statistics")) {
+                allParams.setStatisticsParts(line.getOptionValue("statistics"));
+                params.fieldSet("[id]");
+                params.setVisitorLibrary("CountVisitor");
+            }
+
+            if (line.hasOption("abortonclusterdown")) {
+                allParams.setAbortOnClusterDown(true);
+            }
+            if (line.hasOption("processtime")) {
+                allParams.setProcessTime(((Number) line.getParsedOptionValue("processtime")).intValue());
+            }
+            if (line.hasOption("maxhits")) {
+                params.setMaxFirstPassHits(((Number)line.getParsedOptionValue("maxhits")).intValue());
+            }
+            if (line.hasOption("maxtotalhits")) {
+                params.setMaxTotalHits(((Number)line.getParsedOptionValue("maxtotalhits")).intValue());
+            }
+            if (line.hasOption("tracelevel")) {
+                params.setTraceLevel(((Number)line.getParsedOptionValue("tracelevel")).intValue());
+            }
+            if (line.hasOption("priority")) {
+                try {
+                    DocumentProtocol.Priority priority = DocumentProtocol.getPriorityByName(
+                            line.getOptionValue("priority"));
+                    params.setPriority(priority);
+                } catch (IllegalArgumentException e) {
+                    throw new IllegalArgumentException("Unknown priority name");
+                }
+            } else {
+                // Let bulk visitor jobs have a low priority by default to avoid stalling concurrent
+                // (real time) write and read operations.
+                params.setPriority(DocumentProtocol.Priority.LOW_1);
+            }
+            if (line.hasOption("ordering")) {
+                String opt = line.getOptionValue("ordering");
+                if (opt.equalsIgnoreCase("ascending")) {
+                    params.setVisitorOrdering(OrderingSpecification.ASCENDING);
+                } else if (opt.equalsIgnoreCase("descending")) {
+                    params.setVisitorOrdering(OrderingSpecification.DESCENDING);
+                } else {
+                    throw new IllegalArgumentException("Unknown ordering. Legal values are \"ascending\", \"descending\"");
+                }
+            }
+            if (line.hasOption("skipbucketsonfatalerrors")) {
+                params.skipBucketsOnFatalErrors(true);
+            }
+            if (line.hasOption("maxpendingsuperbuckets")) {
+                StaticThrottlePolicy throttlePolicy = new StaticThrottlePolicy();
+                throttlePolicy.setMaxPendingCount(((Number)line.getParsedOptionValue("maxpendingsuperbuckets")).intValue());
+                params.setThrottlePolicy(throttlePolicy);
+            }
+            if (line.hasOption("jsonoutput")) {
+                allParams.setJsonOutput(true);
+            }
+
+            allParams.setVisitorParameters(params);
+            return allParams;
+        }
+    }
+
+    // For unit testing only
+    protected void setVdsVisitParameters(VdsVisitParameters vdsVisitParameters) {
+        this.params = vdsVisitParameters;
+    }
+
+    protected static String resolveClusterRoute(ClusterList clusters, String wantedCluster) {
+        if (clusters.getStorageClusters().size() == 0) {
+            throw new IllegalArgumentException("Your Vespa cluster does not have any content clusters " +
+                    "declared. Visiting feature is not available.");
+        }
+
+        ClusterDef found = null;
+
+        String names = "";
+        for (ClusterDef c : clusters.getStorageClusters()) {
+            if (!names.isEmpty()) {
+                names += ", ";
+            }
+            names += c.getName();
+        }
+        if (wantedCluster != null) {
+            for (ClusterDef c : clusters.getStorageClusters()) {
+                if (c.getName().equals(wantedCluster)) {
+                    found = c;
+                }
+            }
+            if (found == null) {
+                throw new IllegalArgumentException("Your vespa cluster contains the content clusters " +
+                        names + ", not " + wantedCluster + ". Please select a valid vespa cluster.");
+            }
+        } else if (clusters.getStorageClusters().size() == 1) {
+            found = clusters.getStorageClusters().get(0);
+        } else {
+            throw new IllegalArgumentException("Your vespa cluster contains the content clusters " +
+                    names + ". Please use the -c option to select one of them as a target for visiting.");
+        }
+
+        return "[Storage:cluster=" + found.getName() + ";clusterconfigid=" + found.getConfigId() + "]";
+    }
+
+    protected static void verbosePrintParameters(VdsVisitParameters vdsParams, PrintStream out) {
+        VisitorParameters params = vdsParams.getVisitorParameters();
+        if (params.getTimeoutMs() != -1) {
+            out.println("Time out visitor after " + params.getTimeoutMs() + " ms.");
+        }
+        if (params.getDocumentSelection() == null || params.getDocumentSelection().equals("")) {
+            out.println("Visiting all documents");
+        } else {
+            out.println("Visiting documents matching: " + params.getDocumentSelection());
+        }
+        if (params.getFromTimestamp() != 0 && params.getToTimestamp() != 0) {
+            out.println("Visiting in the inclusive timestamp range "
+                               + params.getFromTimestamp() + " - " + params.getToTimestamp() + ".");
+        } else if (params.getFromTimestamp() != 0) {
+            out.println("Visiting from and including timestamp " + params.getFromTimestamp() + ".");
+        } else if (params.getToTimestamp() != 0) {
+            out.println("Visiting to and including timestamp " + params.getToTimestamp() + ".");
+        }
+        out.println("Visiting field set " + params.fieldSet() + ".");
+        if (params.visitInconsistentBuckets()) {
+            out.println("Visiting inconsistent buckets.");
+        }
+        if (params.visitRemoves()) {
+            out.println("Including remove entries.");
+        }
+        if (params.getResumeFileName() != null && !"".equals(params.getResumeFileName())) {
+            out.println("Tracking progress in file: " + params.getResumeFileName());
+        }
+        if (vdsParams.isPrintIdsOnly()) {
+            out.println("Only showing document identifiers.");
+        }
+        out.println("Let visitor have maximum " + params.getMaxPending() + " replies pending on data handlers per storage node visitor.");
+        out.println("Visit maximum " + params.getMaxBucketsPerVisitor() + " buckets per visitor.");
+        if (params.getRemoteDataHandler() != null) {
+            out.println("Sending data to data handler at: " + params.getRemoteDataHandler());
+        }
+        if (params.getRoute() != null) {
+            out.println("Visiting cluster '" + params.getRoute() + "'.");
+        }
+        if (params.getVisitorLibrary() != null) {
+            out.println("Using visitor library '" + params.getVisitorLibrary() + "'.");
+        }
+        if (params.getLibraryParameters().size() > 0) {
+            out.println("Adding the following library specific parameters:");
+            for (Map.Entry<String, byte[]> entry : params.getLibraryParameters().entrySet()) {
+                out.println("  " + entry.getKey() + " = " +
+                        new String(entry.getValue(), Charset.forName("utf-8")));
+            }
+        }
+        if (params.getPriority() != DocumentProtocol.Priority.NORMAL_3) {
+            out.println("Visitor priority " + params.getPriority().name());
+        }
+        if (params.skipBucketsOnFatalErrors()) {
+            out.println("Skip visiting super buckets with fatal errors.");
+        }
+    }
+
+    private void onDocumentSelectionException(Exception e) {
+        System.err.println("Illegal document selection string '" +
+                params.getVisitorParameters().getDocumentSelection() + "'.\n");
+        System.exit(1);
+    }
+
+    private void onIllegalArgumentException(Exception e) {
+        System.err.println("Illegal arguments : \n");
+        System.err.println(e.getMessage());
+        System.exit(1);
+    }
+
+    public void run() {
+        System.exit(doRun());
+    }
+
+    protected int doRun() {
+        VisitorParameters visitorParameters = params.getVisitorParameters();
+        // If progress file already exists, create resume token from it
+        if (visitorParameters.getResumeFileName() != null &&
+            !"".equals(visitorParameters.getResumeFileName()))
+        {
+            try {
+                File file = new File(visitorParameters.getResumeFileName());
+                FileInputStream fos = new FileInputStream(file);
+
+                StringBuilder builder = new StringBuilder();
+                byte[] b = new byte[100000];
+                int length;
+
+                while ((length = fos.read(b)) > 0) {
+                    builder.append(new String(b, 0, length));
+                }
+                fos.close();
+                visitorParameters.setResumeToken(new ProgressToken(builder.toString()));
+
+                if (params.isVerbose()) {
+                    System.err.format("Resuming visitor already %.1f %% finished.\n",
+                            visitorParameters.getResumeToken().percentFinished());
+                }
+            } catch (FileNotFoundException e) {
+                // Ignore; file has not been created yet but will be shortly.
+            } catch (IOException e) {
+                System.err.println("Could not open progress file: " + visitorParameters.getResumeFileName());
+                e.printStackTrace(System.err);
+                return 1;
+            }
+        }
+
+        initShutdownHook();
+        sessionAccessor = sessionAccessorFactory.createVisitorSessionAccessor();
+
+        VdsVisitHandler handler;
+
+        handler = new StdOutVisitorHandler(
+                params.isPrintIdsOnly(),
+                params.isVerbose(),
+                params.isVerbose(),
+                params.isVerbose(),
+                params.getStatisticsParts() != null,
+                params.getAbortOnClusterDown(),
+                params.getProcessTime(),
+                params.jsonOutput);
+
+        if (visitorParameters.getResumeFileName() != null) {
+            handler.setProgressFileName(visitorParameters.getResumeFileName());
+        }
+
+        visitorParameters.setControlHandler(handler.getControlHandler());
+        if (visitorParameters.getRemoteDataHandler() == null) {
+            visitorParameters.setLocalDataHandler(handler.getDataHandler());
+        }
+
+        if (params.getStatisticsParts() != null) {
+            String[] parts = params.getStatisticsParts().split(",");
+            for (String s : parts) {
+                visitorParameters.setLibraryParameter(s, "true");
+            }
+        }
+
+        try {
+            session = sessionAccessor.createVisitorSession(visitorParameters);
+            while (true) {
+                try {
+                    if (session.waitUntilDone(params.getFullTimeout())) break;
+                } catch (InterruptedException e) {}
+            }
+
+            if (visitorParameters.getTraceLevel() > 0) {
+                System.out.println(session.getTrace().toString());
+            }
+        } catch (ParseException e) {
+            onDocumentSelectionException(e);
+        } catch (IllegalArgumentException e) {
+            onIllegalArgumentException(e);
+        } catch (Exception e) {
+            System.err.println("Document selection string was: " + visitorParameters.getDocumentSelection());
+            System.err.println("Caught unexpected exception: ");
+            e.printStackTrace(System.err);
+            return 1;
+        }
+        if (visitorParameters.getControlHandler().getResult().code
+                == VisitorControlHandler.CompletionCode.SUCCESS)
+        {
+            return 0;
+        } else {
+            return 1;
+        }
+    }
+
+    private void initShutdownHook() {
+        shutdownHookRegistrar.registerShutdownHook(new CleanUpThread());
+    }
+
+    class CleanUpThread extends Thread {
+        public void run() {
+            try {
+                if (session != null) {
+                    session.destroy();
+                }
+            } catch (IllegalStateException ise) {
+                //ignore this
+            }
+            try {
+                if (sessionAccessor != null) {
+                    sessionAccessor.shutdown();
+                }
+            } catch (IllegalStateException ise) {
+                //ignore this too
+            }
+        }
+    }
+}
diff --git a/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisitHandler.java b/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisitHandler.java
new file mode 100644
index 00000000000..ad3e0a0b359
--- /dev/null
+++ b/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisitHandler.java
@@ -0,0 +1,181 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespavisit;
+
+import com.yahoo.documentapi.ProgressToken;
+import com.yahoo.documentapi.VisitorControlHandler;
+import com.yahoo.documentapi.VisitorDataHandler;
+import com.yahoo.vdslib.VisitorStatistics;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+
+import java.util.Date;
+import java.util.TimeZone;
+import java.text.DateFormat;
+import java.text.DecimalFormat;
+import java.text.SimpleDateFormat;
+
+/**
+ * An abstract class that can be subclassed by different visitor handlers.
+ *
+ * @author <a href="mailto:thomasg@yahoo-inc.com">Thomas Gundersen</a>
+ */
+public abstract class VdsVisitHandler {
+    boolean showProgress;
+    boolean showStatistics;
+    boolean abortOnClusterDown;
+    boolean lastLineIsProgress = false;
+    String lastPercentage;
+    final Object printLock = new Object();
+
+    protected String progressFileName = "";
+
+    final VisitorControlHandler controlHandler = new ControlHandler();
+
+    public VdsVisitHandler(boolean showProgress, boolean showStatistics, boolean abortOnClusterDown)
+    {
+        this.showProgress = showProgress;
+        this.showStatistics = showStatistics;
+        this.abortOnClusterDown = abortOnClusterDown;
+    }
+
+    public boolean getShowProgress() {
+        return showProgress;
+    }
+
+    public boolean getShowStatistics() {
+        return showStatistics;
+    }
+
+    public boolean getAbortOnClusterDown() {
+        return abortOnClusterDown;
+    }
+
+    public boolean getLastLineIsProgress() {
+        return lastLineIsProgress;
+    }
+
+    public void setLastLineIsProgress(boolean isProgress) {
+        lastLineIsProgress = isProgress;
+    }
+
+    public String getLastPercentage() {
+        return lastPercentage;
+    }
+
+    public void setLastPercentage(String lastPercentage) {
+        this.lastPercentage = lastPercentage;
+    }
+
+    public Object getPrintLock() {
+        return printLock;
+    }
+
+    public void onDone() {
+    }
+
+    public String getProgressFileName() {
+        return progressFileName;
+    }
+
+    public void setProgressFileName(String progressFileName) {
+        this.progressFileName = progressFileName;
+    }
+
+    public VisitorControlHandler getControlHandler() { return controlHandler; }
+    public abstract VisitorDataHandler getDataHandler();
+
+    class ControlHandler extends VisitorControlHandler {
+        VisitorStatistics statistics;
+
+        public void onProgress(ProgressToken token) {
+            if (progressFileName.length() > 0) {
+                 try {
+                     synchronized (token) {
+                         File file = new File(progressFileName + ".tmp");
+                         FileOutputStream fos = new FileOutputStream(file);
+                         fos.write(token.toString().getBytes());
+                         fos.close();
+                         file.renameTo(new File(progressFileName));
+                     }
+                 }
+                 catch (IOException e) {
+                     e.printStackTrace();
+                 }
+            }
+            if (showProgress) {
+                synchronized (printLock) {
+                    DecimalFormat df = new DecimalFormat("#.#");
+                    String percentage = df.format(token.percentFinished());
+                    if (!percentage.equals(lastPercentage)) {
+                        if (lastLineIsProgress) {
+                            System.err.print('\r');
+                        }
+                        System.err.print(percentage + " % finished.");
+                        lastLineIsProgress = true;
+                        lastPercentage = percentage;
+                    }
+                }
+            }
+            super.onProgress(token);
+        }
+
+        @Override
+        public void onVisitorStatistics(VisitorStatistics visitorStatistics) {
+            statistics = visitorStatistics;
+        }
+
+        private String getDateTime() {
+            DateFormat dateFormat =
+                new SimpleDateFormat("yyyy-MM-dd HH:mm:ss zzz");
+            dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+            Date date = new Date();
+            return dateFormat.format(date);
+        }
+
+        public void onVisitorError(String message) {
+            synchronized (printLock) {
+                if (lastLineIsProgress) {
+                    System.err.print('\r');
+                    lastLineIsProgress = false;
+                }
+                System.err.println("Visitor error (" + getDateTime() + "): " +
+                                   message);
+                if (abortOnClusterDown &&
+                    !isDone() &&
+                    (message.lastIndexOf("Could not resolve")>=0 ||
+                     message.lastIndexOf("don't allow external load")>=0)) {
+                    System.err.println("Aborting visitor as " +
+                                       "--abortonclusterdown flag is set.");
+                    abort();
+                }
+            }
+        }
+        public void onDone(CompletionCode code, String message) {
+            if (lastLineIsProgress) {
+                System.err.print('\n');
+                lastLineIsProgress = false;
+            }
+            if (code != CompletionCode.SUCCESS) {
+                if (code == CompletionCode.ABORTED) {
+                    System.err.println("Visitor aborted: " + message);
+                } else if (code == CompletionCode.TIMEOUT) {
+                    System.err.println("Visitor timed out: " + message);
+                } else {
+                    System.err.println("Visitor aborted due to unknown issue "
+                                     + code + ": " + message);
+                }
+            } else {
+                if (showProgress) {
+                    System.err.println("Completed visiting.");
+                }
+                if (showStatistics) {
+                    System.err.println("*** Visitor statistics");
+                    System.err.println(statistics == null ? "Nothing visited" : statistics.toString());
+                }
+            }
+            super.onDone(code, message);
+        }
+    }
+}
diff --git a/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisitTarget.java b/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisitTarget.java
new file mode 100644
index 00000000000..3ef0619cfd8
--- /dev/null
+++ b/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisitTarget.java
@@ -0,0 +1,286 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespavisit;
+
+import com.yahoo.documentapi.DocumentAccess;
+import com.yahoo.documentapi.VisitorControlHandler;
+import com.yahoo.documentapi.VisitorDataHandler;
+import com.yahoo.documentapi.VisitorDestinationParameters;
+import com.yahoo.documentapi.VisitorDestinationSession;
+import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess;
+import com.yahoo.documentapi.messagebus.MessageBusParams;
+import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet;
+import com.yahoo.log.LogLevel;
+import com.yahoo.log.LogSetup;
+import com.yahoo.messagebus.network.Identity;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.DefaultParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+
+import java.lang.reflect.Constructor;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.logging.Logger;
+
+/**
+ * Example client using visiting
+ *
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>, based on work by <a href="mailto:humbe@yahoo-inc.com">H&aring;kon Humberset</a>
+ */
+public class VdsVisitTarget {
+    private static final Logger log = Logger.getLogger(VdsVisitTarget.class.getName());
+
+    private boolean printIds = false;
+    DocumentAccess access;
+    VisitorDestinationSession session;
+    String slobrokAddress = null;
+    int port = -1;
+    private boolean verbose = false;
+    private int processTime = 0;
+    private String handlerClassName = StdOutVisitorHandler.class.getName();
+    private String[] handlerArgs = null;
+
+    public boolean isPrintIds() {
+        return printIds;
+    }
+
+    public String getSlobrokAddress() {
+        return slobrokAddress;
+    }
+
+    public boolean isVerbose() {
+        return verbose;
+    }
+
+    public int getPort() {
+        return port;
+    }
+
+    public int getProcessTime() {
+        return processTime;
+    }
+
+    public String getHandlerClassName() {
+        return handlerClassName;
+    }
+
+    public String[] getHandlerArgs() {
+        return handlerArgs;
+    }
+
+    public static void main(String args[]) {
+        LogSetup.initVespaLogging("vespavisittarget");
+        VdsVisitTarget visitTarget = new VdsVisitTarget();
+
+
+        try {
+            visitTarget.parseArguments(args);
+            visitTarget.initShutdownHook();
+            visitTarget.run();
+            System.exit(0);
+        } catch (HelpShownException e) {
+            System.exit(0);
+        } catch (IllegalArgumentException e) {
+            System.err.println(e.getMessage());
+            System.exit(1);
+        } catch (org.apache.commons.cli.ParseException e) {
+            System.err.println("Failed to parse arguments. Try --help for syntax. " + e.getMessage());
+            System.exit(1);
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+
+    private static Options createOptions() {
+        Options options = new Options();
+
+        options.addOption("h", "help", false, "Show this syntax page.");
+
+        options.addOption(Option.builder("s")
+                .longOpt("bindtoslobrok")
+                .hasArg(true)
+                .argName("address")
+                .desc("Bind to the given slobrok address.")
+                .build());
+
+        options.addOption(Option.builder("t")
+                .longOpt("bindtosocket")
+                .hasArg(true)
+                .argName("port")
+                .desc("Bind to the given TCP port")
+                .type(Number.class)
+                .build());
+
+        options.addOption(Option.builder("p")
+                .longOpt("processtime")
+                .hasArg(true)
+                .argName("msecs")
+                .desc("Sleep this amount of millisecs before processing message. (Debug option for pretending to be slow client).")
+                .type(Number.class)
+                .build());
+
+        options.addOption(Option.builder("c")
+                .longOpt("visithandler")
+                .hasArg(true)
+                .argName("classname")
+                .desc("Use the given class as a visit handler (defaults to StdOutVisitorHandler)")
+                .build());
+
+        options.addOption(Option.builder("o")
+                .longOpt("visitoptions")
+                .hasArg(true)
+                .argName("args")
+                .desc("Option arguments to pass through to the visitor handler instance")
+                .build());
+
+        options.addOption("i", "printids", false, "Display only document identifiers.");
+        options.addOption("v", "verbose", false, "Indent XML, show progress and info on STDERR.");
+
+        return options;
+    }
+
+    private void printSyntax(Options options) {
+        HelpFormatter formatter = new HelpFormatter();
+        formatter.printHelp("vespavisittarget <options>", "Retrieve results from a visitor", options ,
+                            "One, and only one, of the binding options must be present.\n" +
+                            "\n" +
+                            "For more detailed information, such as defaults and format of\n" +
+                            "arguments, refer to 'man vespavisittarget'.\n");
+    }
+
+    class HelpShownException extends Exception {}
+
+    void parseArguments(String args[]) throws ParseException, HelpShownException {
+        Options options = createOptions();
+
+        CommandLineParser parser = new DefaultParser();
+        CommandLine line = parser.parse(options, args);
+
+        if (line.hasOption("h")) {
+            printSyntax(options);
+            throw new HelpShownException();
+        }
+        if (line.hasOption("s")) {
+            slobrokAddress = line.getOptionValue("s");
+        }
+        if (line.hasOption("t")) {
+            port = ((Number) line.getParsedOptionValue("t")).intValue();
+        }
+        if (line.hasOption("i")) {
+            printIds = true;
+        }
+        if (line.hasOption("p")) {
+            processTime = ((Number) line.getParsedOptionValue("p")).intValue();
+        }
+        if (line.hasOption("v")) {
+            verbose = true;
+        }
+        if (line.hasOption("c")) {
+            handlerClassName = line.getOptionValue("c");
+        }
+        if (line.hasOption("o")) {
+            handlerArgs = line.getOptionValue("o").split(" ");
+        }
+
+        if (!(slobrokAddress == null ^ port == -1)) {
+            throw new IllegalArgumentException("You must specify one, and only one, binding option");
+        }
+        if (port != -1 && port < 0 || port > 65535) {
+            throw new IllegalArgumentException("The port must be in the range 0-65535");
+        }
+        if (verbose) {
+            if (port != -1) {
+                System.err.println("Binding to socket " + getTcpAddress());
+            } else {
+                System.err.println("Binding to slobrok address: " + slobrokAddress + "/visit-destination");
+            }
+        }
+    }
+
+    private String getTcpAddress() {
+        try {
+            InetAddress addr = InetAddress.getLocalHost();
+            String hostname = addr.getHostName();
+            return "tcp/" + hostname + ":" + port + "/visit-destination";
+        } catch (UnknownHostException e) {
+            System.err.println("Failed to detect hostname.");
+            System.exit(1);
+        }
+        return "";
+    }
+
+    @SuppressWarnings("unchecked")
+    public void run() throws Exception {
+        initShutdownHook();
+        log.log(LogLevel.DEBUG, "Starting VdsVisitTarget");
+        MessageBusParams mbusParams = new MessageBusParams(new LoadTypeSet());
+        mbusParams.getRPCNetworkParams().setIdentity(new Identity(slobrokAddress));
+
+        if (port > 0) {
+            mbusParams.getRPCNetworkParams().setListenPort(port);
+        }
+
+        access = new MessageBusDocumentAccess(mbusParams);
+
+        VdsVisitHandler handler;
+
+        Class<?> cls = Thread.currentThread().getContextClassLoader()
+                .loadClass(handlerClassName);
+        try {
+            // Any custom data handlers may have a constructor that takes in args,
+            // so that the user can pass cmd line options to them
+            Class<?>[] consTypes = new Class<?>[] { boolean.class, boolean.class,
+                    boolean.class, boolean.class, boolean.class,
+                    boolean.class, int.class, String[].class };
+            Constructor<?> cons = cls.getConstructor(consTypes);
+            handler = (VdsVisitHandler)cons.newInstance(
+                    printIds, verbose, verbose, verbose, false, false,
+                    processTime, handlerArgs);
+        } catch (NoSuchMethodException e) {
+            // Retry, this time matching the StdOutVisitorHandler constructor
+            // arg list
+            Class<?>[] consTypes = new Class<?>[] { boolean.class, boolean.class,
+                    boolean.class, boolean.class, boolean.class,
+                    boolean.class, int.class, boolean.class };
+            Constructor<?> cons = cls.getConstructor(consTypes);
+            handler = (VdsVisitHandler)cons.newInstance(
+                    printIds, verbose, verbose, verbose, false, false, processTime, false);
+        }
+
+        VisitorDataHandler dataHandler = handler.getDataHandler();
+        VisitorControlHandler controlHandler = handler.getControlHandler();
+
+        VisitorDestinationParameters params = new VisitorDestinationParameters(
+                "visit-destination", dataHandler);
+        session = access.createVisitorDestinationSession(params);
+        while (!controlHandler.isDone()) {
+            Thread.sleep(1000);
+        }
+    }
+
+    private void initShutdownHook() {
+        Runtime.getRuntime().addShutdownHook(new CleanUpThread());
+    }
+
+    class CleanUpThread extends Thread {
+        public void run() {
+            try {
+                if (session != null) {
+                    session.destroy();
+                }
+            } catch (IllegalStateException ise) {
+                //ignore this
+            }
+            try {
+                if (access != null) {
+                    access.shutdown();
+                }
+            } catch (IllegalStateException ise) {
+                //ignore this too
+            }
+        }
+    }
+}
diff --git a/vespaclient-java/src/main/sh/vds-document-statistics.sh b/vespaclient-java/src/main/sh/vds-document-statistics.sh
new file mode 100755
index 00000000000..3677137dbd4
--- /dev/null
+++ b/vespaclient-java/src/main/sh/vds-document-statistics.sh
@@ -0,0 +1,20 @@
+#!/bin/sh
+test -z "$VESPA_HOME" && VESPA_HOME=/home/y
+
+. $VESPA_HOME/libexec/vespa/common-env.sh
+
+function help {
+    echo "Usage: vds-document-statistics [ category, ... ]"
+    echo "  Where category is one or more of: user, group, scheme, namespace"
+    echo ""
+    echo "vds-document-statistics generates documents counts based on one or more categories."
+    exit 0
+}
+if [ "$1" == "-h" ]; then
+  help
+fi
+if [ "$1" == "" ]; then
+  help
+fi
+export MALLOC_ARENA_MAX=1 #Does not need fast allocation
+exec java -Xms32m -Xmx128m $(getJavaOptionsIPV46) -cp ${VESPA_HOME}/lib/jars/vespaclient-java-jar-with-dependencies.jar com.yahoo.vespavisit.Main --statistics "$1"
diff --git a/vespaclient-java/src/main/sh/vdsstat.sh b/vespaclient-java/src/main/sh/vdsstat.sh
new file mode 100644
index 00000000000..ef3e2cdbe20
--- /dev/null
+++ b/vespaclient-java/src/main/sh/vdsstat.sh
@@ -0,0 +1,13 @@
+#!/bin/sh
+test -z "$VESPA_HOME" && VESPA_HOME=/home/y
+
+. $VESPA_HOME/libexec/vespa/common-env.sh
+
+export MALLOC_ARENA_MAX=1 #Does not need fast allocation
+exec java \
+-server -enableassertions \
+-XX:ThreadStackSize=512 \
+-XX:MaxJavaStackTraceDepth=-1 \
+-Djava.awt.headless=true \
+-Xms128m -Xmx1024m $(getJavaOptionsIPV46) \
+-cp ${VESPA_HOME}/lib/jars/vespaclient-java-jar-with-dependencies.jar com.yahoo.vespastat.Main "$@"
diff --git a/vespaclient-java/src/main/sh/vespa-query-profile-dump-tool.sh b/vespaclient-java/src/main/sh/vespa-query-profile-dump-tool.sh
new file mode 100755
index 00000000000..1a70fcb006d
--- /dev/null
+++ b/vespaclient-java/src/main/sh/vespa-query-profile-dump-tool.sh
@@ -0,0 +1,6 @@
+#!/bin/sh
+test -z "$VESPA_HOME" && VESPA_HOME=/home/y
+
+. $VESPA_HOME/libexec/vespa/common-env.sh
+
+java $(getJavaOptionsIPV46) -cp ${VESPA_HOME}/lib/jars/vespaclient-java-jar-with-dependencies.jar com.yahoo.search.query.profile.DumpTool $@
diff --git a/vespaclient-java/src/main/sh/vespa-summary-benchmark.sh b/vespaclient-java/src/main/sh/vespa-summary-benchmark.sh
new file mode 100755
index 00000000000..7534639d07b
--- /dev/null
+++ b/vespaclient-java/src/main/sh/vespa-summary-benchmark.sh
@@ -0,0 +1,15 @@
+#!/bin/sh
+test -z "$VESPA_HOME" && VESPA_HOME=/home/y
+
+. $VESPA_HOME/libexec/vespa/common-env.sh
+
+export VESPA_LOG_TARGET=file:/dev/null
+export MALLOC_ARENA_MAX=1 # Does not need fast allocation
+java \
+-server -enableassertions \
+-XX:ThreadStackSize=512 \
+-XX:MaxJavaStackTraceDepth=-1 \
+-Djava.library.path=${VESPA_HOME}/libexec64/native:${VESPA_HOME}/lib64 \
+-XX:MaxDirectMemorySize=32m -Djava.awt.headless=true \
+-Xms128m -Xmx1024m $(getJavaOptionsIPV46) \
+-cp ${VESPA_HOME}/lib/jars/vespaclient-java-jar-with-dependencies.jar com.yahoo.vespasummarybenchmark.VespaSummaryBenchmark "$@"
diff --git a/vespaclient-java/src/main/sh/vespadestination.sh b/vespaclient-java/src/main/sh/vespadestination.sh
new file mode 100755
index 00000000000..f2168cb6db2
--- /dev/null
+++ b/vespaclient-java/src/main/sh/vespadestination.sh
@@ -0,0 +1,12 @@
+#!/bin/sh
+test -z "$VESPA_HOME" && VESPA_HOME=/home/y
+
+. $VESPA_HOME/libexec/vespa/common-env.sh
+
+export MALLOC_ARENA_MAX=1 #Does not need fast allocation
+exec java \
+-server -enableassertions \
+-XX:ThreadStackSize=512 \
+-Djava.library.path=${VESPA_HOME}/libexec64/native:${VESPA_HOME}/lib64 \
+-XX:MaxDirectMemorySize=32m -Djava.awt.headless=true $(getJavaOptionsIPV46) \
+-cp ${VESPA_HOME}/lib/jars/vespaclient-java-jar-with-dependencies.jar:$CLASSPATH com.yahoo.dummyreceiver.DummyReceiver "$@"
diff --git a/vespaclient-java/src/main/sh/vespafeeder.sh b/vespaclient-java/src/main/sh/vespafeeder.sh
new file mode 100755
index 00000000000..f74bc794ed7
--- /dev/null
+++ b/vespaclient-java/src/main/sh/vespafeeder.sh
@@ -0,0 +1,15 @@
+#!/bin/sh
+test -z "$VESPA_HOME" && VESPA_HOME=/home/y
+
+. $VESPA_HOME/libexec/vespa/common-env.sh
+
+export VESPA_LOG_TARGET=file:/dev/null
+export MALLOC_ARENA_MAX=1 # Does not need fast allocation
+java \
+-server -enableassertions \
+-XX:ThreadStackSize=512 \
+-XX:MaxJavaStackTraceDepth=-1 \
+-Djava.library.path=${VESPA_HOME}/libexec64/native:${VESPA_HOME}/lib64 \
+-XX:MaxDirectMemorySize=32m -Djava.awt.headless=true \
+-Xms128m -Xmx1024m $(getJavaOptionsIPV46) \
+-cp ${VESPA_HOME}/lib/jars/vespaclient-java-jar-with-dependencies.jar com.yahoo.vespafeeder.VespaFeeder "$@"
diff --git a/vespaclient-java/src/main/sh/vespaget.sh b/vespaclient-java/src/main/sh/vespaget.sh
new file mode 100644
index 00000000000..514ff170742
--- /dev/null
+++ b/vespaclient-java/src/main/sh/vespaget.sh
@@ -0,0 +1,14 @@
+#!/bin/sh
+test -z "$VESPA_HOME" && VESPA_HOME=/home/y
+
+. $VESPA_HOME/libexec/vespa/common-env.sh
+
+export MALLOC_ARENA_MAX=1 #Does not need fast allocation
+exec java \
+-server -enableassertions \
+-XX:ThreadStackSize=512 \
+-XX:MaxJavaStackTraceDepth=-1 \
+-Djava.awt.headless=true \
+-DVESPA_LOG_LEVEL="all -debug -spam -config -info -event" \
+-Xms128m -Xmx1024m $(getJavaOptionsIPV46) \
+-cp ${VESPA_HOME}/lib/jars/vespaclient-java-jar-with-dependencies.jar com.yahoo.vespaget.Main "$@"
diff --git a/vespaclient-java/src/main/sh/vespavisit.1 b/vespaclient-java/src/main/sh/vespavisit.1
new file mode 100644
index 00000000000..b9a6c488bf9
--- /dev/null
+++ b/vespaclient-java/src/main/sh/vespavisit.1
@@ -0,0 +1,159 @@
+.TH VESPAVISIT 1 2008-03-07 "Vespa" "Vespa Documentation"
+.SH NAME
+vespavisit \- Visit documents from a Vespa installation
+.SH SYNPOSIS
+.B vespavisit
+[\fIOPTION\fR]...
+.SH DESCRIPTION
+.PP
+In the regular case, retrieve documents stored in VESPA, and either print
+them to STDOUT or send them to a given MessageBus route.
+.PP
+A Vespa visit operation processes a set of stored documents, in undefined
+order, locally on the storage nodes where they are stored. A visitor library
+available on all storage nodes will receive the documents stored locally, and
+can process these and send messages to the visitor data handler. The regular
+case is to use the DumpVisitor library to merely send the documents themselves
+in blocks back to the data handler, which by default is this client that will
+write the documents to STDOUT.
+.PP
+Mandatory arguments to long options are mandatory for short options too.
+Short options can not currently be concatenated together.
+.TP
+\fB\-s\fR, \fB\-\-selection\fR \fISELECTION\fR
+A document selection string, specifying what documents to visit. Documentation
+on the language itself can be found in the documentation. Note that this argument
+should probably be quoted to prevent your shell from invalidating your
+selection.
+.TP
+\fB\-f\fR, \fB\-\-from\fR \fITIME\fR
+If this option is given, only documents from given timestamp or newer will be
+visited. The time is given in microseconds since 1970.
+.TP
+\fB\-t\fR, \fB\-\-to\fR \fITIME\fR
+If this option is given, only documents up to and including the given timestamp
+will be visited. The time is given in microseconds since 1970.
+.TP
+\fB\-e\fR, \fB\-\-headersonly\fR
+By default, the whole documents stored are processed. If this option is given
+only the header parts of documents will be processed. By defining the big
+document fields as body fields, you can efficiently visit all the header fields
+using this option.
+.TP
+\fB\-i\fR, \fB\-\-printids\fR
+Using this option, only the document identifiers will be printed to STDOUT.
+In addition, if visiting removes, an additional tag will be added so you can
+see whether document has been removed or not. This option implies headers only
+visiting, and can only be used if no datahandler is specified.
+.TP
+\fB\-d\fR, \fB\-\-datahandler\fR \fIVISITTARGET\fR
+The data handler is the destination of messages sent from the visitor library.
+By default, the data handler is the vespavisit process you start, which will
+merely print all returned data to STDOUT. A visit target can be specified
+instead. See the chapter below on visit targets.
+.TP
+\fB\-p\fR, \fB\-\-progress\fR \fIFILE\fR
+By setting a progress file, current visitor progress will be saved to this
+file at regular intervals. If this file exists on startup, the visitor will
+continue from this point.
+.TP
+\fB\-o\fR, \fB\-\-timeout\fR \fITIMEOUT\fR
+Time out the visitor after given number of milliseconds.
+.TP
+\fB\-r\fR, \fB\-\-visitremoves\fR
+By default, only documents existing in Vespa will be processed. By giving
+this option, also entries identifying documents previously existing will
+be returned. This is useful for secondary copies of data that wants to know
+whether documents it has stored has been removed. Note that documents deleted
+a long time ago will no longer be tracked. Vespa keeps remove entries for
+a configurable amount of time.
+.TP
+\fB\-m\fR, \fB\-\-maxpending\fR \fINUM\fR
+Maximum pending docblock messages to data handlers. This may be used to
+increase or reduce visiting speed, but should not be set too high so that data
+handlers run out of memory. To get an estimate of memory consumption on each
+data handler, multiply maxpending with defaultdocblocksize in stor-visitor
+config and divide by number of data handlers. Default value for maxpending is
+16.
+.TP
+\fB\-c\fR, \fB\-\-cluster\fR \fICLUSTER\fR
+Visit the given VDS cluster.
+.TP
+\fB\-v\fR, \fB\-\-verbose\fR
+More verbose output. Indent XML and add progress and info to STDERR.
+.TP
+\fB\-h\fR, \fB\-\-help\fR
+Shows a short syntax reminder.
+.PP
+Advanced options:
+.PP
+The below options are used for advanced usage or for testing.
+.TP
+\fB\-\-visitlibrary\fR \fILIBRARY\fR
+By default, the DumpVisitor library, sending documents back to the data handler,
+is used when visiting. Another library can be specified using this option. The
+library filename should be the name given here, with lib prepended and .so
+appended.
+.TP
+\fB\-\-libraryparam\fR \fIKEY\fR \fIVALUE\fR
+The default DumpVisitor library has no options to set, but custom libraries
+may need user specifiable options. Here such options can be specified. Look
+at visitor library documentation for legal parameters.
+.TP
+\fB\-\-polling\fR \fIarg\fR
+The document API implements both a polling and a callback visitor API. The
+callback API is most efficient and used by default. The polling API might be
+simpler for users used to such APIs. Some VESPA system tests use this option
+to test that the polling API works.
+.TP
+\fB\-\-visitinconsistentbuckets\fR
+In some cases Vespa may temporarily be in an inconsistent state, that is,
+different nodes contain different copies of the data. Collections of documents
+are grouped into so-called buckets. The normal behavior of visiting is to wait
+for the inconsistencies to resolve before actually visiting the data. This
+might be a problem for time critical applications. Setting this option will
+result in the bucket copy with most documents to be visited in case of
+inconsistencies, which means that the data returned by the visitor are not
+guaranteed to be correct.
+.SH VISIT TARGET
+Results from visiting can be sent to many different kind of targets.
+.TP
+\fBMessage bus routes\fR
+You can specify a message bus route name directly, and this route will be used
+to send the results. This is typically used when doing reprocessing within
+Vespa. Message bus routes are set up in the application package. In addition
+some routes may have been autogenerated in simple setups, for instance a
+route called \fIdefault\fR is generated if your setup is so simple that Vespa
+can guess where you want to send your data.
+.TP
+\fBSlobrok address\fR
+You can also specify a slobrok address for data to be sent to. A slobrok address
+is a slash separated path where you can use asterisk to mean any element within
+this path. For instance, if you have a docproc cluster called \fImydpcluster\fR
+it will have registered its nodes with slobrok names like
+\fIdocproc/cluster.mydpcluster/docproc/0/feed_processor\fR, where the 0 here
+indicates the first node in the cluster. You can thus specify to send visit data
+to this docproc cluster by stating a slobrok address of
+\fIdocproc/cluster.mydpcluster/docproc/*/feed_processor\fR. Note that this will
+not send all the data to one or all the nodes. The data sent from the visitor
+will be distributed among the matching nodes, but each message will just be sent
+to one node.
+
+Slobrok names may also be used if you use the \fBvespavisittarget\fR tool to
+retrieve the data at some location. If you start vespavisittarget on two nodes,
+listening to slobrok names \fImynode/0/visit-destination\fR and
+\fImynode/1/visit-destination\fR you can send the results to these nodes by
+specifying \fImynode/*/visit-destination\fR as the data handler. See
+\fBman vespavisittarget\fR for naming conventions used for such targets.
+.TP
+\fBTCP socket\fR
+TCP sockets can also be specified directly. This requires that the endpoint
+speaks FNET RPC though. This is typically done, either by using the
+\fBvespavisittarget\fR tool, or by using a visitor destination programmatically
+by using utility class in the document API. A socket address looks like the
+following: tcp/\fIhostname\fR:\fIport\fR/\fIservicename\fR. For instance, an
+address generated by the \fBvespavisittarget\fR tool might look like the
+following: \fItcp/myhost.com:12345/visit-destination\fR.
+
+.SH AUTHOR
+Written by Haakon Humberset.
diff --git a/vespaclient-java/src/main/sh/vespavisit.sh b/vespaclient-java/src/main/sh/vespavisit.sh
new file mode 100755
index 00000000000..eb6c9487b88
--- /dev/null
+++ b/vespaclient-java/src/main/sh/vespavisit.sh
@@ -0,0 +1,14 @@
+#!/bin/sh
+test -z "$VESPA_HOME" && VESPA_HOME=/home/y
+
+. $VESPA_HOME/libexec/vespa/common-env.sh
+
+export MALLOC_ARENA_MAX=1 #Does not need fast allocation
+exec java \
+-server -enableassertions \
+-XX:ThreadStackSize=512 \
+-XX:MaxJavaStackTraceDepth=-1 \
+-Djava.library.path=${VESPA_HOME}/libexec64/native:${VESPA_HOME}/lib64 \
+-XX:MaxDirectMemorySize=32m -Djava.awt.headless=true \
+-Xms128m -Xmx1024m $(getJavaOptionsIPV46) \
+-cp ${VESPA_HOME}/lib/jars/vespaclient-java-jar-with-dependencies.jar com.yahoo.vespavisit.VdsVisit "$@"
diff --git a/vespaclient-java/src/main/sh/vespavisittarget.1 b/vespaclient-java/src/main/sh/vespavisittarget.1
new file mode 100644
index 00000000000..7f02215d558
--- /dev/null
+++ b/vespaclient-java/src/main/sh/vespavisittarget.1
@@ -0,0 +1,40 @@
+.TH VESPAVISITTARGET 1 2008-03-07 "Vespa" "Vespa Documentation"
+.SH NAME
+vespavisittarget \- An endpoint for documents visited from a Vespa installation
+.SH SYNPOSIS
+.B vespavisittarget
+[\fIOPTION\fR]...
+.SH DESCRIPTION
+.PP
+When visiting data from Vespa, you might not want to send the data back to the
+controlling process. By using separate visitor targets you can divide load
+between multiple nodes and have the controlling process run at another location.
+The document API has utility classes to set up end points for visitor data from
+Vespa. This application is a small tool that uses these utilities and merely
+writes the data retrieved to STDOUT in XML format.
+.PP
+Mandatory arguments to long options are mandatory for short options too.
+Short options can not currently be concatenated together.
+.TP
+\fB\-s\fR, \fB\-\-bindtoslobrok\fR \fISLOBROKADDRESS\fR
+Bind to the given slobrok address. Note that the value \fI/visit-destination\fR
+will be appended to the given address.
+.TP
+\fB\-t\fR, \fB\-\-bindtosocket\fR \fIPORT\fR
+Bind to the given TCP socket. This will make sure we listen to the given port.
+No slobrok registration is done using this option, so you need to specify
+TCP socket address in visitors to get data sent to this destination.
+.TP
+\fB\-i\fR, \fB\-\-printids\fR
+Using this option, only the document identifiers will be printed to STDOUT.
+In addition, if visiting removes, an additional tag will be added so you can
+see whether document has been removed or not. This option implies headers only
+visiting, and can only be used if no datahandler is specified.
+.TP
+\fB\-v\fR, \fB\-\-verbose\fR
+More verbose output. Indent XML.
+.TP
+\fB\-h\fR, \fB\-\-help\fR
+Shows a short syntax reminder.
+.SH AUTHOR
+Written by Haakon Humberset.
diff --git a/vespaclient-java/src/main/sh/vespavisittarget.sh b/vespaclient-java/src/main/sh/vespavisittarget.sh
new file mode 100755
index 00000000000..7eb9fe17e04
--- /dev/null
+++ b/vespaclient-java/src/main/sh/vespavisittarget.sh
@@ -0,0 +1,13 @@
+#!/bin/sh
+test -z "$VESPA_HOME" && VESPA_HOME=/home/y
+
+. $VESPA_HOME/libexec/vespa/common-env.sh
+
+export MALLOC_ARENA_MAX=1 #Does not need fast allocation
+exec java \
+-server -enableassertions \
+-XX:ThreadStackSize=512 \
+-XX:MaxJavaStackTraceDepth=-1 \
+-Djava.library.path=${VESPA_HOME}/libexec64/native:${VESPA_HOME}/lib64 \
+-XX:MaxDirectMemorySize=32m -Djava.awt.headless=true $(getJavaOptionsIPV46) \
+-cp ${VESPA_HOME}/lib/jars/vespaclient-java-jar-with-dependencies.jar:$CLASSPATH com.yahoo.vespavisit.VdsVisitTarget "$@"
diff --git a/vespaclient-java/src/test/files/documentmanager.cfg b/vespaclient-java/src/test/files/documentmanager.cfg
new file mode 100644
index 00000000000..966361d1fe3
--- /dev/null
+++ b/vespaclient-java/src/test/files/documentmanager.cfg
@@ -0,0 +1,113 @@
+enablecompression false
+datatype[10]
+datatype[0].id 1002
+datatype[0].arraytype[1]
+datatype[0].arraytype[0].datatype 2
+datatype[0].weightedsettype[0]
+datatype[0].structtype[0]
+datatype[0].documenttype[0]
+datatype[1].id 1000
+datatype[1].arraytype[1]
+datatype[1].arraytype[0].datatype 0
+datatype[1].weightedsettype[0]
+datatype[1].structtype[0]
+datatype[1].documenttype[0]
+datatype[2].id 1004
+datatype[2].arraytype[1]
+datatype[2].arraytype[0].datatype 4
+datatype[2].weightedsettype[0]
+datatype[2].structtype[0]
+datatype[2].documenttype[0]
+datatype[3].id 1016
+datatype[3].arraytype[1]
+datatype[3].arraytype[0].datatype 16
+datatype[3].weightedsettype[0]
+datatype[3].structtype[0]
+datatype[3].documenttype[0]
+datatype[4].id 1001
+datatype[4].arraytype[1]
+datatype[4].arraytype[0].datatype 1
+datatype[4].weightedsettype[0]
+datatype[4].structtype[0]
+datatype[4].documenttype[0]
+datatype[5].id 2001
+datatype[5].arraytype[0]
+datatype[5].weightedsettype[1]
+datatype[5].weightedsettype[0].datatype 0
+datatype[5].weightedsettype[0].createifnonexistant false
+datatype[5].weightedsettype[0].removeifzero false
+datatype[5].structtype[0]
+datatype[5].documenttype[0]
+datatype[6].id 2002
+datatype[6].arraytype[0]
+datatype[6].weightedsettype[1]
+datatype[6].weightedsettype[0].datatype 2
+datatype[6].weightedsettype[0].createifnonexistant false
+datatype[6].weightedsettype[0].removeifzero false
+datatype[6].structtype[0]
+datatype[6].documenttype[0]
+datatype[7].id -628990518
+datatype[7].arraytype[0]
+datatype[7].weightedsettype[0]
+datatype[7].structtype[1]
+datatype[7].structtype[0].name news.header
+datatype[7].structtype[0].version 0
+datatype[7].structtype[0].field[6]
+datatype[7].structtype[0].field[0].name url
+datatype[7].structtype[0].field[0].id[0]
+datatype[7].structtype[0].field[0].datatype 10
+datatype[7].structtype[0].field[1].name title
+datatype[7].structtype[0].field[1].id[0]
+datatype[7].structtype[0].field[1].datatype 2
+datatype[7].structtype[0].field[2].name last_downloaded
+datatype[7].structtype[0].field[2].id[0]
+datatype[7].structtype[0].field[2].datatype 0
+datatype[7].structtype[0].field[3].name value_long
+datatype[7].structtype[0].field[3].id[0]
+datatype[7].structtype[0].field[3].datatype 4
+datatype[7].structtype[0].field[4].name value_content
+datatype[7].structtype[0].field[4].id[0]
+datatype[7].structtype[0].field[4].datatype 2
+datatype[7].structtype[0].field[5].name stringarr
+datatype[7].structtype[0].field[5].id[0]
+datatype[7].structtype[0].field[5].datatype 1002
+datatype[7].documenttype[0]
+datatype[8].id 538588767
+datatype[8].arraytype[0]
+datatype[8].weightedsettype[0]
+datatype[8].structtype[1]
+datatype[8].structtype[0].name news.body
+datatype[8].structtype[0].version 0
+datatype[8].structtype[0].field[7]
+datatype[8].structtype[0].field[0].name intarr
+datatype[8].structtype[0].field[0].id[0]
+datatype[8].structtype[0].field[0].datatype 1000
+datatype[8].structtype[0].field[1].name longarr
+datatype[8].structtype[0].field[1].id[0]
+datatype[8].structtype[0].field[1].datatype 1004
+datatype[8].structtype[0].field[2].name bytearr
+datatype[8].structtype[0].field[2].id[0]
+datatype[8].structtype[0].field[2].datatype 1016
+datatype[8].structtype[0].field[3].name floatarr
+datatype[8].structtype[0].field[3].id[0]
+datatype[8].structtype[0].field[3].datatype 1001
+datatype[8].structtype[0].field[4].name weightedsetint
+datatype[8].structtype[0].field[4].id[0]
+datatype[8].structtype[0].field[4].datatype 2001
+datatype[8].structtype[0].field[5].name weightedsetstring
+datatype[8].structtype[0].field[5].id[0]
+datatype[8].structtype[0].field[5].datatype 2002
+datatype[8].structtype[0].field[6].name content
+datatype[8].structtype[0].field[6].id[0]
+datatype[8].structtype[0].field[6].datatype 2
+datatype[8].documenttype[0]
+datatype[9].id -1048827947
+datatype[9].arraytype[0]
+datatype[9].weightedsettype[0]
+datatype[9].structtype[0]
+datatype[9].documenttype[1]
+datatype[9].documenttype[0].name news
+datatype[9].documenttype[0].version 0
+datatype[9].documenttype[0].inherits[0]
+datatype[9].documenttype[0].headerstruct -628990518
+datatype[9].documenttype[0].bodystruct 538588767
diff --git a/vespaclient-java/src/test/files/malformedfeed.json b/vespaclient-java/src/test/files/malformedfeed.json
new file mode 100644
index 00000000000..26691ada676
--- /dev/null
+++ b/vespaclient-java/src/test/files/malformedfeed.json
@@ -0,0 +1,13 @@
+[
+{
+    "put": "id:test:news::foo",
+    "fields": {}
+},
+{
+    "update": "id:test:news::foo",
+    "fields": {}
+},
+{
+    "remove": "id:test:news::foo"
+},
+]
diff --git a/vespaclient-java/src/test/files/myfeed.json b/vespaclient-java/src/test/files/myfeed.json
new file mode 100644
index 00000000000..544f370e62a
--- /dev/null
+++ b/vespaclient-java/src/test/files/myfeed.json
@@ -0,0 +1,13 @@
+[
+{
+    "put": "id:test:news::foo",
+    "fields": {}
+},
+{
+    "update": "id:test:news::foo",
+    "fields": {}
+},
+{
+    "remove": "id:test:news::foo"
+}
+]
diff --git a/vespaclient-java/src/test/files/myfeed.xml b/vespaclient-java/src/test/files/myfeed.xml
new file mode 100644
index 00000000000..008d4fcba13
--- /dev/null
+++ b/vespaclient-java/src/test/files/myfeed.xml
@@ -0,0 +1,5 @@
+<vespafeed>
+    <document documentid="doc:test:foo" documenttype="news"/>
+    <update documentid="doc:test:foo" documenttype="news"/>
+    <remove documentid="doc:test:foo"/>
+</vespafeed>
+\ No newline at end of file
diff --git a/vespaclient-java/src/test/files/progress.txt b/vespaclient-java/src/test/files/progress.txt
new file mode 100644
index 00000000000..77323926140
--- /dev/null
+++ b/vespaclient-java/src/test/files/progress.txt
@@ -0,0 +1,8 @@
+VDS bucket progress file
+14
+4704
+4701
+16384
+3800000000000004:8c00000600000004
+3800000000000002:8c00000000000002
+3800000000003e92:0
diff --git a/vespaclient-java/src/test/java/com/yahoo/vespafeeder/BenchmarkProgressPrinterTest.java b/vespaclient-java/src/test/java/com/yahoo/vespafeeder/BenchmarkProgressPrinterTest.java
new file mode 100644
index 00000000000..2b0b4cc9048
--- /dev/null
+++ b/vespaclient-java/src/test/java/com/yahoo/vespafeeder/BenchmarkProgressPrinterTest.java
@@ -0,0 +1,77 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespafeeder;
+
+import com.yahoo.clientmetrics.RouteMetricSet;
+import com.yahoo.concurrent.Timer;
+import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage;
+import com.yahoo.documentapi.messagebus.protocol.UpdateDocumentMessage;
+import com.yahoo.messagebus.EmptyReply;
+import junit.framework.TestCase;
+
+import java.io.ByteArrayOutputStream;
+import java.io.PrintStream;
+
+/**
+ */
+public class BenchmarkProgressPrinterTest extends TestCase {
+
+    class DummyTimer implements Timer {
+        long ms;
+
+        public long milliTime() { return ms; }
+    }
+
+    public void testSimple() {
+        ByteArrayOutputStream output = new ByteArrayOutputStream();
+        DummyTimer timer = new DummyTimer();
+        timer.ms = 0;
+        BenchmarkProgressPrinter printer = new BenchmarkProgressPrinter(timer, new PrintStream(output));
+        RouteMetricSet metrics = new RouteMetricSet("foobar", printer);
+
+        {
+            EmptyReply reply = new EmptyReply();
+            reply.setMessage(PutDocumentMessage.createEmpty().setTimeReceived(1));
+            metrics.addReply(reply);
+        }
+
+        timer.ms = 1200;
+
+        {
+            EmptyReply reply = new EmptyReply();
+            reply.setMessage(PutDocumentMessage.createEmpty().setTimeReceived(2));
+            metrics.addReply(reply);
+        }
+
+        {
+            EmptyReply reply = new EmptyReply();
+            reply.setMessage(UpdateDocumentMessage.createEmpty().setTimeReceived(3));
+            metrics.addReply(reply);
+        }
+
+        timer.ms = 2400;
+
+        {
+            EmptyReply reply = new EmptyReply();
+            reply.setMessage(UpdateDocumentMessage.createEmpty().setTimeReceived(4));
+            reply.addError(new com.yahoo.messagebus.Error(32, "foo"));
+            metrics.addReply(reply);
+        }
+
+        timer.ms = 62000;
+
+        {
+            EmptyReply reply = new EmptyReply();
+            reply.setMessage(UpdateDocumentMessage.createEmpty().setTimeReceived(5));
+            reply.addError(new com.yahoo.messagebus.Error(64, "bar"));
+            metrics.addReply(reply);
+        }
+
+        metrics.done();
+
+        String val = output.toString().split("\n")[1];
+
+        String correctPattern = "62000,\\s*3,\\s*2,\\s*\\d+,\\s*\\d+,\\s*\\d+$";
+        assertTrue("Value '" + val + "' does not match pattern '" + correctPattern + "'", val.matches(correctPattern));
+    }
+
+}
diff --git a/vespaclient-java/src/test/java/com/yahoo/vespafeeder/ProgressPrinterTest.java b/vespaclient-java/src/test/java/com/yahoo/vespafeeder/ProgressPrinterTest.java
new file mode 100644
index 00000000000..ae49bb1318d
--- /dev/null
+++ b/vespaclient-java/src/test/java/com/yahoo/vespafeeder/ProgressPrinterTest.java
@@ -0,0 +1,90 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespafeeder;
+
+import com.yahoo.clientmetrics.RouteMetricSet;
+import com.yahoo.concurrent.Timer;
+import com.yahoo.documentapi.messagebus.protocol.DocumentIgnoredReply;
+import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage;
+import com.yahoo.documentapi.messagebus.protocol.UpdateDocumentMessage;
+import com.yahoo.messagebus.EmptyReply;
+import junit.framework.TestCase;
+
+import java.io.ByteArrayOutputStream;
+import java.io.PrintStream;
+
+/**
+ */
+public class ProgressPrinterTest extends TestCase {
+
+    class DummyTimer implements Timer {
+        long ms;
+
+        public long milliTime() { return ms; }
+    }
+
+    public void testSimple() {
+        ByteArrayOutputStream output = new ByteArrayOutputStream();
+        DummyTimer timer = new DummyTimer();
+        timer.ms = 0;
+        ProgressPrinter printer = new ProgressPrinter(timer, new PrintStream(output));
+        RouteMetricSet metrics = new RouteMetricSet("foobar", printer);
+
+        {
+            EmptyReply reply = new EmptyReply();
+            reply.setMessage(PutDocumentMessage.createEmpty().setTimeReceived(1));
+            metrics.addReply(reply);
+        }
+
+        timer.ms = 1200;
+
+        {
+            EmptyReply reply = new EmptyReply();
+            reply.setMessage(PutDocumentMessage.createEmpty().setTimeReceived(2));
+            metrics.addReply(reply);
+        }
+
+        {
+            EmptyReply reply = new EmptyReply();
+            reply.setMessage(UpdateDocumentMessage.createEmpty().setTimeReceived(3));
+            metrics.addReply(reply);
+        }
+
+        timer.ms = 2400;
+
+        {
+            DocumentIgnoredReply reply = new DocumentIgnoredReply();
+            reply.setMessage(PutDocumentMessage.createEmpty().setTimeReceived(0));
+            metrics.addReply(reply);
+        }
+
+        {
+            EmptyReply reply = new EmptyReply();
+            reply.setMessage(UpdateDocumentMessage.createEmpty().setTimeReceived(5));
+            reply.addError(new com.yahoo.messagebus.Error(32, "foo"));
+            metrics.addReply(reply);
+        }
+
+        timer.ms = 62000;
+
+        {
+            EmptyReply reply = new EmptyReply();
+            reply.setMessage(UpdateDocumentMessage.createEmpty().setTimeReceived(6));
+            reply.addError(new com.yahoo.messagebus.Error(64, "bar"));
+            metrics.addReply(reply);
+        }
+
+        String val = output.toString().replaceAll("latency\\(min, max, avg\\): .*", "latency(min, max, avg): 0, 0, 0");
+
+        String correct =
+                "\rSuccessfully sent 2 messages so far" +
+                "\rSuccessfully sent 3 messages so far" +
+                "\n\n" +
+                "Messages sent to vespa (route foobar) :\n" +
+                "---------------------------------------\n" +
+                "PutDocument:\tok: 2 msgs/sec: 0.03 failed: 0 ignored: 1 latency(min, max, avg): 0, 0, 0\n" +
+                "UpdateDocument:\tok: 1 msgs/sec: 0.02 failed: 2 ignored: 0 latency(min, max, avg): 0, 0, 0\n";
+
+        assertEquals(correct, val);
+    }
+
+}
diff --git a/vespaclient-java/src/test/java/com/yahoo/vespafeeder/VespaFeederTestCase.java b/vespaclient-java/src/test/java/com/yahoo/vespafeeder/VespaFeederTestCase.java
new file mode 100644
index 00000000000..42d4b082ff3
--- /dev/null
+++ b/vespaclient-java/src/test/java/com/yahoo/vespafeeder/VespaFeederTestCase.java
@@ -0,0 +1,208 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespafeeder;
+
+import static org.junit.Assert.*;
+
+import java.io.BufferedInputStream;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.util.ArrayList;
+import java.util.Arrays;
+
+import com.yahoo.clientmetrics.RouteMetricSet;
+import com.yahoo.document.DocumentTypeManager;
+import com.yahoo.document.DocumentTypeManagerConfigurer;
+import com.yahoo.document.DocumentUpdate;
+import com.yahoo.documentapi.messagebus.protocol.*;
+import com.yahoo.feedapi.DummySessionFactory;
+import com.yahoo.feedhandler.VespaFeedHandler;
+import com.yahoo.text.Utf8;
+import com.yahoo.vespaclient.config.FeederConfig;
+import com.yahoo.vespafeeder.Arguments.HelpShownException;
+
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+public class VespaFeederTestCase {
+
+    @Rule
+    public ExpectedException exception = ExpectedException.none();
+
+    @Test
+    public void testParseArgs() throws Exception {
+        String argsS="--abortondataerror false --abortonsenderror false --file foo.xml --maxpending 10" +
+                " --maxpendingsize 11 --maxfeedrate 29 --mode benchmark --noretry --retrydelay 12 --route e6 --timeout 13 --trace 4" +
+                " --validate -v bar.xml --priority LOW_1";
+
+        Arguments arguments = new Arguments(argsS.split(" "), DummySessionFactory.createWithAutoReply());
+
+        FeederConfig config = arguments.getFeederConfig();
+        assertEquals(false, config.abortondocumenterror());
+        assertEquals(13.0, config.timeout(), 0.00001);
+        assertEquals(false, config.retryenabled());
+        assertEquals(12.0, config.retrydelay(), 0.0001);
+        assertEquals("e6", config.route());
+        assertEquals(4, config.tracelevel());
+        assertEquals(false, config.abortonsenderror());
+        assertEquals(10, config.maxpendingdocs());
+        assertEquals(11, config.maxpendingbytes());
+        assertEquals(29.0, config.maxfeedrate(), 0.0001);
+        assertTrue(arguments.isVerbose());
+        assertFalse(config.createifnonexistent());
+
+        assertEquals("LOW_1", arguments.getPriority());
+        assertEquals("benchmark", arguments.getMode());
+        assertEquals("foo.xml", arguments.getFiles().get(0));
+        assertEquals("bar.xml", arguments.getFiles().get(1));
+    }
+
+    @Test
+    public void requireThatCreateIfNonExistentArgumentCanBeParsed() throws Exception {
+        String argsS="--create-if-non-existent --file foo.xml";
+        Arguments arguments = new Arguments(argsS.split(" "), DummySessionFactory.createWithAutoReply());
+        assertTrue(arguments.getFeederConfig().createifnonexistent());
+    }
+
+    @Test
+    public void testHelp() throws Exception {
+        String argsS="-h";
+
+        try {
+            new Arguments(argsS.split(" "), null);
+            assertTrue(false);
+        } catch (Arguments.HelpShownException e) {
+
+        }
+    }
+
+    @Test
+    public void requireCorrectInputTypeDetection() throws IOException {
+        {
+            BufferedInputStream b = new BufferedInputStream(
+                    new ByteArrayInputStream(Utf8.toBytes("[]")));
+            InputStreamRequest r = new InputStreamRequest(b);
+            VespaFeeder.setJsonInput(r, b);
+            assertEquals("true", r.getProperty(VespaFeedHandler.JSON_INPUT));
+        }
+        {
+            BufferedInputStream b = new BufferedInputStream(
+                    new ByteArrayInputStream(Utf8.toBytes("<document />")));
+            InputStreamRequest r = new InputStreamRequest(b);
+            VespaFeeder.setJsonInput(r, b);
+            assertEquals("false", r.getProperty(VespaFeedHandler.JSON_INPUT));
+        }
+    }
+
+    public void assertRenderErrorOutput(String expected, String[] errors) {
+        ArrayList<String> l = new ArrayList<String>();
+        l.addAll(Arrays.asList(errors));
+        assertEquals(expected, VespaFeeder.renderErrors(l).getMessage());
+    }
+
+    @Test
+    public void testRenderErrors() {
+        {
+            String[] errors = { "foo" };
+            assertRenderErrorOutput("Errors:\n" +
+                                    "-------\n" +
+                                    "    foo\n", errors);
+        }
+
+        {
+            String[] errors = { "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11"};
+            assertRenderErrorOutput("First 10 errors (of 11):\n" +
+                    "------------------------\n" +
+                    "    1\n    2\n    3\n    4\n    5\n    6\n    7\n    8\n    9\n    10\n", errors);
+        }
+    }
+
+    public RouteMetricSet.ProgressCallback getProgressPrinter(String args) throws Exception {
+        Arguments arguments = new Arguments(args.split(" "), DummySessionFactory.createWithAutoReply());
+        return new VespaFeeder(arguments, null).createProgressCallback(System.out);
+    }
+
+    @Test
+    public void testCreateProgressPrinter() throws Exception {
+        assert(getProgressPrinter("--mode benchmark") instanceof BenchmarkProgressPrinter);
+        assert(getProgressPrinter("") instanceof ProgressPrinter);
+    }
+
+    private static class FeedFixture {
+        DummySessionFactory sessionFactory = DummySessionFactory.createWithAutoReply();
+        ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
+        PrintStream printStream = new PrintStream(outputStream);
+        DocumentTypeManager typeManager = new DocumentTypeManager();
+        FeedFixture() {
+            DocumentTypeManagerConfigurer.configure(typeManager, "file:src/test/files/documentmanager.cfg");
+        }
+    }
+
+    @Test
+    public void feedFile() throws Exception {
+        FeedFixture f = new FeedFixture();
+        Arguments arguments = new Arguments("--file src/test/files/myfeed.xml --priority LOW_1".split(" "), f.sessionFactory);
+        new VespaFeeder(arguments, f.typeManager).parseFiles(System.in, f.printStream);
+
+        assertEquals(3, f.sessionFactory.messages.size());
+        assertEquals(DocumentProtocol.Priority.LOW_1, ((PutDocumentMessage)f.sessionFactory.messages.get(0)).getPriority());
+        assertEquals("doc:test:foo", ((PutDocumentMessage) f.sessionFactory.messages.get(0)).getDocumentPut().getDocument().getId().toString());
+        DocumentUpdate update = ((UpdateDocumentMessage) f.sessionFactory.messages.get(1)).getDocumentUpdate();
+        assertEquals("doc:test:foo", update.getId().toString());
+        assertFalse(update.getCreateIfNonExistent());
+        assertEquals("doc:test:foo", ((RemoveDocumentMessage) f.sessionFactory.messages.get(2)).getDocumentId().toString());
+
+        assertTrue(f.outputStream.toString().contains("Messages sent to vespa"));
+    }
+
+    @Test
+    public void feedJson() throws Exception {
+        FeedFixture feedFixture = feed("src/test/files/myfeed.json", true);
+
+        assertJsonFeedState(feedFixture);
+    }
+
+    protected void assertJsonFeedState(FeedFixture feedFixture) {
+        assertEquals(3, feedFixture.sessionFactory.messages.size());
+        assertEquals(DocumentProtocol.Priority.LOW_1, ((PutDocumentMessage)feedFixture.sessionFactory.messages.get(0)).getPriority());
+        assertEquals("id:test:news::foo", ((PutDocumentMessage) feedFixture.sessionFactory.messages.get(0)).getDocumentPut().getDocument().getId().toString());
+        DocumentUpdate update = ((UpdateDocumentMessage) feedFixture.sessionFactory.messages.get(1)).getDocumentUpdate();
+        assertEquals("id:test:news::foo", update.getId().toString());
+        assertFalse(update.getCreateIfNonExistent());
+        assertEquals("id:test:news::foo", ((RemoveDocumentMessage) feedFixture.sessionFactory.messages.get(2)).getDocumentId().toString());
+
+        assertTrue(feedFixture.outputStream.toString().contains("Messages sent to vespa"));
+    }
+
+    @Test
+    public void requireThatCreateIfNonExistentArgumentIsUsed() throws Exception {
+        FeedFixture f = new FeedFixture();
+        Arguments arguments = new Arguments("--file src/test/files/myfeed.xml --create-if-non-existent".split(" "), f.sessionFactory);
+        new VespaFeeder(arguments, f.typeManager).parseFiles(System.in, f.printStream);
+
+        assertEquals(3, f.sessionFactory.messages.size());
+        DocumentUpdate update = ((UpdateDocumentMessage) f.sessionFactory.messages.get(1)).getDocumentUpdate();
+        assertTrue(update.getCreateIfNonExistent());
+    }
+
+    @Test
+    public void feedMalformedJson() throws Exception {
+        exception.expect(VespaFeeder.FeedErrorException.class);
+        exception.expectMessage("JsonParseException");
+        feed("src/test/files/malformedfeed.json", false);
+    }
+
+    protected FeedFixture feed(String feed, boolean abortOnDataError) throws HelpShownException,
+            FileNotFoundException, Exception {
+        String abortOnDataErrorArgument = abortOnDataError ? "" : " --abortondataerror no";
+        FeedFixture feedFixture = new FeedFixture();
+        Arguments arguments = new Arguments(("--file "
+                + feed
+                + " --priority LOW_1" + abortOnDataErrorArgument).split(" "), feedFixture.sessionFactory);
+        new VespaFeeder(arguments, feedFixture.typeManager).parseFiles(System.in, feedFixture.printStream);
+        return feedFixture;
+    }
+}
diff --git a/vespaclient-java/src/test/java/com/yahoo/vespaget/CommandLineOptionsTest.java b/vespaclient-java/src/test/java/com/yahoo/vespaget/CommandLineOptionsTest.java
new file mode 100644
index 00000000000..3e707b04256
--- /dev/null
+++ b/vespaclient-java/src/test/java/com/yahoo/vespaget/CommandLineOptionsTest.java
@@ -0,0 +1,195 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespaget;
+
+import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+import java.io.*;
+import java.util.Iterator;
+
+import static org.junit.Assert.*;
+
+/**
+ * Test class for {@link CommandLineOptions}
+ *
+ * @author bjorncs
+ * @since 5.26
+ */
+public class CommandLineOptionsTest {
+
+    private final InputStream emptyStream = new InputStream() {
+
+        @Override
+        public int read() throws IOException {
+            return -1;
+        }
+    };
+
+    @Rule
+    public final ExpectedException exception = ExpectedException.none();
+
+    private ClientParameters getParsedOptions(InputStream in, String... args) {
+        CommandLineOptions options = new CommandLineOptions(in);
+        return options.parseCommandLineArguments(args);
+    }
+
+    private ClientParameters getParsedOptions(String... args) {
+        return getParsedOptions(emptyStream, args);
+    }
+
+    @Test
+    public void testDefaultOptions() {
+        ClientParameters params = getParsedOptions();
+        assertFalse(params.help);
+        assertFalse(params.documentIds.hasNext());
+        assertFalse(params.printIdsOnly);
+        assertEquals("[all]", params.fieldSet);
+        assertEquals("default", params.route);
+        assertTrue(params.cluster.isEmpty());
+        assertEquals("client", params.configId);
+        assertFalse(params.showDocSize);
+        assertEquals(0, params.timeout, 0);
+        assertFalse(params.noRetry);
+        assertEquals(0, params.traceLevel);
+        assertEquals(DocumentProtocol.Priority.NORMAL_2, params.priority);
+        assertTrue(params.loadTypeName.isEmpty());
+    }
+
+    @Test
+    public void testValidOptions() {
+        ClientParameters params = getParsedOptions(
+                "--fieldset", "[fieldset]",
+                "--route", "dummyroute",
+                "--configid", "dummyconfig",
+                "--showdocsize",
+                "--timeout", "0.25",
+                "--noretry",
+                "--trace", "1",
+                "--priority", Integer.toString(DocumentProtocol.Priority.HIGH_3.getValue()),
+                "--loadtype", "dummyloadtype",
+                "id:1", "id:2"
+        );
+
+        assertEquals("[fieldset]", params.fieldSet);
+        assertEquals("dummyroute", params.route);
+        assertEquals("dummyconfig", params.configId);
+        assertTrue(params.showDocSize);
+        assertEquals(0.25, params.timeout, 0.0001);
+        assertTrue(params.noRetry);
+        assertEquals(1, params.traceLevel);
+        assertEquals(DocumentProtocol.Priority.HIGH_3, params.priority);
+        assertEquals("dummyloadtype", params.loadTypeName);
+
+        Iterator<String> documentsIds = params.documentIds;
+        assertEquals("id:1", documentsIds.next());
+        assertEquals("id:2", documentsIds.next());
+        assertFalse(documentsIds.hasNext());
+    }
+
+    @Test
+    public void testInvalidCombination1() {
+        exception.expect(IllegalArgumentException.class);
+        exception.expectMessage("Print ids and headers only options are mutually exclusive.");
+        getParsedOptions("--headersonly", "--printids");
+    }
+
+    @Test
+    public void testInvalidCombination2() {
+        exception.expect(IllegalArgumentException.class);
+        exception.expectMessage("Field set option can not be used in combination with print ids or headers only options.");
+        getParsedOptions("--headersonly", "--fieldset", "[header]");
+    }
+
+    @Test
+    public void testInvalidCombination3() {
+        exception.expect(IllegalArgumentException.class);
+        exception.expectMessage("Field set option can not be used in combination with print ids or headers only options.");
+        getParsedOptions("--printids", "--fieldset", "[header]");
+    }
+
+    @Test
+    public void testInvalidCombination4() {
+        exception.expect(IllegalArgumentException.class);
+        exception.expectMessage("Cluster and route options are mutually exclusive.");
+        getParsedOptions("--route", "dummyroute", "--cluster", "dummycluster");
+    }
+
+    @Test
+    public void testInvalidPriority() {
+        exception.expect(IllegalArgumentException.class);
+        exception.expectMessage("Invalid priority: 16");
+        getParsedOptions("--priority", "16");
+    }
+
+    @Test
+    public void testInvalidTraceLevel1() {
+        exception.expect(IllegalArgumentException.class);
+        exception.expectMessage("Invalid tracelevel: -1");
+        getParsedOptions("--trace", "-1");
+    }
+
+    @Test
+    public void testInvalidTraceLevel2() {
+        exception.expect(IllegalArgumentException.class);
+        exception.expectMessage("Invalid tracelevel: 10");
+        getParsedOptions("--trace", "10");
+    }
+
+    @Test
+    public void testPrintids() {
+        ClientParameters params = getParsedOptions("--printids");
+        assertEquals("[id]", params.fieldSet);
+    }
+
+    @Test
+    public void testHeadersOnly() {
+        ClientParameters params = getParsedOptions("--headersonly");
+        assertEquals("[header]", params.fieldSet);
+    }
+
+    @Test
+    public void testCluster() {
+        ClientParameters params = getParsedOptions("--cluster", "dummycluster");
+        assertEquals("dummycluster", params.cluster);
+        assertTrue(params.route.isEmpty());
+    }
+
+    @Test
+    public void testHelp() {
+        ClientParameters params = getParsedOptions("--help");
+        assertTrue(params.help);
+    }
+
+    @Test
+    public void testDocumentIdsFromInputStream() throws UnsupportedEncodingException {
+        InputStream in = new ByteArrayInputStream("id:1 id:2 id:3".getBytes("UTF-8"));
+        ClientParameters params = getParsedOptions(in, "");
+
+        Iterator<String> documentsIds = params.documentIds;
+        assertEquals("id:1", documentsIds.next());
+        assertEquals("id:2", documentsIds.next());
+        assertEquals("id:3", documentsIds.next());
+        assertFalse(documentsIds.hasNext());
+    }
+
+    @Test
+    public void testPrintHelp() {
+        ByteArrayOutputStream outContent = new ByteArrayOutputStream();
+        PrintStream oldOut = System.out;
+        System.setOut(new PrintStream(outContent));
+        try {
+            CommandLineOptions options = new CommandLineOptions(emptyStream);
+            options.printHelp();
+
+            String output = outContent.toString();
+            assertTrue(output.contains("vespaget <options> [documentid...]"));
+            assertTrue(output.contains("Fetch a document from a Vespa Content cluster."));
+        } finally {
+            System.setOut(oldOut);
+            outContent.reset();
+
+        }
+    }
+}
diff --git a/vespaclient-java/src/test/java/com/yahoo/vespaget/DocumentRetrieverTest.java b/vespaclient-java/src/test/java/com/yahoo/vespaget/DocumentRetrieverTest.java
new file mode 100644
index 00000000000..c3d3fcc71e9
--- /dev/null
+++ b/vespaclient-java/src/test/java/com/yahoo/vespaget/DocumentRetrieverTest.java
@@ -0,0 +1,376 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespaget;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonParseException;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.yahoo.document.DataType;
+import com.yahoo.document.Document;
+import com.yahoo.document.DocumentId;
+import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess;
+import com.yahoo.documentapi.messagebus.MessageBusParams;
+import com.yahoo.documentapi.messagebus.MessageBusSyncSession;
+import com.yahoo.documentapi.messagebus.loadtypes.LoadType;
+import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet;
+import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol;
+import com.yahoo.documentapi.messagebus.protocol.GetDocumentMessage;
+import com.yahoo.documentapi.messagebus.protocol.GetDocumentReply;
+import com.yahoo.messagebus.Error;
+import com.yahoo.messagebus.Reply;
+import com.yahoo.vespaclient.ClusterDef;
+import com.yahoo.vespaclient.ClusterList;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.mockito.ArgumentMatcher;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.argThat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+/**
+ * Test class for {@link DocumentRetriever}
+ *
+ * @author bjorncs
+ */
+public class DocumentRetrieverTest {
+
+    public static final String DOC_ID_1 = "id:storage_test:document::1";
+    public static final String DOC_ID_2 = "id:storage_test:document::2";
+    public static final String DOC_ID_3 = "id:storage_test:document::3";
+
+    private final ByteArrayOutputStream outContent = new ByteArrayOutputStream();
+    private final ByteArrayOutputStream errContent = new ByteArrayOutputStream();
+
+    private DocumentAccessFactory mockedFactory;
+    private MessageBusDocumentAccess mockedDocumentAccess;
+    private MessageBusSyncSession mockedSession;
+    private PrintStream oldOut;
+    private PrintStream oldErr;
+
+    @Rule
+    public final ExpectedException exception = ExpectedException.none();
+
+    @Before
+    public void setUpStreams() {
+        oldOut = System.out;
+        oldErr = System.err;
+        System.setOut(new PrintStream(outContent));
+        System.setErr(new PrintStream(errContent));
+    }
+
+    @Before
+    public void prepareMessageBusMocks() {
+        this.mockedFactory = mock(DocumentAccessFactory.class);
+        this.mockedDocumentAccess = mock(MessageBusDocumentAccess.class);
+        this.mockedSession = mock(MessageBusSyncSession.class);
+        when(mockedFactory.createDocumentAccess(any())).thenReturn(mockedDocumentAccess);
+        when(mockedDocumentAccess.createSyncSession(any())).thenReturn(mockedSession);
+    }
+
+    @After
+    public void cleanUpStreams() {
+        System.setOut(oldOut);
+        System.setErr(oldErr);
+        outContent.reset();
+        errContent.reset();
+    }
+
+    private static ClientParameters.Builder createParameters() {
+        return new ClientParameters.Builder()
+                .setPriority(DocumentProtocol.Priority.NORMAL_2)
+                .setCluster("")
+                .setRoute("default")
+                .setConfigId("client")
+                .setFieldSet("[all]")
+                .setPrintIdsOnly(false)
+                .setHelp(false)
+                .setShowDocSize(false)
+                .setLoadTypeName("")
+                .setNoRetry(false)
+                .setTraceLevel(0)
+                .setTimeout(0)
+                .setDocumentIds(Collections.emptyIterator());
+    }
+
+    private static Iterator<String> asIterator(String... docIds) {
+        return Arrays.asList(docIds).iterator();
+    }
+
+    private static Reply createDocumentReply(String docId) {
+        return new GetDocumentReply(new Document(DataType.DOCUMENT, new DocumentId(docId)));
+    }
+
+    private void assertContainsDocument(String documentId) {
+        assertTrue(outContent.toString().contains(String.format(
+                "<document documenttype=\"document\" documentid=\"%s\"/>", documentId)));
+    }
+
+    private DocumentRetriever createDocumentRetriever(ClientParameters params) {
+        return createDocumentRetriever(params, new ClusterList());
+    }
+
+    private DocumentRetriever createDocumentRetriever(ClientParameters params, ClusterList clusterList) {
+        return new DocumentRetriever(
+                clusterList,
+                mockedFactory,
+                new LoadTypeSet(),
+                params);
+    }
+
+    @Test
+    public void testSendSingleMessage() throws DocumentRetrieverException {
+        ClientParameters params = createParameters()
+                .setDocumentIds(asIterator(DOC_ID_1))
+                .setPriority(DocumentProtocol.Priority.HIGH_1)
+                .setNoRetry(true)
+                .setLoadTypeName("loadtype")
+                .build();
+
+        when(mockedSession.syncSend(any())).thenReturn(createDocumentReply(DOC_ID_1));
+
+        LoadTypeSet loadTypeSet = new LoadTypeSet();
+        loadTypeSet.addLoadType(1, "loadtype", DocumentProtocol.Priority.HIGH_1);
+        DocumentRetriever documentRetriever = new DocumentRetriever(
+                new ClusterList(),
+                mockedFactory,
+                loadTypeSet,
+                params);
+        documentRetriever.retrieveDocuments();
+
+        verify(mockedSession, times(1)).syncSend(argThat(new ArgumentMatcher<GetDocumentMessage>() {
+            @Override
+            public boolean matches(Object o) {
+                GetDocumentMessage msg = (GetDocumentMessage) o;
+                return msg.getPriority().equals(DocumentProtocol.Priority.HIGH_1) &&
+                        !msg.getRetryEnabled() &&
+                        msg.getLoadType().equals(new LoadType(1, "loadtype", DocumentProtocol.Priority.HIGH_1));
+            }
+        }));
+        assertContainsDocument(DOC_ID_1);
+    }
+
+    @Test
+    public void testMultipleMessages() throws DocumentRetrieverException {
+        ClientParameters params = createParameters()
+                .setDocumentIds(asIterator(DOC_ID_1, DOC_ID_2, DOC_ID_3))
+                .build();
+
+        when(mockedSession.syncSend(any())).thenReturn(
+                createDocumentReply(DOC_ID_1),
+                createDocumentReply(DOC_ID_2),
+                createDocumentReply(DOC_ID_3));
+
+        DocumentRetriever documentRetriever = createDocumentRetriever(params);
+        documentRetriever.retrieveDocuments();
+
+        verify(mockedSession, times(3)).syncSend(any());
+        assertContainsDocument(DOC_ID_1);
+        assertContainsDocument(DOC_ID_2);
+        assertContainsDocument(DOC_ID_3);
+    }
+
+    @Test
+    public void testJsonOutput() throws DocumentRetrieverException, JsonParseException, IOException {
+        ClientParameters params = createParameters()
+                .setDocumentIds(asIterator(DOC_ID_1, DOC_ID_2, DOC_ID_3))
+                .setJsonOutput(true)
+                .build();
+
+        when(mockedSession.syncSend(any())).thenReturn(
+                createDocumentReply(DOC_ID_1),
+                createDocumentReply(DOC_ID_2),
+                createDocumentReply(DOC_ID_3));
+
+        DocumentRetriever documentRetriever = createDocumentRetriever(params);
+        documentRetriever.retrieveDocuments();
+
+        verify(mockedSession, times(3)).syncSend(any());
+        ObjectMapper m = new ObjectMapper();
+        @SuppressWarnings("unchecked")
+        List<Map<String, Object>> feed = m.readValue(outContent.toByteArray(), List.class);
+        assertEquals(DOC_ID_1, feed.get(0).get("id"));
+        assertEquals(DOC_ID_2, feed.get(1).get("id"));
+        assertEquals(DOC_ID_3, feed.get(2).get("id"));
+    }
+
+    @Test
+    public void testShutdownHook() throws DocumentRetrieverException {
+        ClientParameters params = createParameters()
+                .setDocumentIds(asIterator(DOC_ID_1))
+                .build();
+
+        when(mockedSession.syncSend(any())).thenReturn(createDocumentReply(DOC_ID_1));
+
+        DocumentRetriever documentRetriever = createDocumentRetriever(params);
+        documentRetriever.retrieveDocuments();
+        documentRetriever.shutdown();
+
+        verify(mockedSession, times(1)).destroy();
+        verify(mockedDocumentAccess, times(1)).shutdown();
+    }
+
+    @Test
+    public void testInvalidLoadType() throws DocumentRetrieverException {
+        exception.expect(DocumentRetrieverException.class);
+        exception.expectMessage("Loadtype with name 'undefinedloadtype' does not exist.\n");
+
+        ClientParameters params = createParameters()
+                .setLoadTypeName("undefinedloadtype")
+                .build();
+
+        DocumentRetriever documentRetriever = createDocumentRetriever(params);
+        documentRetriever.retrieveDocuments();
+    }
+
+    @Test
+    public void testClusterLookup() throws DocumentRetrieverException {
+        final String cluster = "storage", configId = "content/cluster.foo/storage",
+                expectedRoute = "[Storage:cluster=storage;clusterconfigid=content/cluster.foo/storage]";
+
+        ClientParameters params = createParameters()
+                .setCluster(cluster)
+                .build();
+
+        ClusterList clusterList = new ClusterList(Collections.singletonList(new ClusterDef(cluster, configId)));
+
+        DocumentRetriever documentRetriever = createDocumentRetriever(params, clusterList);
+        documentRetriever.retrieveDocuments();
+
+        verify(mockedFactory).createDocumentAccess(argThat(new ArgumentMatcher<MessageBusParams>() {
+            @Override
+            public boolean matches(Object o) {
+                return ((MessageBusParams) o).getRoute().equals(expectedRoute);
+            }
+        }));
+    }
+
+    @Test
+    public void testInvalidClusterName() throws DocumentRetrieverException {
+        exception.expect(DocumentRetrieverException.class);
+        exception.expectMessage("The Vespa cluster contains the content clusters storage, not invalidclustername. Please select a valid vespa cluster.");
+
+        ClientParameters params = createParameters()
+                .setCluster("invalidclustername")
+                .build();
+
+        ClusterList clusterList = new ClusterList(Collections.singletonList(new ClusterDef("storage", "content/cluster.foo/storage")));
+
+        DocumentRetriever documentRetriever = createDocumentRetriever(params, clusterList);
+        documentRetriever.retrieveDocuments();
+    }
+
+    @Test
+    public void testEmtpyClusterList() throws DocumentRetrieverException {
+        exception.expect(DocumentRetrieverException.class);
+        exception.expectMessage("The Vespa cluster does not have any content clusters declared.");
+
+        ClientParameters params = createParameters()
+                .setCluster("invalidclustername")
+                .build();
+
+        DocumentRetriever documentRetriever = createDocumentRetriever(params);
+        documentRetriever.retrieveDocuments();
+    }
+
+    @Test
+    public void testHandlingErrorFromMessageBus() throws DocumentRetrieverException {
+        ClientParameters params = createParameters()
+                .setDocumentIds(asIterator(DOC_ID_1))
+                .build();
+
+        Reply r = new GetDocumentReply(null);
+        r.addError(new Error(0, "Error message"));
+        when(mockedSession.syncSend(any())).thenReturn(r);
+
+        DocumentRetriever documentRetriever = createDocumentRetriever(params);
+        documentRetriever.retrieveDocuments();
+
+        assertTrue(errContent.toString().contains("Request failed"));
+    }
+
+    @Test
+    public void testShowDocSize() throws DocumentRetrieverException {
+        ClientParameters params = createParameters()
+                .setDocumentIds(asIterator(DOC_ID_1))
+                .setShowDocSize(true)
+                .build();
+
+        Document document = new Document(DataType.DOCUMENT, new DocumentId(DOC_ID_1));
+        when(mockedSession.syncSend(any())).thenReturn(new GetDocumentReply(document));
+
+        DocumentRetriever documentRetriever = createDocumentRetriever(params);
+        documentRetriever.retrieveDocuments();
+
+        assertTrue(outContent.toString().contains(String.format("Document size: %d bytes", document.getSerializedSize())));
+    }
+
+    @Test
+    public void testPrintIdOnly() throws DocumentRetrieverException {
+        ClientParameters params = createParameters()
+                .setDocumentIds(asIterator(DOC_ID_1))
+                .setPrintIdsOnly(true)
+                .build();
+
+        when(mockedSession.syncSend(any())).thenReturn(createDocumentReply(DOC_ID_1));
+
+        DocumentRetriever documentRetriever = createDocumentRetriever(params);
+        documentRetriever.retrieveDocuments();
+
+        assertEquals(DOC_ID_1 + "\n", outContent.toString());
+    }
+
+    @Test
+    public void testDocumentNotFound() throws DocumentRetrieverException {
+        ClientParameters params = createParameters()
+                .setDocumentIds(asIterator(DOC_ID_1))
+                .setPrintIdsOnly(true)
+                .build();
+
+        when(mockedSession.syncSend(any())).thenReturn(new GetDocumentReply(null));
+
+        DocumentRetriever documentRetriever = createDocumentRetriever(params);
+        documentRetriever.retrieveDocuments();
+
+        verify(mockedSession, times(1)).syncSend(any());
+        assertEquals(outContent.toString(), "Document not found.\n");
+    }
+
+    @Test
+    public void testTrace() throws DocumentRetrieverException {
+        final int traceLevel = 9;
+        ClientParameters params = createParameters()
+                .setDocumentIds(asIterator(DOC_ID_1))
+                .setTraceLevel(traceLevel)
+                .build();
+
+        GetDocumentReply reply = new GetDocumentReply(new Document(DataType.DOCUMENT, new DocumentId(DOC_ID_1)));
+        reply.getTrace().getRoot().addChild("childnode");
+        when(mockedSession.syncSend(any())).thenReturn(reply);
+
+        DocumentRetriever documentRetriever = createDocumentRetriever(params);
+        documentRetriever.retrieveDocuments();
+
+        verify(mockedSession, times(1)).setTraceLevel(traceLevel);
+        assertTrue(outContent.toString().contains("<trace>"));
+    }
+
+}
diff --git a/vespaclient-java/src/test/java/com/yahoo/vespastat/BucketStatsPrinterTest.java b/vespaclient-java/src/test/java/com/yahoo/vespastat/BucketStatsPrinterTest.java
new file mode 100644
index 00000000000..2df8b2c4751
--- /dev/null
+++ b/vespaclient-java/src/test/java/com/yahoo/vespastat/BucketStatsPrinterTest.java
@@ -0,0 +1,87 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespastat;
+
+import com.yahoo.document.BucketId;
+import com.yahoo.documentapi.messagebus.protocol.GetBucketListReply;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.io.PrintStream;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class BucketStatsPrinterTest {
+
+    private BucketStatsRetriever retriever;
+    private final ByteArrayOutputStream out = new ByteArrayOutputStream();
+
+    @Before
+    public void mockBucketStatsRetriever() throws BucketStatsException {
+        retriever = mock(BucketStatsRetriever.class);
+        when(retriever.getBucketIdForType(any(), any())).thenReturn(new BucketId(0x42));
+        when(retriever.retrieveBucketList(any())).thenReturn(Collections.emptyList());
+        when(retriever.retrieveBucketStats(any(), any(), any())).thenReturn("");
+    }
+
+    @After
+    public void resetOutputMock() {
+        out.reset();
+    }
+
+    private String getOutputString() {
+        String content = out.toString();
+        out.reset();
+        return content;
+    }
+
+    private String retreiveAndPrintBucketStats(ClientParameters.SelectionType type, String id, boolean dumpData) throws BucketStatsException {
+        BucketStatsPrinter printer = new BucketStatsPrinter(retriever, new PrintStream(out));
+        printer.retrieveAndPrintBucketStats(type, id, dumpData);
+        return getOutputString();
+    }
+
+    @Test
+    public void testShouldPrintBucketIdForUserAndGroup() throws BucketStatsException {
+        String output = retreiveAndPrintBucketStats(ClientParameters.SelectionType.USER, "1234", false);
+        assertTrue(output.contains("Generated 32-bit bucket id"));
+
+        output = retreiveAndPrintBucketStats(ClientParameters.SelectionType.GROUP, "mygroup", false);
+        assertTrue(output.contains("Generated 32-bit bucket id"));
+    }
+
+    @Test
+    public void testShouldPrintWarningIfBucketListEmpty() throws BucketStatsException {
+        String output = retreiveAndPrintBucketStats(ClientParameters.SelectionType.USER, "1234", false);
+        assertTrue(output.contains("No actual files were stored for this bucket"));
+    }
+
+    @Test
+    public void testShouldPrintBucketList() throws BucketStatsException {
+        List<GetBucketListReply.BucketInfo> bucketList = new ArrayList<>();
+        String dummyInfoString = "dummyinformation";
+        bucketList.add(new GetBucketListReply.BucketInfo(new BucketId(0), dummyInfoString));
+        when(retriever.retrieveBucketList(any())).thenReturn(bucketList);
+
+        String output = retreiveAndPrintBucketStats(ClientParameters.SelectionType.USER, "1234", false);
+        assertTrue(output.contains(dummyInfoString));
+    }
+
+    @Test
+    public void testShouldPrintBucketStats() throws BucketStatsException {
+        String dummyBucketStats = "dummystats";
+        GetBucketListReply.BucketInfo bucketInfo = new GetBucketListReply.BucketInfo(new BucketId(0), "dummy");
+        when(retriever.retrieveBucketList(any())).thenReturn(Collections.singletonList(bucketInfo));
+        when(retriever.retrieveBucketStats(any(), any(), any())).thenReturn(dummyBucketStats);
+
+        String output = retreiveAndPrintBucketStats(ClientParameters.SelectionType.USER, "1234", true);
+        assertTrue(output.contains(dummyBucketStats));
+    }
+}
diff --git a/vespaclient-java/src/test/java/com/yahoo/vespastat/BucketStatsRetrieverTest.java b/vespaclient-java/src/test/java/com/yahoo/vespastat/BucketStatsRetrieverTest.java
new file mode 100644
index 00000000000..38a79aa7b5d
--- /dev/null
+++ b/vespaclient-java/src/test/java/com/yahoo/vespastat/BucketStatsRetrieverTest.java
@@ -0,0 +1,141 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespastat;
+
+import com.yahoo.document.BucketId;
+import com.yahoo.document.BucketIdFactory;
+import com.yahoo.document.DocumentId;
+import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess;
+import com.yahoo.documentapi.messagebus.MessageBusSyncSession;
+import com.yahoo.documentapi.messagebus.protocol.GetBucketListReply;
+import com.yahoo.documentapi.messagebus.protocol.StatBucketReply;
+import com.yahoo.messagebus.Error;
+import com.yahoo.messagebus.Message;
+import com.yahoo.messagebus.routing.Route;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.ArgumentMatcher;
+
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.argThat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+public class BucketStatsRetrieverTest {
+    private final BucketIdFactory bucketIdFactory = new BucketIdFactory();
+
+    private DocumentAccessFactory mockedFactory;
+    private MessageBusDocumentAccess mockedDocumentAccess;
+    private MessageBusSyncSession mockedSession;
+
+
+    @Before
+    public void prepareMessageBusMocks() {
+        this.mockedFactory = mock(DocumentAccessFactory.class);
+        this.mockedDocumentAccess = mock(MessageBusDocumentAccess.class);
+        this.mockedSession = mock(MessageBusSyncSession.class);
+        when(mockedFactory.createDocumentAccess()).thenReturn(mockedDocumentAccess);
+        when(mockedDocumentAccess.createSyncSession(any())).thenReturn(mockedSession);
+    }
+
+    @Test
+    public void testGetBucketId() throws BucketStatsException {
+        BucketStatsRetriever retriever = createRetriever();
+
+        assertEquals("BucketId(0x80000000000004d2)",
+                retriever.getBucketIdForType(ClientParameters.SelectionType.USER, "1234").toString());
+        assertEquals("BucketId(0x800000003a7455d7)",
+                retriever.getBucketIdForType(ClientParameters.SelectionType.GROUP, "mygroup").toString());
+        assertEquals("BucketId(0x800000003a7455d7)",
+                retriever.getBucketIdForType(ClientParameters.SelectionType.BUCKET, "0x800000003a7455d7").toString());
+        assertEquals("BucketId(0xeb018ac5e5732db3)",
+                retriever.getBucketIdForType(ClientParameters.SelectionType.DOCUMENT, "id:ns:type::another").toString());
+        assertEquals("BucketId(0xeadd5fe811a2012c)",
+                retriever.getBucketIdForType(ClientParameters.SelectionType.GID, "0x2c01a21163cb7d0ce85fddd6").toString());
+    }
+
+    @Test
+    public void testRetrieveBucketList() throws BucketStatsException {
+        String bucketInfo = "I like turtles!";
+        BucketId bucketId = bucketIdFactory.getBucketId(new DocumentId("id:ns:type::another"));
+
+        GetBucketListReply reply = new GetBucketListReply();
+        reply.getBuckets().add(new GetBucketListReply.BucketInfo(bucketId, bucketInfo));
+        when(mockedSession.syncSend(any())).thenReturn(reply);
+
+        List<GetBucketListReply.BucketInfo> bucketList = createRetriever().retrieveBucketList(bucketId);
+
+        verify(mockedSession, times(1)).syncSend(any());
+        assertEquals(1, bucketList.size());
+        assertEquals(bucketInfo, bucketList.get(0).getBucketInformation());
+    }
+
+    @Test
+    public void testRetrieveBucketStats() throws BucketStatsException {
+        String docId = "id:ns:type::another";
+        String bucketInfo = "I like turtles!";
+        BucketId bucketId = bucketIdFactory.getBucketId(new DocumentId(docId));
+
+        StatBucketReply reply = new StatBucketReply();
+        reply.setResults(bucketInfo);
+        when(mockedSession.syncSend(any())).thenReturn(reply);
+        String result = createRetriever().retrieveBucketStats(ClientParameters.SelectionType.DOCUMENT, docId, bucketId);
+
+        verify(mockedSession, times(1)).syncSend(any());
+        assertEquals(bucketInfo, result);
+    }
+
+    @Test
+    public void testShutdownHook() {
+        class MockShutdownRegistrar implements BucketStatsRetriever.ShutdownHookRegistrar {
+            public Runnable shutdownRunnable;
+            @Override
+            public void registerShutdownHook(Runnable runnable) {
+                shutdownRunnable = runnable;
+            }
+        }
+        MockShutdownRegistrar registrar = new MockShutdownRegistrar();
+        new BucketStatsRetriever(mockedFactory, "default", registrar);
+        registrar.shutdownRunnable.run();
+
+        verify(mockedSession, times(1)).destroy();
+        verify(mockedDocumentAccess, times(1)).shutdown();
+    }
+
+    @Test(expected = BucketStatsException.class)
+    public void testShouldFailOnReplyError() throws BucketStatsException {
+        GetBucketListReply reply = new GetBucketListReply();
+        reply.addError(new Error(0, "errormsg"));
+        when(mockedSession.syncSend(any())).thenReturn(reply);
+
+        createRetriever().retrieveBucketList(new BucketId(1));
+    }
+
+    @Test
+    public void testRoute() throws BucketStatsException {
+        String route = "default";
+        BucketId bucketId = bucketIdFactory.getBucketId(new DocumentId("id:ns:type::another"));
+        GetBucketListReply reply = new GetBucketListReply();
+        reply.getBuckets().add(new GetBucketListReply.BucketInfo(bucketId, "I like turtles!"));
+        when(mockedSession.syncSend(any())).thenReturn(reply);
+
+        BucketStatsRetriever retriever = new BucketStatsRetriever(mockedFactory, route, t -> {});
+        retriever.retrieveBucketList(new BucketId(0));
+
+        verify(mockedSession).syncSend(argThat(new ArgumentMatcher<Message>() {
+            @Override
+            public boolean matches(Object o) {
+                return ((Message) o).getRoute().equals(Route.parse(route));
+            }
+        }));
+    }
+
+    private BucketStatsRetriever createRetriever() {
+        return new BucketStatsRetriever(mockedFactory, "default", t -> {});
+    }
+
+}
diff --git a/vespaclient-java/src/test/java/com/yahoo/vespastat/CommandLineOptionsTest.java b/vespaclient-java/src/test/java/com/yahoo/vespastat/CommandLineOptionsTest.java
new file mode 100644
index 00000000000..e90c47e3150
--- /dev/null
+++ b/vespaclient-java/src/test/java/com/yahoo/vespastat/CommandLineOptionsTest.java
@@ -0,0 +1,78 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespastat;
+
+import org.junit.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.io.PrintStream;
+
+import static org.junit.Assert.*;
+
+public class CommandLineOptionsTest {
+
+    private ClientParameters getParsedOptions(String... args) {
+        CommandLineOptions parser = new CommandLineOptions();
+        return parser.parseCommandLineArguments(args);
+    }
+
+    @Test
+    public void testHelp() {
+        assertTrue(getParsedOptions("--help").help);
+    }
+
+    @Test
+    public void testMultipleOptions() {
+        ClientParameters params = getParsedOptions("--dump", "--route", "dummyroute", "--user", "userid");
+        assertTrue(params.dumpData);
+        assertEquals("dummyroute", params.route);
+        assertEquals(ClientParameters.SelectionType.USER, params.selectionType);
+        assertEquals("userid", params.id);
+    }
+
+    @Test
+    public void testSelectionTypes() {
+        assertEquals(ClientParameters.SelectionType.USER, getParsedOptions("--user", "id").selectionType);
+        assertEquals(ClientParameters.SelectionType.DOCUMENT, getParsedOptions("--document", "id").selectionType);
+        assertEquals(ClientParameters.SelectionType.BUCKET, getParsedOptions("--bucket", "id").selectionType);
+        assertEquals(ClientParameters.SelectionType.GROUP, getParsedOptions("--group", "id").selectionType);
+        assertEquals(ClientParameters.SelectionType.GID, getParsedOptions("--gid", "id").selectionType);
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testMissingSelectionType() {
+       getParsedOptions();
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testFailOnMultipleDumpTypes() {
+        getParsedOptions("--user", "id", "--document", "id", "--group", "id", "--gid", "id");
+    }
+
+    @Test
+    public void testForceDumpOnDocumentOrGid() {
+        assertTrue(getParsedOptions("--document", "docid").dumpData);
+        assertTrue(getParsedOptions("--gid", "gid").dumpData);
+    }
+
+    @Test
+    public void testPrintHelp() {
+        ByteArrayOutputStream outContent = new ByteArrayOutputStream();
+        PrintStream oldOut = System.out;
+        System.setOut(new PrintStream(outContent));
+        try {
+            CommandLineOptions options = new CommandLineOptions();
+            options.printHelp();
+            String output = outContent.toString();
+            assertTrue(output.contains("vdsstat [options]"));
+        } finally {
+            System.setOut(oldOut);
+            outContent.reset();
+        }
+    }
+
+    @Test
+    public void testDefaultRoute() {
+        assertEquals("default", getParsedOptions("--user", "dummyuser").route);
+    }
+
+}
diff --git a/vespaclient-java/src/test/java/com/yahoo/vespavisit/VdsVisitTargetTestCase.java b/vespaclient-java/src/test/java/com/yahoo/vespavisit/VdsVisitTargetTestCase.java
new file mode 100644
index 00000000000..4eec05f7bc7
--- /dev/null
+++ b/vespaclient-java/src/test/java/com/yahoo/vespavisit/VdsVisitTargetTestCase.java
@@ -0,0 +1,56 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespavisit;
+
+import org.junit.Test;
+import static org.junit.Assert.*;
+
+public class VdsVisitTargetTestCase {
+
+    @Test
+    public void testParametersSlobrok() throws Exception {
+        VdsVisitTarget target = new VdsVisitTarget();
+        target.parseArguments(new String[]{
+                "--bindtoslobrok", "myname",
+                "--processtime", "34",
+                "--visithandler", "Foo",
+                "--visitoptions", "foo bar zoo",
+                "-i",
+                "-v"
+        });
+
+        assertEquals("myname", target.getSlobrokAddress());
+        assertEquals(34, target.getProcessTime());
+        assertEquals("Foo", target.getHandlerClassName());
+        assertEquals(3, target.getHandlerArgs().length);
+        assertEquals("foo", target.getHandlerArgs()[0]);
+        assertEquals("bar", target.getHandlerArgs()[1]);
+        assertEquals("zoo", target.getHandlerArgs()[2]);
+        assertTrue(target.isVerbose());
+        assertTrue(target.isPrintIds());
+    }
+
+    @Test
+    public void testParametersPort() throws Exception {
+        VdsVisitTarget target = new VdsVisitTarget();
+        target.parseArguments("--bindtosocket 1234".split(" "));
+        assertEquals(1234, target.getPort());
+        assertEquals(null, target.getSlobrokAddress());
+    }
+
+    public void assertException(String params) {
+        try {
+            VdsVisitTarget target = new VdsVisitTarget();
+            target.parseArguments(params.split(" "));
+            assertTrue(false);
+        } catch (Exception e) {
+
+        }
+    }
+
+    @Test
+    public void testPortAndSlobrok() {
+        assertException("--bindtoslobrok foo --bindtosocket 1234");
+        assertException("--bindtoport foo");
+    }
+
+}
diff --git a/vespaclient-java/src/test/java/com/yahoo/vespavisit/VdsVisitTestCase.java b/vespaclient-java/src/test/java/com/yahoo/vespavisit/VdsVisitTestCase.java
new file mode 100644
index 00000000000..49060a5715f
--- /dev/null
+++ b/vespaclient-java/src/test/java/com/yahoo/vespavisit/VdsVisitTestCase.java
@@ -0,0 +1,475 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespavisit;
+
+import com.yahoo.document.select.OrderingSpecification;
+import com.yahoo.document.select.parser.ParseException;
+import com.yahoo.documentapi.*;
+import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol;
+import com.yahoo.messagebus.StaticThrottlePolicy;
+import com.yahoo.messagebus.Trace;
+import com.yahoo.vespaclient.ClusterDef;
+import com.yahoo.vespaclient.ClusterList;
+import org.apache.commons.cli.Options;
+import org.junit.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.io.PrintStream;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import static org.junit.Assert.*;
+
+public class VdsVisitTestCase {
+
+    private VdsVisit.ArgumentParser createMockArgumentParser() {
+        Options opts = VdsVisit.createOptions();
+        return new VdsVisit.ArgumentParser(opts);
+    }
+
+    @Test
+    public void testCommandLineShortOptions() throws Exception {
+        // short options testing (for options that do not collide with each other)
+        String[] args = new String[] {
+                "-d", "foo.remote",
+                "-s", "'id.user=1234'",
+                "-f", "5678",
+                "-t", "9012",
+                "-l", "foodoc.bar,foodoc.baz",
+                "-m", "6000",
+                "-b", "5",
+                "-p", "foo-progress.txt",
+                "-u", "123456789",
+                "-c", "kittens",
+                "-r",
+                "-v"
+        };
+        VdsVisit.ArgumentParser parser = createMockArgumentParser();
+        VdsVisit.VdsVisitParameters allParams = parser.parse(args);
+        assertNotNull(allParams);
+
+        VisitorParameters params = allParams.getVisitorParameters();
+        assertNotNull(params);
+        assertEquals("foo.remote", params.getRemoteDataHandler());
+        assertEquals("'id.user=1234'", params.getDocumentSelection());
+        assertEquals(5678, params.getFromTimestamp());
+        assertEquals(9012, params.getToTimestamp());
+        assertEquals("foodoc.bar,foodoc.baz", params.getFieldSet());
+        assertEquals(6000, params.getMaxPending());
+        assertEquals(5, params.getMaxBucketsPerVisitor());
+        assertEquals("foo-progress.txt", params.getResumeFileName());
+        assertEquals(123456789, params.getTimeoutMs());
+        assertEquals(7 * 24 * 60 * 60 * 1000, allParams.getFullTimeout());
+        assertEquals("kittens", allParams.getCluster());
+        assertTrue(allParams.isVerbose());
+    }
+
+    /**
+     * Test the parameters that could not be used in conjunction with
+     * those in the first parameter test.
+     * @throws Exception
+     */
+    @Test
+    public void testCommandLineShortOptions2() throws Exception {
+        // Short options testing (for options that do not collide with each other)
+        String[] args = new String[] {
+                "-o", "654321",
+                "-e"
+        };
+        VdsVisit.ArgumentParser parser = createMockArgumentParser();
+        VdsVisit.VdsVisitParameters allParams = parser.parse(args);
+        assertNotNull(allParams);
+
+        VisitorParameters params = allParams.getVisitorParameters();
+        assertNotNull(params);
+        assertEquals(654321, allParams.getFullTimeout());
+        assertEquals(654321, params.getTimeoutMs());
+        assertEquals("[header]", params.getFieldSet());
+    }
+
+    @Test
+    public void testCommandLineShortOptionsPrintIdsOnly() throws Exception {
+        // Short options testing (for options that do not collide with each other)
+        String[] args = new String[] {
+                "-i"
+        };
+        VdsVisit.ArgumentParser parser = createMockArgumentParser();
+        VdsVisit.VdsVisitParameters allParams = parser.parse(args);
+        assertNotNull(allParams);
+
+        VisitorParameters params = allParams.getVisitorParameters();
+        assertNotNull(params);
+        assertEquals("[id]", params.getFieldSet());
+        assertTrue(allParams.isPrintIdsOnly());
+    }
+
+    @Test
+    public void testCommandLineLongOptions() throws Exception {
+        // short options testing (for options that do not collide with each other)
+        String[] args = new String[] {
+                "--datahandler", "foo.remote",
+                "--selection", "'id.user=1234'",
+                "--from", "5678",
+                "--to", "9012",
+                "--fieldset", "foodoc.bar,foodoc.baz",
+                "--maxpending", "6000",
+                "--maxbuckets", "5",
+                "--progress", "foo-progress.txt",
+                "--maxpendingsuperbuckets", "3",
+                "--buckettimeout", "123456789",
+                "--cluster", "kittens",
+                "--visitinconsistentbuckets",
+                "--visitlibrary", "fnord",
+                "--libraryparam", "asdf", "rargh",
+                "--libraryparam", "pinkie", "pie",
+                "--processtime", "555",
+                "--maxhits", "1001",
+                "--maxtotalhits", "2002",
+                "--tracelevel", "8",
+                "--priority", "NORMAL_1",
+                "--ordering", "ascending",
+                "--skipbucketsonfatalerrors",
+                "--abortonclusterdown",
+                "--visitremoves"
+        };
+        VdsVisit.ArgumentParser parser = createMockArgumentParser();
+        VdsVisit.VdsVisitParameters allParams = parser.parse(args);
+        assertNotNull(allParams);
+
+        VisitorParameters params = allParams.getVisitorParameters();
+        assertNotNull(params);
+
+        assertEquals("foo.remote", params.getRemoteDataHandler());
+        assertEquals("'id.user=1234'", params.getDocumentSelection());
+        assertEquals(5678, params.getFromTimestamp());
+        assertEquals(9012, params.getToTimestamp());
+        assertEquals("foodoc.bar,foodoc.baz", params.getFieldSet());
+        assertEquals(6000, params.getMaxPending());
+        assertEquals(5, params.getMaxBucketsPerVisitor());
+        assertEquals("foo-progress.txt", params.getResumeFileName());
+        assertEquals(123456789, params.getTimeoutMs());
+        assertEquals(7 * 24 * 60 * 60 * 1000, allParams.getFullTimeout());
+        assertEquals("kittens", allParams.getCluster());
+
+        assertTrue(params.getThrottlePolicy() instanceof StaticThrottlePolicy);
+        assertEquals(3, ((StaticThrottlePolicy)params.getThrottlePolicy()).getMaxPendingCount());
+
+        assertTrue(params.visitInconsistentBuckets());
+        assertEquals("fnord", params.getVisitorLibrary());
+        // TODO: FIXME? multiple library params doesn't work
+        assertTrue(Arrays.equals("rargh".getBytes(), params.getLibraryParameters().get("asdf")));
+        //assertTrue(Arrays.equals("pie".getBytes(), params.getLibraryParameters().get("pinkie")));
+        assertEquals(555, allParams.getProcessTime());
+        assertEquals(1001, params.getMaxFirstPassHits());
+        assertEquals(2002, params.getMaxTotalHits());
+        assertEquals(8, params.getTraceLevel());
+        assertEquals(DocumentProtocol.Priority.NORMAL_1, params.getPriority());
+        assertEquals(OrderingSpecification.ASCENDING, params.getVisitorOrdering());
+        assertTrue(allParams.getAbortOnClusterDown());
+        assertTrue(params.visitRemoves());
+
+        ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
+        PrintStream printStream = new PrintStream(outputStream);
+        VdsVisit.verbosePrintParameters(allParams, printStream);
+        printStream.flush();
+        String nl = System.getProperty("line.separator"); // the joys of running tests on windows
+        assertEquals(
+                "Time out visitor after 123456789 ms." + nl +
+                        "Visiting documents matching: 'id.user=1234'" + nl +
+                        "Visiting in the inclusive timestamp range 5678 - 9012." + nl +
+                        "Visiting field set foodoc.bar,foodoc.baz." + nl +
+                        "Visiting inconsistent buckets." + nl +
+                        "Including remove entries." + nl +
+                        "Tracking progress in file: foo-progress.txt" + nl +
+                        "Let visitor have maximum 6000 replies pending on data handlers per storage node visitor." + nl +
+                        "Visit maximum 5 buckets per visitor." + nl +
+                        "Sending data to data handler at: foo.remote" + nl +
+                        "Using visitor library 'fnord'." + nl +
+                        "Adding the following library specific parameters:" + nl +
+                        "  asdf = rargh" + nl +
+                        "Visitor priority NORMAL_1" + nl +
+                        "Skip visiting super buckets with fatal errors." + nl,
+                outputStream.toString("utf-8"));
+
+        args = new String[] {
+                "--ordering", "descending"
+        };
+        allParams = parser.parse(args);
+        params = allParams.getVisitorParameters();
+        assertEquals(OrderingSpecification.DESCENDING, params.getVisitorOrdering());
+    }
+
+    private static String[] emptyArgList() { return new String[]{}; }
+
+    @Test
+    public void visitor_priority_is_low1_by_default() throws Exception {
+        VdsVisit.VdsVisitParameters allParams = createMockArgumentParser().parse(emptyArgList());
+
+        VisitorParameters params = allParams.getVisitorParameters();
+        assertEquals(DocumentProtocol.Priority.LOW_1, params.getPriority());
+    }
+
+    @Test
+    public void testBadPriorityValue() throws Exception {
+        String[] args = new String[] {
+                "--priority", "super_hyper_important"
+        };
+        VdsVisit.ArgumentParser parser = createMockArgumentParser();
+        try {
+            parser.parse(args);
+            fail("no exception thrown");
+        } catch (IllegalArgumentException e) {
+            assertTrue(e.getMessage().contains("Unknown priority name"));
+        }
+    }
+
+    @Test
+    public void testBadOrderingValue() throws Exception {
+        String[] args = new String[] {
+                "--ordering", "yonder"
+        };
+        VdsVisit.ArgumentParser parser = createMockArgumentParser();
+        try {
+            parser.parse(args);
+            fail("no exception thrown");
+        } catch (IllegalArgumentException e) {
+            assertTrue(e.getMessage().contains("Unknown ordering"));
+        }
+    }
+
+    @Test
+    public void testCommandLineShortOptionsInvokeHelp() throws Exception {
+        // Short options testing (for options that do not collide with each other)
+        String[] args = new String[] {
+                "-h"
+        };
+        VdsVisit.ArgumentParser parser = createMockArgumentParser();
+        VdsVisit.VdsVisitParameters allParams = parser.parse(args);
+        assertNull(allParams);
+    }
+
+    @Test
+    public void testAutoSelectClusterRoute() throws Exception {
+        List<ClusterDef> clusterDefs = new ArrayList<>();
+        clusterDefs.add(new ClusterDef("storage", "content/cluster.foo/storage"));
+        ClusterList clusterList = new ClusterList(clusterDefs);
+
+        String route = VdsVisit.resolveClusterRoute(clusterList, null);
+        assertEquals("[Storage:cluster=storage;clusterconfigid=content/cluster.foo/storage]", route);
+    }
+
+    @Test
+    public void testBadClusterName() throws Exception {
+        List<ClusterDef> clusterDefs = new ArrayList<>();
+        clusterDefs.add(new ClusterDef("storage", "content/cluster.foo/storage"));
+        ClusterList clusterList = new ClusterList(clusterDefs);
+        try {
+            VdsVisit.resolveClusterRoute(clusterList, "borkbork");
+        } catch (IllegalArgumentException e) {
+            assertTrue(e.getMessage().contains("Your vespa cluster contains the content clusters storage, not borkbork."));
+        }
+    }
+
+    @Test
+    public void testRequireClusterOptionIfMultipleClusters() {
+        List<ClusterDef> clusterDefs = new ArrayList<>();
+        clusterDefs.add(new ClusterDef("storage", "content/cluster.foo/storage"));
+        clusterDefs.add(new ClusterDef("storage2", "content/cluster.bar/storage"));
+        ClusterList clusterList = new ClusterList(clusterDefs);
+        try {
+            VdsVisit.resolveClusterRoute(clusterList, null);
+        } catch (IllegalArgumentException e) {
+            assertTrue(e.getMessage().contains("Please use the -c option to select one of them"));
+        }
+    }
+
+    @Test
+    public void testExplicitClusterOptionWithMultipleClusters() {
+        List<ClusterDef> clusterDefs = new ArrayList<>();
+        clusterDefs.add(new ClusterDef("storage", "content/cluster.foo/storage"));
+        clusterDefs.add(new ClusterDef("storage2", "content/cluster.bar/storage"));
+        ClusterList clusterList = new ClusterList(clusterDefs);
+
+        String route = VdsVisit.resolveClusterRoute(clusterList, "storage2");
+        assertEquals("[Storage:cluster=storage2;clusterconfigid=content/cluster.bar/storage]", route);
+    }
+
+    @Test
+    public void testFailIfNoContentClustersAvailable() {
+        List<ClusterDef> clusterDefs = new ArrayList<>();
+        ClusterList clusterList = new ClusterList(clusterDefs);
+        try {
+            VdsVisit.resolveClusterRoute(clusterList, null);
+        } catch (IllegalArgumentException e) {
+            assertTrue(e.getMessage().contains("Your Vespa cluster does not have any content clusters"));
+        }
+    }
+
+    @Test
+    public void testStatistics() throws Exception {
+        String[] args = new String[] {
+                "--statistics", "foo"
+        };
+        VdsVisit.ArgumentParser parser = createMockArgumentParser();
+        VdsVisit.VdsVisitParameters allParams = parser.parse(args);
+        assertNotNull(allParams);
+
+        VisitorParameters params = allParams.getVisitorParameters();
+        assertNotNull(params);
+        assertEquals("foo", allParams.getStatisticsParts());
+        assertEquals("[id]", params.getFieldSet());
+        assertEquals("CountVisitor", params.getVisitorLibrary());
+    }
+
+    // TODO: use DummyVisitorSession instead?
+    private static class MockVisitorSession implements VisitorSession {
+        private VisitorParameters params;
+
+        public MockVisitorSession(VisitorParameters params) {
+            this.params = params;
+            params.getLocalDataHandler().setSession(this);
+        }
+
+        @Override
+        public boolean isDone() {
+            return true;
+        }
+
+        @Override
+        public ProgressToken getProgress() {
+            return null;
+        }
+
+        @Override
+        public Trace getTrace() {
+            return null;
+        }
+
+        @Override
+        public boolean waitUntilDone(long l) throws InterruptedException {
+            params.getControlHandler().onDone(VisitorControlHandler.CompletionCode.SUCCESS, "woo!");
+            // Return immediately
+            return true;
+        }
+
+        @Override
+        public void ack(AckToken ackToken) {
+        }
+
+        @Override
+        public void abort() {
+        }
+
+        @Override
+        public VisitorResponse getNext() {
+            return null;
+        }
+
+        @Override
+        public VisitorResponse getNext(int i) throws InterruptedException {
+            return null;
+        }
+
+        @Override
+        public void destroy() {
+        }
+    }
+
+    private static class MockVisitorSessionAccessor implements VdsVisit.VisitorSessionAccessor {
+        boolean shutdown = false;
+        @Override
+        public VisitorSession createVisitorSession(VisitorParameters params) throws ParseException {
+            return new MockVisitorSession(params);
+        }
+
+        @Override
+        public void shutdown() {
+            shutdown = true;
+        }
+
+        public boolean isShutdown() {
+            return shutdown;
+        }
+    }
+
+    private static class MockVisitorSessionAccessorFactory implements VdsVisit.VisitorSessionAccessorFactory {
+
+        private MockVisitorSessionAccessor lastCreatedAccessor = null;
+
+        @Override
+        public VdsVisit.VisitorSessionAccessor createVisitorSessionAccessor() {
+            lastCreatedAccessor = new MockVisitorSessionAccessor();
+            return lastCreatedAccessor;
+        }
+
+        public MockVisitorSessionAccessor getLastCreatedAccessor() {
+            return lastCreatedAccessor;
+        }
+    }
+
+    private static class MockShutdownHookRegistrar implements VdsVisit.ShutdownHookRegistrar {
+        Thread cleanUpThread;
+
+        @Override
+        public void registerShutdownHook(Thread thread) {
+            cleanUpThread = thread;
+        }
+
+        public Thread getCleanUpThread() {
+            return cleanUpThread;
+        }
+    }
+
+    @Test
+    public void testVdsVisitRunLogic() {
+        MockVisitorSessionAccessorFactory accessorFactory = new MockVisitorSessionAccessorFactory();
+        MockShutdownHookRegistrar shutdownHookRegistrar = new MockShutdownHookRegistrar();
+        VdsVisit vdsVisit = new VdsVisit(accessorFactory, shutdownHookRegistrar);
+
+        VdsVisit.VdsVisitParameters params = new VdsVisit.VdsVisitParameters();
+        VisitorParameters visitorParameters = new VisitorParameters("");
+        params.setVisitorParameters(visitorParameters);
+
+        visitorParameters.setResumeFileName("src/test/files/progress.txt");
+        vdsVisit.setVdsVisitParameters(params);
+
+        int code = vdsVisit.doRun();
+        assertEquals(0, code);
+
+        assertNotNull(shutdownHookRegistrar.getCleanUpThread());
+        shutdownHookRegistrar.getCleanUpThread().run();
+
+        assertNotNull(accessorFactory.getLastCreatedAccessor());
+        assertTrue(accessorFactory.getLastCreatedAccessor().isShutdown());
+
+        // Ensure progress token stuff was read from file
+        ProgressToken progress = visitorParameters.getResumeToken();
+        assertNotNull(progress);
+        assertEquals(14, progress.getDistributionBitCount());
+        assertEquals(3, progress.getPendingBucketCount());
+    }
+
+    @Test
+    public void testVdsVisitRunLogicProgressFileNotYetCreated() {
+        MockVisitorSessionAccessorFactory accessorFactory = new MockVisitorSessionAccessorFactory();
+        MockShutdownHookRegistrar shutdownHookRegistrar = new MockShutdownHookRegistrar();
+        VdsVisit vdsVisit = new VdsVisit(accessorFactory, shutdownHookRegistrar);
+
+        VdsVisit.VdsVisitParameters params = new VdsVisit.VdsVisitParameters();
+        VisitorParameters visitorParameters = new VisitorParameters("");
+        params.setVisitorParameters(visitorParameters);
+
+        visitorParameters.setResumeFileName("src/test/files/progress-not-existing.txt");
+        vdsVisit.setVdsVisitParameters(params);
+
+        // Should not fail with file not found
+        int code = vdsVisit.doRun();
+        assertEquals(0, code);
+
+        assertNotNull(shutdownHookRegistrar.getCleanUpThread());
+        shutdownHookRegistrar.getCleanUpThread().run();
+
+        assertNotNull(accessorFactory.getLastCreatedAccessor());
+        assertTrue(accessorFactory.getLastCreatedAccessor().isShutdown());
+    }
+}
diff --git a/vespaclient/src/vespa/vespaclient/spoolmaster/application.cpp b/vespaclient/src/vespa/vespaclient/spoolmaster/application.cpp
index fe0880d9ea1..e708e623592 100644
--- a/vespaclient/src/vespa/vespaclient/spoolmaster/application.cpp
+++ b/vespaclient/src/vespa/vespaclient/spoolmaster/application.cpp
@@ -1,10 +1,8 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 #include <vespa/defaults.h>
-#include <vector>
-#include <string>
+#include <thread>
 #include <iostream>
 #include <algorithm>
-#include <cstdio>
 #include <dirent.h>
 #include <unistd.h>
 
@@ -189,7 +187,7 @@ Application::Main()
             if (scanInbox() && findOutboxes()) {
                 moveLinks();
             } else {
-                FastOS_Thread::Sleep(200);
+                std::this_thread::sleep_for(std::chrono::milliseconds(200));
             }
         }
     }
diff --git a/vespaclient/src/vespa/vespaclient/vespadoclocator/locator.cpp b/vespaclient/src/vespa/vespaclient/vespadoclocator/locator.cpp
index 030f284c182..af805d461fc 100644
--- a/vespaclient/src/vespa/vespaclient/vespadoclocator/locator.cpp
+++ b/vespaclient/src/vespa/vespaclient/vespadoclocator/locator.cpp
@@ -6,8 +6,7 @@
 #include <vespa/messagebus/iconfighandler.h>
 #include <vespa/messagebus/routing/routingspec.h>
 #include <vespa/vdslib/bucketdistribution.h>
-#include <vespa/messagebus/config-messagebus.h>
-#include <vespa/config/helper/configgetter.h>
+#include <vespa/vespalib/util/stringfmt.h>
 #include <vespa/config/helper/configgetter.hpp>
 
 
diff --git a/vespaclient/src/vespa/vespaclient/vesparoute/application.cpp b/vespaclient/src/vespa/vespaclient/vesparoute/application.cpp
index e9482c1eddc..baf7c01d631 100644
--- a/vespaclient/src/vespa/vespaclient/vesparoute/application.cpp
+++ b/vespaclient/src/vespa/vespaclient/vesparoute/application.cpp
@@ -12,7 +12,7 @@
 #include <vespa/messagebus/rpcmessagebus.h>
 #include <vespa/slobrok/sbmirror.h>
 #include <vespa/config/helper/configgetter.hpp>
-
+#include <vespa/vespalib/util/stringfmt.h>
 
 using config::ConfigGetter;
 using document::DocumenttypesConfig;
diff --git a/vespalib/src/tests/atomic/atomic_bench.cpp b/vespalib/src/tests/atomic/atomic_bench.cpp
index 003ef25ba45..6294f956507 100644
--- a/vespalib/src/tests/atomic/atomic_bench.cpp
+++ b/vespalib/src/tests/atomic/atomic_bench.cpp
@@ -2,6 +2,7 @@
 
 #include <vespa/vespalib/testkit/testapp.h>
 #include <vespa/vespalib/util/atomic.h>
+#include <vespa/fastos/thread.h>
 #include <vector>
 #include <algorithm>
 #include <sstream>
diff --git a/vespalib/src/tests/atomic/atomic_test.cpp b/vespalib/src/tests/atomic/atomic_test.cpp
index 1133d9b51bd..09fc99edeb9 100644
--- a/vespalib/src/tests/atomic/atomic_test.cpp
+++ b/vespalib/src/tests/atomic/atomic_test.cpp
@@ -1,12 +1,14 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-#include <vespa/log/log.h>
-LOG_SETUP("atomic_test");
+
 #include <vespa/vespalib/testkit/testapp.h>
 #include <vespa/vespalib/util/atomic.h>
+#include <vespa/fastos/thread.h>
 #include <vector>
 #include <algorithm>
 #include <sstream>
 
+#include <vespa/log/log.h>
+LOG_SETUP("atomic_test");
 
 class Test : public vespalib::TestApp
 {
diff --git a/vespalib/src/tests/benchmark_timer/benchmark_timer_test.cpp b/vespalib/src/tests/benchmark_timer/benchmark_timer_test.cpp
index b0cd577cc66..ac6a57358d6 100644
--- a/vespalib/src/tests/benchmark_timer/benchmark_timer_test.cpp
+++ b/vespalib/src/tests/benchmark_timer/benchmark_timer_test.cpp
@@ -8,7 +8,7 @@ TEST("require that the benchmark timer can be used as advertised") {
     BenchmarkTimer timer(1.0);
     while (timer.has_budget()) {
         timer.before();
-        FastOS_Thread::Sleep(5);
+        std::this_thread::sleep_for(std::chrono::milliseconds(5));
         timer.after();
     }
     EXPECT_TRUE(timer.min_time() >= 0.0);
@@ -17,15 +17,15 @@ TEST("require that the benchmark timer can be used as advertised") {
 
 TEST("require that the benchmark timer all-in-one benchmarking works") {
     uint32_t sleep_time = 5;
-    double t = BenchmarkTimer::benchmark([sleep_time](){FastOS_Thread::Sleep(sleep_time);}, 1.0);
+    double t = BenchmarkTimer::benchmark([sleep_time](){std::this_thread::sleep_for(std::chrono::milliseconds(sleep_time));}, 1.0);
     fprintf(stderr, "5 ms sleep takes: %g ms\n", t * 1000.0);
 }
 
 TEST("require that the benchmark timer all-in-one benchmarking with baseline works") {
     uint32_t work_time = 10;
     uint32_t baseline_time = 5;
-    double t = BenchmarkTimer::benchmark([&](){FastOS_Thread::Sleep(work_time);},
-                                         [&](){FastOS_Thread::Sleep(baseline_time);}, 1.0);
+    double t = BenchmarkTimer::benchmark([&](){std::this_thread::sleep_for(std::chrono::milliseconds(work_time));},
+                                         [&](){std::this_thread::sleep_for(std::chrono::milliseconds(baseline_time));}, 1.0);
     fprintf(stderr, "10 ms sleep - 5 ms sleep takes: %g ms\n", t * 1000.0);
 }
 
@@ -33,8 +33,8 @@ TEST("require that the benchmark timer all-in-one benchmarking with baseline and
     uint32_t work_time = 2;
     uint32_t baseline_time = 1;
     uint32_t loop_cnt = 0;
-    double t = BenchmarkTimer::benchmark([&](){FastOS_Thread::Sleep(work_time); ++loop_cnt;},
-                                         [&](){FastOS_Thread::Sleep(baseline_time);}, 7, 0.0);
+    double t = BenchmarkTimer::benchmark([&](){std::this_thread::sleep_for(std::chrono::milliseconds(work_time)); ++loop_cnt;},
+                                         [&](){std::this_thread::sleep_for(std::chrono::milliseconds(baseline_time));}, 7, 0.0);
     EXPECT_EQUAL(loop_cnt, 7u);
     fprintf(stderr, "2 ms sleep - 1 ms sleep takes: %g ms\n", t * 1000.0);
 }
diff --git a/vespalib/src/tests/delegatelist/delegatelist.cpp b/vespalib/src/tests/delegatelist/delegatelist.cpp
index 11dd4d7d9e2..6a9751440a6 100644
--- a/vespalib/src/tests/delegatelist/delegatelist.cpp
+++ b/vespalib/src/tests/delegatelist/delegatelist.cpp
@@ -3,10 +3,7 @@
 #include <vespa/vespalib/testkit/testapp.h>
 #include <vespa/vespalib/util/delegatelist.hpp>
 #include <vespa/vespalib/util/guard.h>
-
-#include <memory>
-#include <algorithm>
-#include <vector>
+#include <vespa/fastos/thread.h>
 #include <queue>
 
 #include <vespa/log/log.h>
diff --git a/vespalib/src/tests/exception_classes/exception_classes_test.cpp b/vespalib/src/tests/exception_classes/exception_classes_test.cpp
index 946c3fa32e4..a418cb421ed 100644
--- a/vespalib/src/tests/exception_classes/exception_classes_test.cpp
+++ b/vespalib/src/tests/exception_classes/exception_classes_test.cpp
@@ -1,6 +1,7 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 #include <vespa/vespalib/testkit/test_kit.h>
 #include <vespa/vespalib/util/exceptions.h>
+#include <vespa/vespalib/util/stringfmt.h>
 
 using namespace vespalib;
 
diff --git a/vespalib/src/tests/time/time_box_test.cpp b/vespalib/src/tests/time/time_box_test.cpp
index 478d749e15d..32cf1eb66b3 100644
--- a/vespalib/src/tests/time/time_box_test.cpp
+++ b/vespalib/src/tests/time/time_box_test.cpp
@@ -10,13 +10,13 @@ TEST("require that long-lived timebox returns falling time left numbers") {
         double timeLeft = box.timeLeft();
         EXPECT_TRUE(timeLeft <= last_timeLeft);
         last_timeLeft = timeLeft;
-        FastOS_Thread::Sleep(10);
+        std::this_thread::sleep_for(std::chrono::milliseconds(10));
     }
 }
 
 TEST("require that short-lived timebox times out") {
     vespalib::TimeBox box(0.125);
-    FastOS_Thread::Sleep(150);
+    std::this_thread::sleep_for(std::chrono::milliseconds(150));
     EXPECT_FALSE(box.hasTimeLeft());
     EXPECT_EQUAL(box.timeLeft(), 0.0);
 }
@@ -27,7 +27,7 @@ TEST("require that short-lived timebox always returns at least minimum time") {
         double timeLeft = box.timeLeft();
         EXPECT_TRUE(timeLeft <= 0.250);
         EXPECT_TRUE(timeLeft >= 0.125);
-        FastOS_Thread::Sleep(30);
+        std::this_thread::sleep_for(std::chrono::milliseconds(30));
     }
     EXPECT_FALSE(box.hasTimeLeft());
     EXPECT_EQUAL(box.timeLeft(), 0.125);
diff --git a/vespalib/src/vespa/vespalib/io/fileutil.cpp b/vespalib/src/vespa/vespalib/io/fileutil.cpp
index c9778971efd..2e3994d329f 100644
--- a/vespalib/src/vespa/vespalib/io/fileutil.cpp
+++ b/vespalib/src/vespa/vespalib/io/fileutil.cpp
@@ -1,16 +1,12 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "fileutil.h"
-
-#include <errno.h>
-#include <ostream>
-#include <sys/stat.h>
-#include <sys/types.h>
-#include <unistd.h>
 #include <vespa/vespalib/util/exceptions.h>
-#include <vespa/vespalib/util/error.h>
 #include <vespa/vespalib/stllike/asciistream.h>
+#include <vespa/vespalib/util/stringfmt.h>
 #include <vespa/fastos/file.h>
+#include <ostream>
+#include <sys/stat.h>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".vespalib.io.fileutil");
diff --git a/vespalib/src/vespa/vespalib/objects/nbostream.cpp b/vespalib/src/vespa/vespalib/objects/nbostream.cpp
index ec04ffb8b3b..e37399bdfcc 100644
--- a/vespalib/src/vespa/vespalib/objects/nbostream.cpp
+++ b/vespalib/src/vespa/vespalib/objects/nbostream.cpp
@@ -2,7 +2,7 @@
 #include "nbostream.hpp"
 #include "hexdump.h"
 #include <vespa/vespalib/util/exceptions.h>
-
+#include <vespa/vespalib/util/stringfmt.h>
 
 namespace vespalib {
 
diff --git a/vespalib/src/vespa/vespalib/util/buffer.h b/vespalib/src/vespa/vespalib/util/buffer.h
index 88fe6359aae..3a43a2c0dc4 100644
--- a/vespalib/src/vespa/vespalib/util/buffer.h
+++ b/vespalib/src/vespa/vespalib/util/buffer.h
@@ -1,7 +1,7 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 #pragma once
 
-#include <sys/types.h>
+#include <cstddef>
 
 namespace vespalib {
 
diff --git a/vespalib/src/vespa/vespalib/util/exception.h b/vespalib/src/vespa/vespalib/util/exception.h
index 44911ea5555..05f48295bfa 100644
--- a/vespalib/src/vespa/vespalib/util/exception.h
+++ b/vespalib/src/vespa/vespalib/util/exception.h
@@ -5,7 +5,6 @@
 
 #include <vespa/vespalib/util/macro.h>
 #include <vespa/vespalib/util/error.h>
-#include <vespa/vespalib/util/stringfmt.h>
 #include <exception>
 
 #define VESPALIB_EXCEPTION_USEBACKTRACES
diff --git a/vespalib/src/vespa/vespalib/util/exceptions.h b/vespalib/src/vespa/vespalib/util/exceptions.h
index 65d17baf654..ca05dac107f 100644
--- a/vespalib/src/vespa/vespalib/util/exceptions.h
+++ b/vespalib/src/vespa/vespalib/util/exceptions.h
@@ -13,6 +13,7 @@
 #pragma once
 
 #include <vespa/vespalib/util/exception.h>
+#include <vespa/vespalib/util/stringfmt.h>
 
 namespace vespalib {
 
diff --git a/vespalib/src/vespa/vespalib/util/memory.h b/vespalib/src/vespa/vespalib/util/memory.h
index 2d940882320..85c8e46dcc2 100644
--- a/vespalib/src/vespa/vespalib/util/memory.h
+++ b/vespalib/src/vespa/vespalib/util/memory.h
@@ -4,8 +4,8 @@
 #pragma once
 
 #include <memory>
-#include <string.h>
-#include <stdlib.h>
+#include <cstring>
+#include <cstdlib>
 
 /// Macro to give you number of elements in a defined array.
 #define VESPA_NELEMS(a)    (sizeof(a)/sizeof(a[0]))
diff --git a/vespalib/src/vespa/vespalib/util/sync.h b/vespalib/src/vespa/vespalib/util/sync.h
index a44763ec755..315bb79487e 100644
--- a/vespalib/src/vespa/vespalib/util/sync.h
+++ b/vespalib/src/vespa/vespalib/util/sync.h
@@ -34,7 +34,6 @@ public:
 };
 #endif
 
-
 /**
  * @brief A Lock is a synchronization primitive used to ensure mutual
  * exclusion.
diff --git a/vespalog/src/test/threads/testthreads.cpp b/vespalog/src/test/threads/testthreads.cpp
index a708777f350..056f5aad2a2 100644
--- a/vespalog/src/test/threads/testthreads.cpp
+++ b/vespalog/src/test/threads/testthreads.cpp
@@ -1,12 +1,12 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 #include <vespa/fastos/app.h>
 #include <vespa/fastos/time.h>
+#include <vespa/fastos/thread.h>
 #include <vespa/log/log.h>
 #include <iostream>
 
 using std::string;
 
-
 LOG_SETUP(".threadtest");
 
 class FileThread : public FastOS_Runnable
diff --git a/vsm/src/tests/docsum/docsum.cpp b/vsm/src/tests/docsum/docsum.cpp
index d854421ecb5..1b0fbf70623 100644
--- a/vsm/src/tests/docsum/docsum.cpp
+++ b/vsm/src/tests/docsum/docsum.cpp
@@ -1,8 +1,7 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 #include <vespa/vespalib/testkit/testapp.h>
-
-#include <vector>
 #include <vespa/document/fieldvalue/fieldvalues.h>
+#include <vespa/document/datatype/structdatatype.h>
 #include <vespa/vsm/common/docsum.h>
 #include <vespa/vsm/vsm/flattendocsumwriter.h>
 #include <vespa/vsm/vsm/slimefieldwriter.h>
diff --git a/vsm/src/tests/document/document.cpp b/vsm/src/tests/document/document.cpp
index 35ca55826ba..629e2954953 100644
--- a/vsm/src/tests/document/document.cpp
+++ b/vsm/src/tests/document/document.cpp
@@ -2,10 +2,10 @@
 #include <vespa/vespalib/testkit/testapp.h>
 
 #include <vespa/document/fieldvalue/fieldvalues.h>
+#include <vespa/document/datatype/documenttype.h>
 #include <vespa/vsm/common/storagedocument.h>
 #include <vespa/vespalib/stllike/asciistream.h>
 
-
 using namespace document;
 
 namespace vsm {
diff --git a/vsm/src/vespa/vsm/common/documenttypemapping.cpp b/vsm/src/vespa/vsm/common/documenttypemapping.cpp
index 96a7effae57..9a8bbb28f65 100644
--- a/vsm/src/vespa/vsm/common/documenttypemapping.cpp
+++ b/vsm/src/vespa/vsm/common/documenttypemapping.cpp
@@ -2,6 +2,7 @@
 
 #include "documenttypemapping.h"
 #include <vespa/document/repo/documenttyperepo.h>
+#include <vespa/document/datatype/documenttype.h>
 #include <vespa/vespalib/stllike/hash_map.hpp>
 
 #include <vespa/log/log.h>
diff --git a/vsm/src/vespa/vsm/searcher/utf8stringfieldsearcherbase.cpp b/vsm/src/vespa/vsm/searcher/utf8stringfieldsearcherbase.cpp
index 8e444102930..c713e0284ac 100644
--- a/vsm/src/vespa/vsm/searcher/utf8stringfieldsearcherbase.cpp
+++ b/vsm/src/vespa/vsm/searcher/utf8stringfieldsearcherbase.cpp
@@ -1,6 +1,7 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "utf8stringfieldsearcherbase.h"
+#include <cassert>
 
 using search::QueryTerm;
 using search::QueryTermList;
diff --git a/vsm/src/vespa/vsm/searcher/utf8substringsnippetmodifier.cpp b/vsm/src/vespa/vsm/searcher/utf8substringsnippetmodifier.cpp
index 23e64d0bf5e..b5d49f463ef 100644
--- a/vsm/src/vespa/vsm/searcher/utf8substringsnippetmodifier.cpp
+++ b/vsm/src/vespa/vsm/searcher/utf8substringsnippetmodifier.cpp
@@ -1,5 +1,6 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 #include "utf8substringsnippetmodifier.h"
+#include <cassert>
 
 using search::byte;
 using search::QueryTerm;
author	Harald Musum <musum@yahoo-inc.com>	2017-06-07 10:38:51 +0200
committer	Harald Musum <musum@yahoo-inc.com>	2017-06-07 10:38:51 +0200
commit	125a73cd75b8db5caf0a67f7019faf57411e3a45 (patch)
tree	c8033a7954d57393580afe3640acf39af1193f3c
parent	f7ca4ca6e113aba0f7648d6ddcac18ee7757fb79 (diff)
parent	86e1f3e7b711c2d7787ed8a546f9c809c4ccc787 (diff)