Merge with master

author: Jon Bratseth <bratseth@yahoo-inc.com> 2017-06-08 11:14:36 +0200
committer: Jon Bratseth <bratseth@yahoo-inc.com> 2017-06-08 11:14:36 +0200
commit: 2b5100db52c300ce7b336da6dae2008baad57358 (patch)
tree: bd56fb339921d458b71f320317b03a2318df272f
parent: 739833c51230d2bdb7699906b7acaa6a8bb76044 (diff)
parent: ab131f2a48628dd7ed86e112819860c67e8df75b (diff)
630 files changed, 11939 insertions, 4535 deletions
diff --git a/.gitignore b/.gitignore
index 7ad1a865a5e..6e7db58ea54 100644
--- a/.gitignore
+++ b/.gitignore
@@ -41,3 +41,4 @@ Testing
 /vespa-hadoop/dependency-reduced-pom.xml
 .preprocessed/
 .DS_Store
+install_manifest.txt
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/InstanceResolver.java b/config-model/src/main/java/com/yahoo/vespa/model/InstanceResolver.java
index 9c761e425a7..fdb9fdf9796 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/InstanceResolver.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/InstanceResolver.java
@@ -11,6 +11,7 @@ import com.yahoo.vespa.config.buildergen.ConfigDefinition;
 import com.yahoo.yolean.Exceptions;
 import com.yahoo.vespa.config.*;
 
+import java.lang.reflect.Constructor;
 import java.lang.reflect.Field;
 import java.lang.reflect.Method;
 import java.util.List;
@@ -46,13 +47,19 @@ class InstanceResolver {
      * @param targetDef the def to use
      * @return the config instance or null of no producer for this found in model
      */
+    @SuppressWarnings("unchecked")
     static ConfigInstance resolveToInstance(ConfigKey<?> key, ConfigBuilder builder, InnerCNode targetDef) {
-        ConfigDefinitionKey defKey = new ConfigDefinitionKey(key);
         try {
             if (targetDef != null) applyDef(builder, targetDef);
-            ConfigInstance instance = getInstance(defKey, builder.getClass().getClassLoader());
-            Class<? extends ConfigInstance> clazz = instance.getClass();
-            return clazz.getConstructor(new Class<?>[]{builder.getClass()}).newInstance(builder);
+            Class<?> clazz = builder.getClass().getEnclosingClass();
+            if (!(ConfigInstance.class.isAssignableFrom(clazz))) {
+                throw new ConfigurationRuntimeException("Cannot produce config for the name '" + key.getName() + ", as "
+                                                                + clazz.getName() + " is not a ConfigInstance.");
+            }
+            Class<? extends ConfigInstance> configClass = (Class<? extends ConfigInstance>)clazz;
+            Constructor<? extends ConfigInstance> constructor = configClass.getDeclaredConstructor(builder.getClass(), boolean.class);
+            constructor.setAccessible(true);
+            return constructor.newInstance(builder, /*throwIfUninitialized*/ false);
         } catch (Exception e) {
             throw new ConfigurationRuntimeException(e);
         }
@@ -143,51 +150,6 @@ class InstanceResolver {
         }
     }
 
-    /**
-     * Create a ConfigBuilder given a definition key and a payload
-     * @param key The key to use to create the correct builder.
-     * @param payload The payload to populate the builder with.
-     * @return A ConfigBuilder initialized with payload.
-     */
-    static ConfigBuilder createBuilderFromPayload(ConfigDefinitionKey key, VespaModel model, ConfigPayload payload, ConfigDefinition targetDef) {
-        ConfigBuilder builderInstance = model.createBuilder(key, targetDef);
-        if (builderInstance == null || builderInstance instanceof GenericConfig.GenericConfigBuilder) {
-            if (log.isLoggable(LogLevel.SPAM)) {
-                log.log(LogLevel.SPAM, "Creating generic builder for key=" + key);
-            }
-            return new GenericConfig.GenericConfigBuilder(key, new ConfigPayloadBuilder(payload));
-        }
-        ConfigTransformer transformer = new ConfigTransformer(builderInstance.getClass().getDeclaringClass());
-        return transformer.toConfigBuilder(payload);
-    }
-
-    /**
-     * Returns a {@link ConfigInstance} of right type for given key using reflection
-     *
-     * @param  cKey a ConfigKey
-     * @return a {@link ConfigInstance} or null if not available in classpath
-     */
-    static ConfigInstance getInstance(ConfigDefinitionKey cKey, ClassLoader instanceLoader) {
-        String className = ConfigGenerator.createClassName(cKey.getName());
-        Class<?> clazz;
-        String fullClassName = packageName(cKey) + "." + className;
-        try {
-            clazz = instanceLoader != null ? instanceLoader.loadClass(fullClassName) : Class.forName(fullClassName);
-        } catch (ClassNotFoundException e) {
-            return null;
-        }
-        Object i;
-        try {
-            i = clazz.newInstance();
-        } catch (InstantiationException | IllegalAccessException e) {
-            throw new ConfigurationRuntimeException(e);
-        }
-        if (!(i instanceof ConfigInstance)) {
-            throw new ConfigurationRuntimeException(fullClassName + " is not a ConfigInstance, can not produce config for the name '" + cKey.getName() + "'.");
-        }
-        return (ConfigInstance) i;
-    }
-
     static String packageName(ConfigDefinitionKey cKey) {
         String prefix = "com.yahoo.";
         return prefix + (cKey.getNamespace().isEmpty() ? CNode.DEFAULT_NAMESPACE : cKey.getNamespace());
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/SystemMetrics.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/SystemMetrics.java
index 5f637090b73..5967771d904 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/SystemMetrics.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/SystemMetrics.java
@@ -35,10 +35,9 @@ public class SystemMetrics {
                 // Disk metrics should be based on /home, or else '/' - or simply add filesystem as dimension
                 ImmutableSet.of(new Metric("cpu.busy.pct", CPU_UTIL),
                                 new Metric("mem.used.pct", MEM_UTIL),
-                                new Metric("memory.usage", MEM_USED),
+                                new Metric("mem.active.kb", MEM_USED),
                                 new Metric("mem.total.kb", MEM_LIMIT),
-                                new Metric("fs.used.kb", DISK_USED),
-                                new Metric("fs.capacity.kb", DISK_LIMIT)
+                                new Metric("used.kb", DISK_USED)
                 );
 
         Set<Metric> systemMetrics = ImmutableSet.<Metric>builder()
diff --git a/config-model/src/main/perl/deploy b/config-model/src/main/perl/deploy
index 6c165951ab0..dc7b2132792 100755
--- a/config-model/src/main/perl/deploy
+++ b/config-model/src/main/perl/deploy
@@ -72,7 +72,7 @@ readConfFile();
 use strict;
 use warnings;
 use feature qw(switch say);
-use vars qw/ $opt_h $opt_n $opt_v $opt_f $opt_t $opt_1 $opt_a $opt_e $opt_E $opt_r $opt_i $opt_p $opt_H $opt_R /;
+use vars qw/ $opt_c $opt_h $opt_n $opt_v $opt_f $opt_t $opt_a $opt_e $opt_E $opt_r $opt_i $opt_p $opt_H $opt_R /;
 use Env qw($HOME);
 use JSON;
 use Getopt::Std;
@@ -99,15 +99,20 @@ my $environment = "prod";
 my $region = "default";
 my $instance = "default";
 my $version = "v2";
+my $configserver = "";
 my $port = "19071";
 
-getopts('fhnt:v1e:E:r:a:i:p:HR:');
+getopts('c:fhnt:ve:E:r:a:i:p:HR:');
 
 if ($opt_h) {
   usage();
   exit 0;
 }
 
+if ($opt_c) {
+    $configserver = $opt_c;
+}
+
 if ($opt_e) {
   $tenant = $opt_e;
 }
@@ -132,13 +137,7 @@ if ($opt_p) {
   $port = $opt_p;
 }
 
-# Use v1 of application API
-if ($opt_1) {
-  $version = "v1";
-  $pathPrefix = "/application/v1/session";
-} else {
-  $pathPrefix = "/application/v2/tenant/$tenant/session";
-}
+$pathPrefix = "/application/v2/tenant/$tenant/session";
 
 create_cloudconfig_dir();
 
@@ -303,7 +302,13 @@ sub fetch_directory {
 sub get_configsource_url {
   my ($command) = @_;
 
-  my @configsources = split(' ', `$VESPA_HOME/bin/vespa-print-default configservers_http`);
+  my @configsources;
+  if ($configserver and $configserver ne "") {
+      @configsources = ('http://' . $configserver . ':' . $port . '/');
+  } else {
+      @configsources = split(' ', `$VESPA_HOME/bin/vespa-print-default configservers_http`);
+  }
+
   my $configsource_url = shift(@configsources);
   if (!$configsource_url) {
     die "Could not get url to config server, make sure that VESPA_HOME and services.addr_configserver is set\n";
diff --git a/config/src/tests/configholder/configholder.cpp b/config/src/tests/configholder/configholder.cpp
index 8901ba65ad4..c103f86f6e9 100644
--- a/config/src/tests/configholder/configholder.cpp
+++ b/config/src/tests/configholder/configholder.cpp
@@ -61,7 +61,7 @@ TEST_MT_F("Require that wait is interrupted", 2, ConfigHolder)
         TEST_BARRIER();
     } else {
         TEST_BARRIER();
-        FastOS_Thread::Sleep(500);
+        std::this_thread::sleep_for(std::chrono::milliseconds(500));
         f.interrupt();
         TEST_BARRIER();
     }
diff --git a/config/src/tests/subscription/subscription.cpp b/config/src/tests/subscription/subscription.cpp
index 5974279518a..52b9ec624ac 100644
--- a/config/src/tests/subscription/subscription.cpp
+++ b/config/src/tests/subscription/subscription.cpp
@@ -79,7 +79,7 @@ TEST_MT_F("requireThatNextUpdateReturnsWhenNotified", 2, SubscriptionFixture(Con
         ASSERT_TRUE(f1.sub.nextUpdate(2, 5000));
         ASSERT_TRUE(timer.MilliSecsToNow() > 200.0);
     } else {
-        FastOS_Thread::Sleep(500);
+        std::this_thread::sleep_for(std::chrono::milliseconds(500));
         f1.holder->handle(ConfigUpdate::UP(new ConfigUpdate(ConfigValue(), 1, 1)));
     }
 }
@@ -94,7 +94,7 @@ TEST_MT_F("requireThatNextUpdateReturnsInterrupted", 2, SubscriptionFixture(Conf
         ASSERT_TRUE(f1.sub.nextUpdate(1, 5000));
         ASSERT_TRUE(timer.MilliSecsToNow() > 300.0);
     } else {
-        FastOS_Thread::Sleep(500);
+        std::this_thread::sleep_for(std::chrono::milliseconds(500));
         f1.sub.close();
     }
 }
diff --git a/config/src/vespa/config/common/configcontext.h b/config/src/vespa/config/common/configcontext.h
index c00bceaef47..4f19be9a928 100644
--- a/config/src/vespa/config/common/configcontext.h
+++ b/config/src/vespa/config/common/configcontext.h
@@ -5,8 +5,6 @@
 #include "timingvalues.h"
 #include "configmanager.h"
 #include <vespa/config/subscription/sourcespec.h>
-#include <vespa/vespalib/util/sync.h>
-#include <vespa/vespalib/stllike/hash_map.h>
 
 namespace config {
 
diff --git a/config/src/vespa/config/common/configmanager.cpp b/config/src/vespa/config/common/configmanager.cpp
index a73e8caf206..f15880fea01 100644
--- a/config/src/vespa/config/common/configmanager.cpp
+++ b/config/src/vespa/config/common/configmanager.cpp
@@ -3,9 +3,8 @@
 #include "exceptions.h"
 #include "configholder.h"
 #include <vespa/vespalib/util/atomic.h>
-#include <memory>
 #include <thread>
-#include <chrono>
+#include <sstream>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".config.common.configmanager");
diff --git a/config/src/vespa/config/common/configparser.cpp b/config/src/vespa/config/common/configparser.cpp
index fde55f257dc..e0a0b0138b9 100644
--- a/config/src/vespa/config/common/configparser.cpp
+++ b/config/src/vespa/config/common/configparser.cpp
@@ -1,9 +1,8 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-#include <vespa/config/common/configparser.h>
-#include <vespa/vespalib/stllike/string.h>
-#include <vespa/vespalib/stllike/asciistream.h>
-#include <stdio.h>
+
+#include "configparser.h"
 #include "misc.h"
+#include <vespa/vespalib/stllike/asciistream.h>
 
 namespace config {
 
diff --git a/config/src/vespa/config/common/configparser.h b/config/src/vespa/config/common/configparser.h
index 3b1185a48fa..613dfc33d94 100644
--- a/config/src/vespa/config/common/configparser.h
+++ b/config/src/vespa/config/common/configparser.h
@@ -2,12 +2,12 @@
 #pragma once
 
 #include <vespa/config/common/exceptions.h>
+#include <vespa/vespalib/util/stringfmt.h>
 #include <map>
 #include <set>
 #include <vector>
-#include <errno.h>
-#include <stdint.h>
-#include <vespa/vespalib/stllike/string.h>
+#include <cerrno>
+#include <cstdint>
 
 namespace config {
 
diff --git a/config/src/vespa/config/common/exceptions.cpp b/config/src/vespa/config/common/exceptions.cpp
index d19c27b5999..b5cbe7f0e06 100644
--- a/config/src/vespa/config/common/exceptions.cpp
+++ b/config/src/vespa/config/common/exceptions.cpp
@@ -1,6 +1,7 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "exceptions.h"
+#include <vespa/vespalib/util/stringfmt.h>
 
 namespace config {
 
diff --git a/config/src/vespa/config/frt/protocol.cpp b/config/src/vespa/config/frt/protocol.cpp
index ccc56fd51b9..b089af71782 100644
--- a/config/src/vespa/config/frt/protocol.cpp
+++ b/config/src/vespa/config/frt/protocol.cpp
@@ -3,6 +3,7 @@
 #include <lz4.h>
 #include <vespa/vespalib/util/stringfmt.h>
 #include <vespa/vespalib/data/slime/slime.h>
+#include <sstream>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".config.frt.protocol");
diff --git a/config/src/vespa/config/set/configsetsource.cpp b/config/src/vespa/config/set/configsetsource.cpp
index c62452a6415..75dcd3d3f65 100644
--- a/config/src/vespa/config/set/configsetsource.cpp
+++ b/config/src/vespa/config/set/configsetsource.cpp
@@ -2,9 +2,8 @@
 
 #include "configsetsource.h"
 #include <vespa/config/print/asciiconfigwriter.h>
-#include <vespa/config/common/misc.h>
 #include <vespa/config/common/exceptions.h>
-#include <vespa/vespalib/stllike/asciistream.h>
+
 #include <vespa/log/log.h>
 LOG_SETUP(".config.set.configsetsource");
 
@@ -48,7 +47,7 @@ ConfigSetSource::getConfig()
 void
 ConfigSetSource::reload(int64_t generation)
 {
-    LOG(debug, "Running update with generation(%" PRId64 ")", generation);
+    LOG(debug, "Running update with generation(%zd)", generation);
     _generation = generation;
 }
 
diff --git a/configgen/src/main/java/com/yahoo/config/codegen/CppClassBuilder.java b/configgen/src/main/java/com/yahoo/config/codegen/CppClassBuilder.java
index 997416e5bfc..49f5275c7fe 100644
--- a/configgen/src/main/java/com/yahoo/config/codegen/CppClassBuilder.java
+++ b/configgen/src/main/java/com/yahoo/config/codegen/CppClassBuilder.java
@@ -8,10 +8,12 @@ import java.io.IOException;
 import java.io.FileReader;
 import java.io.StringWriter;
 import java.io.Writer;
-import java.util.Collections;
-import java.util.HashMap;
 import java.util.Map;
+import java.util.HashMap;
+import java.util.Collections;
+import java.util.Arrays;
 import java.util.StringTokenizer;
+import java.util.stream.Collectors;
 
 
 /**
@@ -189,15 +191,19 @@ public class CppClassBuilder implements ClassBuilder {
     }
 
     void writeNameSpaceBegin(Writer w, String [] namespaceList) throws IOException {
-        for (int i = 0; i < namespaceList.length; i++) {
-            w.write("namespace " + namespaceList[i] + " {\n\n");
-        }
+        w.write("namespace ");
+        w.write(getNestedNameSpace(namespaceList));
+        w.write(" {\n");
+    }
+
+    String getNestedNameSpace(String [] namespaceList) {
+        return Arrays.stream(namespaceList).map(String::toString).collect(Collectors.joining("::"));
     }
 
     void writeNameSpaceEnd(Writer w, String [] namespaceList) throws IOException {
-        for (int i = 0; i < namespaceList.length; i++) {
-            w.write("} // namespace " + namespaceList[i] + "\n\n");
-        }
+        w.write("} // namespace ");
+        w.write(getNestedNameSpace(namespaceList));
+        w.write("\n");
     }
 
     void writeHeaderHeader(Writer w, CNode root) throws IOException {
@@ -229,17 +235,21 @@ public class CppClassBuilder implements ClassBuilder {
                 + "#ifndef CLOUD_CONFIG_" + defineName + "_H\n"
                 + "#define CLOUD_CONFIG_" + defineName + "_H\n"
                 + "\n"
-                + "#include <vespa/config/common/configvalue.h>\n"
-                + "#include <vespa/config/configgen/configpayload.h>\n"
                 + "#include <vespa/config/configgen/configinstance.h>\n"
-                + "#include <vespa/config/print/configdatabuffer.h>\n"
                 + "#include <vespa/vespalib/stllike/string.h>\n"
                 + "#include <vector>\n"
                 + "#include <map>\n"
                 + "\n");
+        w.write("namespace config {\n");
+        w.write("    class ConfigValue;\n");
+        w.write("    class ConfigPayload;\n");
+        w.write("}\n\n");
+        w.write("namespace vespalib::slime {\n");
+        w.write("    class Inspector;\n");
+        w.write("    class Cursor;\n");
+        w.write("}\n\n");
         writeNameSpaceBegin(w, namespaceList);
-        w.write("\n");
-        w.write("namespace internal {\n\n");
+        w.write("\nnamespace internal {\n\n");
         w.write(""
                 + "/**\n"
                 + " * This class contains the config. DO NOT USE THIS CLASS DIRECTLY. Use the typedeffed\n"
@@ -551,17 +561,18 @@ public class CppClassBuilder implements ClassBuilder {
             w.write("#include <" + subdir + "/" + getFileName(root, "h") + ">");
         }
         w.write("\n");
+        w.write("#include <vespa/config/common/configvalue.h>\n");
+        w.write("#include <vespa/config/configgen/configpayload.h>\n");
+        w.write("#include <vespa/config/print/configdatabuffer.h>\n");
         w.write("#include <vespa/config/common/configparser.h>\n");
         w.write("#include <vespa/config/configgen/vector_inserter.h>\n");
         w.write("#include <vespa/config/configgen/map_inserter.h>\n");
         w.write("#include <vespa/vespalib/data/slime/convenience.h>\n");
         w.write("#include <vespa/vespalib/data/slime/slime.h>\n");
         w.write("#include <vespa/vespalib/stllike/asciistream.h>\n");
-        w.write("#include <set>\n");
-        w.write("\n\n");
-        writeNameSpaceBegin(w, generateCppNameSpace(root));
         w.write("\n");
-        w.write("namespace internal {\n\n");
+        writeNameSpaceBegin(w, generateCppNameSpace(root));
+        w.write("\nnamespace internal {\n\n");
         w.write("using ::config::ConfigParser;\n");
         w.write("using ::config::InvalidConfigException;\n");
         w.write("using ::config::ConfigInstance;\n");
diff --git a/configgen/src/main/scala/com/yahoo/config/codegen/ConfigGenerator.scala b/configgen/src/main/scala/com/yahoo/config/codegen/ConfigGenerator.scala
index 716f2a60c33..533b9b2be23 100644
--- a/configgen/src/main/scala/com/yahoo/config/codegen/ConfigGenerator.scala
+++ b/configgen/src/main/scala/com/yahoo/config/codegen/ConfigGenerator.scala
@@ -180,8 +180,14 @@ object ConfigGenerator {
       }
     }
 
-    // TODO: The default ctor can be removed if the config library uses builders to set values from payload, but ...
-    //       a default ctor is also needed for all innerArrays, because of InnerNodeVector.createNew()
+    // TODO: The default ctor is possibly still needed for innerArrays, because of InnerNodeVector.createNew()
+    def defaultConstructorIfArray = {
+      inner match {
+        case array: InnerCNode if inner.isArray => defaultConstructor
+        case scalar: InnerCNode => ""
+      }
+    }
+
     def defaultConstructor = {
       // TODO @link gives javadoc warnings, although the syntax seems to be valid
       //def link = "{@link " + {nodeClass(inner)} + "#" + {nodeClass(inner)} + "(Builder)}"
@@ -202,7 +208,7 @@ object ConfigGenerator {
 
     // TODO: merge these two constructors into one when the config library uses builders to set values from payload.
     <code>
-      |{defaultConstructor}
+      |{defaultConstructorIfArray}
       |
       |public {nodeClass(inner)}(Builder builder) {{
       |  this(builder, true);
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
index 3e132137468..229f24152c3 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
@@ -10,6 +10,7 @@ import com.yahoo.config.application.api.ApplicationMetaData;
 import com.yahoo.config.application.api.DeployLogger;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.Deployer;
 import com.yahoo.config.provision.HostFilter;
 import com.yahoo.config.provision.Provisioner;
 import com.yahoo.config.provision.TenantName;
@@ -43,8 +44,12 @@ import java.io.File;
 import java.net.URI;
 import java.time.Clock;
 import java.time.Duration;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
 import java.util.logging.Logger;
 
 /**
@@ -66,6 +71,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
     private final HttpProxy httpProxy;
     private final Clock clock;
     private final DeployLogger logger = new SilentDeployLogger();
+    private final ConfigserverConfig configserverConfig;
     private final Environment environment;
 
     @Inject
@@ -83,6 +89,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
         this.convergeChecker = applicationConvergenceChecker;
         this.httpProxy = httpProxy;
         this.clock = Clock.systemUTC();
+        this.configserverConfig = configserverConfig;
         this.environment = Environment.from(configserverConfig.environment());
     }
 
@@ -327,6 +334,24 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
         return session.getSessionId();
     }
 
+    void redeployAllApplications(Deployer deployer) {
+        ExecutorService deploymentExecutor = Executors.newFixedThreadPool(configserverConfig.numParallelTenantLoaders());
+        tenants.getAllTenants().forEach(tenant -> listApplicationIds(tenant)
+                .forEach(applicationId -> redeployApplication(applicationId, deployer, deploymentExecutor)));
+    }
+
+    private void redeployApplication(ApplicationId applicationId, Deployer deployer, ExecutorService deploymentExecutor) {
+        log.log(LogLevel.DEBUG, () -> "Redeploying " + applicationId);
+        deployer.deployFromLocalActive(applicationId, Duration.ofMinutes(30))
+                .ifPresent(deployment -> deploymentExecutor.execute(() -> {
+                    try {
+                        deployment.activate();
+                    } catch (RuntimeException e) {
+                        log.log(LogLevel.ERROR, "Redeploying " + applicationId + " failed", e);
+                    }
+                }));
+    }
+
     public ApplicationFile getApplicationFileFromSession(TenantName tenantName, long sessionId, String path, LocalSession.Mode mode) {
         Tenant tenant = tenants.getTenant(tenantName);
         return getLocalSession(tenant, sessionId).getApplicationFile(Path.fromString(path), mode);
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java
index 1af101c074e..e0def2b0e4b 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java
@@ -6,7 +6,6 @@ import com.yahoo.component.AbstractComponent;
 import com.yahoo.config.provision.Deployer;
 import com.yahoo.log.LogLevel;
 import com.yahoo.vespa.config.server.rpc.RpcServer;
-import com.yahoo.vespa.config.server.tenant.Tenants;
 import com.yahoo.vespa.config.server.version.VersionState;
 
 /**
@@ -19,7 +18,7 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable
 
     private static final java.util.logging.Logger log = java.util.logging.Logger.getLogger(ConfigServerBootstrap.class.getName());
 
-    private final Tenants tenants;
+    private final ApplicationRepository applicationRepository;
     private final RpcServer server;
     private final Thread serverThread;
     private final Deployer deployer;
@@ -29,8 +28,9 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable
     // added to the rpc server before it starts answering rpc requests.
     @SuppressWarnings("UnusedParameters")
     @Inject
-    public ConfigServerBootstrap(Tenants tenants, RpcServer server, Deployer deployer, VersionState versionState) {
-        this.tenants = tenants;
+    public ConfigServerBootstrap(ApplicationRepository applicationRepository, RpcServer server,
+                                 Deployer deployer, VersionState versionState) {
+        this.applicationRepository = applicationRepository;
         this.server = server;
         this.deployer = deployer;
         this.versionState = versionState;
@@ -52,8 +52,9 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable
     @Override
     public void run() {
         if (versionState.isUpgraded()) {
-            log.log(LogLevel.INFO, "Configserver upgraded from " + versionState.storedVersion() + " to " + versionState.currentVersion() + ". Redeploying all applications");
-            tenants.redeployApplications(deployer);
+            log.log(LogLevel.INFO, "Configserver upgraded from " + versionState.storedVersion() + " to "
+                    + versionState.currentVersion() + ". Redeploying all applications");
+            applicationRepository.redeployAllApplications(deployer);
             log.log(LogLevel.INFO, "All applications redeployed");
         }
         versionState.saveNewVersion();
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListTenantsHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListTenantsHandler.java
index 84df1834712..28591f8f198 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListTenantsHandler.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListTenantsHandler.java
@@ -27,7 +27,7 @@ public class ListTenantsHandler extends HttpHandler {
 
     @Override
     protected HttpResponse handleGET(HttpRequest request) {
-        return new ListTenantsResponse(tenants.getAllTenants());
+        return new ListTenantsResponse(tenants.getAllTenantNames());
     }
 
 }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/Tenant.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/Tenant.java
index e03de29a9e4..b430b78038b 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/Tenant.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/Tenant.java
@@ -163,21 +163,5 @@ public class Tenant implements TenantHandlerProvider {
         curator.delete(path);
     }
 
-    public void redeployApplications(Deployer deployer) {
-        // TODO: Configurable timeout
-        applicationRepo.listApplications().forEach(applicationId -> redeployApplication(applicationId, deployer));
-    }
-
-    private void redeployApplication(ApplicationId applicationId, Deployer deployer) {
-        try {
-            log.log(LogLevel.DEBUG, "Redeploying " + applicationId);
-            deployer.deployFromLocalActive(applicationId, Duration.ofMinutes(30))
-                    .ifPresent(deployment -> {
-                        deployment.prepare();
-                        deployment.activate();
-                    });
-        } catch (Exception e) {
-            log.log(LogLevel.ERROR, "Redeploying " + applicationId + " failed", e);
-        }
-    }
+
 }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/Tenants.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/Tenants.java
index e0f470001fa..cc21616050a 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/Tenants.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/Tenants.java
@@ -5,7 +5,6 @@ import com.google.common.collect.ImmutableSet;
 import com.google.inject.Inject;
 import com.yahoo.concurrent.ThreadFactoryFactory;
 import com.yahoo.config.provision.ApplicationId;
-import com.yahoo.config.provision.Deployer;
 import com.yahoo.config.provision.TenantName;
 import com.yahoo.log.LogLevel;
 import com.yahoo.path.Path;
@@ -87,7 +86,7 @@ public class Tenants implements ConnectionStateListener, PathChildrenCacheListen
         // Note: unit tests may want to use the constructor below to avoid setting watch by calling readTenants().
         this.globalComponentRegistry = globalComponentRegistry;
         this.curator = globalComponentRegistry.getCurator();
-        metricUpdater = metrics.getOrCreateMetricUpdater(Collections.<String, String>emptyMap());
+        metricUpdater = metrics.getOrCreateMetricUpdater(Collections.emptyMap());
         this.tenantListeners.add(globalComponentRegistry.getTenantListener());
         curator.framework().getConnectionStateListenable().addListener(this);
 
@@ -95,19 +94,13 @@ public class Tenants implements ConnectionStateListener, PathChildrenCacheListen
         createSystemTenants();
         curator.create(vespaPath);
 
-        this.directoryCache = globalComponentRegistry.getCurator().createDirectoryCache(tenantsPath.getAbsolute(), false, false, pathChildrenExecutor);
+        this.directoryCache = curator.createDirectoryCache(tenantsPath.getAbsolute(), false, false, pathChildrenExecutor);
         directoryCache.start();
         directoryCache.addListener(this);
         tenantsChanged(readTenants());
         notifyTenantsLoaded();
     }
 
-    private void notifyTenantsLoaded() {
-        for (TenantListener tenantListener : tenantListeners) {
-            tenantListener.onTenantsLoaded();
-        }
-    }
-
     /**
      * New instance containing the given tenants. This will not watch in ZooKeeper. 
      * @param globalComponentRegistry a {@link com.yahoo.vespa.config.server.GlobalComponentRegistry} instance
@@ -124,6 +117,12 @@ public class Tenants implements ConnectionStateListener, PathChildrenCacheListen
         this.tenants.putAll(addTenants(tenants));
     }
 
+    private void notifyTenantsLoaded() {
+        for (TenantListener tenantListener : tenantListeners) {
+            tenantListener.onTenantsLoaded();
+        }
+    }
+
     // Pre-condition: tenants path needs to exist in zk
     private LinkedHashMap<TenantName, Tenant> addTenants(Collection<Tenant> newTenants) {
         LinkedHashMap<TenantName, Tenant> sessionTenants = new LinkedHashMap<>();
@@ -326,18 +325,6 @@ public class Tenants implements ConnectionStateListener, PathChildrenCacheListen
         pathChildrenExecutor.shutdown();
     }
 
-    // TODO: Deploy applications in parallel (with throttling)
-    public void redeployApplications(Deployer deployer) {
-        Set<Tenant> allTenants = ImmutableSet.copyOf(tenants.values());
-        int totalNumberOfApplications = allTenants.stream()
-                .mapToInt(tenant -> tenant.getApplicationRepo().listApplications().size()).sum();
-        int applicationsRedeployed = 0;
-        for (Tenant tenant : allTenants) {
-            tenant.redeployApplications(deployer);
-            applicationsRedeployed += redeployProgress(tenant, applicationsRedeployed, totalNumberOfApplications);
-        }
-    }
-
     public boolean checkThatTenantExists(TenantName tenant) {
         return tenants.containsKey(tenant);
     }
@@ -346,16 +333,12 @@ public class Tenants implements ConnectionStateListener, PathChildrenCacheListen
         return tenants.get(tenantName);
     }
 
-    public Set<TenantName> getAllTenants() {
+    public Set<TenantName> getAllTenantNames() {
         return ImmutableSet.copyOf(tenants.keySet());
     }
 
-    private static int redeployProgress(Tenant tenant, int applicationsRedeployed, int totalNumberOfApplications) {
-        int size = tenant.getApplicationRepo().listApplications().size();
-        if (size > 0) {
-            log.log(LogLevel.INFO, String.format("Redeployed %s of %s applications", applicationsRedeployed + size, totalNumberOfApplications));
-        }
-        return size;
+    public Collection<Tenant> getAllTenants() {
+        return ImmutableSet.copyOf(tenants.values());
     }
 
     /**
diff --git a/configserver/src/main/resources/logd/logd.cfg b/configserver/src/main/resources/logd/logd.cfg
index a4677deb3f6..137f3742ef2 100644
--- a/configserver/src/main/resources/logd/logd.cfg
+++ b/configserver/src/main/resources/logd/logd.cfg
@@ -7,3 +7,4 @@ loglevel.info.forward false
 loglevel.event.forward false
 loglevel.debug.forward false
 loglevel.spam.forward false
+remove.totalmegabytes 5000
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java
index 5fbe4fba64d..2443748a99c 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java
@@ -2,14 +2,27 @@
 package com.yahoo.vespa.config.server;
 
 import com.yahoo.cloud.config.ConfigserverConfig;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.ApplicationName;
+import com.yahoo.config.provision.InstanceName;
 import com.yahoo.config.provision.TenantName;
 import com.yahoo.io.IOUtils;
+import com.yahoo.vespa.config.server.application.ApplicationConvergenceChecker;
+import com.yahoo.vespa.config.server.application.HttpProxy;
+import com.yahoo.vespa.config.server.application.LogServerLogGrabber;
+import com.yahoo.vespa.config.server.deploy.MockDeployer;
 import com.yahoo.vespa.config.server.host.HostRegistries;
+import com.yahoo.vespa.config.server.http.SimpleHttpFetcher;
+import com.yahoo.vespa.config.server.http.v2.SessionActiveHandlerTest;
 import com.yahoo.vespa.config.server.monitoring.Metrics;
+import com.yahoo.vespa.config.server.provision.HostProvisionerProvider;
 import com.yahoo.vespa.config.server.rpc.UncompressedConfigResponseFactory;
+import com.yahoo.vespa.config.server.tenant.Tenant;
 import com.yahoo.vespa.config.server.tenant.TenantRequestHandler;
 import com.yahoo.vespa.config.server.tenant.TestWithTenant;
 import com.yahoo.vespa.config.server.version.VersionState;
+import org.hamcrest.core.Is;
+import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
@@ -29,9 +42,28 @@ import static org.junit.Assert.assertTrue;
  * @since 5.1
  */
 public class ConfigServerBootstrapTest extends TestWithTenant {
+    private final TenantName tenant1 = TenantName.from("tenant1");
+    private final TenantName tenant2 = TenantName.from("tenant2");
+
+    private ApplicationRepository applicationRepository;
+
     @Rule
     public TemporaryFolder temporaryFolder = new TemporaryFolder();
 
+    @Before
+    public void setup() {
+        tenants.writeTenantPath(tenant1);
+        tenants.writeTenantPath(tenant2);
+
+        applicationRepository = new ApplicationRepository(tenants,
+                                                          HostProvisionerProvider.withProvisioner(new SessionActiveHandlerTest.MockProvisioner()),
+                                                          curator,
+                                                          new LogServerLogGrabber(),
+                                                          new ApplicationConvergenceChecker(),
+                                                          new HttpProxy(new SimpleHttpFetcher()),
+                                                          new ConfigserverConfig(new ConfigserverConfig.Builder()));
+    }
+
     @Test
     public void testConfigServerBootstrap() throws Exception {
         File versionFile = temporaryFolder.newFile();
@@ -43,7 +75,7 @@ public class ConfigServerBootstrapTest extends TestWithTenant {
         assertFalse(myServer.stopped);
         VersionState versionState = new VersionState(versionFile);
         assertTrue(versionState.isUpgraded());
-        ConfigServerBootstrap bootstrap = new ConfigServerBootstrap(tenants, rpc, (application, timeout) -> Optional.empty(), versionState);
+        ConfigServerBootstrap bootstrap = new ConfigServerBootstrap(applicationRepository, rpc, (application, timeout) -> Optional.empty(), versionState);
         waitUntilStarted(rpc, 60000);
         assertFalse(versionState.isUpgraded());
         assertThat(versionState.currentVersion(), is(versionState.storedVersion()));
@@ -55,6 +87,16 @@ public class ConfigServerBootstrapTest extends TestWithTenant {
         assertTrue(rpc.stopped);
     }
 
+    @Test
+    public void testTenantRedeployment() throws Exception {
+        MockDeployer deployer = new MockDeployer();
+        Tenant tenant = tenants.getTenant(tenant1);
+        ApplicationId id = ApplicationId.from(tenant1, ApplicationName.defaultName(), InstanceName.defaultName());
+        tenant.getApplicationRepo().createPutApplicationTransaction(id, 3).commit();
+        applicationRepository.redeployAllApplications(deployer);
+        assertThat(deployer.lastDeployed, Is.is(id));
+    }
+
     private void waitUntilStarted(MockRpc server, long timeout) throws InterruptedException {
         long start = System.currentTimeMillis();
         while ((System.currentTimeMillis() - start) < timeout) {
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantsTestCase.java b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantsTestCase.java
index 31e8d545b7a..a2b4335d4ac 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantsTestCase.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantsTestCase.java
@@ -37,7 +37,7 @@ import static org.junit.Assert.fail;
 
 public class TenantsTestCase extends TestWithCurator {
     private Tenants tenants;
-    TestComponentRegistry globalComponentRegistry;
+    private TestComponentRegistry globalComponentRegistry;
     private TenantRequestHandlerTest.MockReloadListener listener;
     private MockTenantListener tenantListener;
     private final TenantName tenant1 = TenantName.from("tenant1");
@@ -80,19 +80,19 @@ public class TenantsTestCase extends TestWithCurator {
     @Test
     public void testTenantListenersNotified() throws Exception {
         tenants.writeTenantPath(tenant3);
-        assertThat("tenant3 not the last created tenant. Tenants: " + tenants.getAllTenants() + ", /config/v2/tenants: " + readZKChildren("/config/v2/tenants"), tenantListener.tenantCreatedName, is(tenant3));
+        assertThat("tenant3 not the last created tenant. Tenants: " + tenants.getAllTenantNames() + ", /config/v2/tenants: " + readZKChildren("/config/v2/tenants"), tenantListener.tenantCreatedName, is(tenant3));
         tenants.deleteTenant(tenant2);
-        assertFalse(tenants.getAllTenants().contains(tenant2));
+        assertFalse(tenants.getAllTenantNames().contains(tenant2));
         assertThat(tenantListener.tenantDeletedName, is(tenant2));
     }
 
     @Test
     public void testAddTenant() throws Exception {
-        Set<TenantName> allTenants = tenants.getAllTenants();
+        Set<TenantName> allTenants = tenants.getAllTenantNames();
         assertTrue(allTenants.contains(tenant1));
         assertTrue(allTenants.contains(tenant2));
         tenants.writeTenantPath(tenant3);
-        allTenants = tenants.getAllTenants();
+        allTenants = tenants.getAllTenantNames();
         assertTrue(allTenants.contains(tenant1));
         assertTrue(allTenants.contains(tenant2));
         assertTrue(allTenants.contains(tenant3));
@@ -108,7 +108,7 @@ public class TenantsTestCase extends TestWithCurator {
     public void testRemove() throws Exception {
         assertNotNull(globalComponentRegistry.getCurator().framework().checkExists().forPath(tenants.tenantZkPath(tenant1)));
         tenants.deleteTenant(tenant1);
-        assertFalse(tenants.getAllTenants().contains(tenant1));
+        assertFalse(tenants.getAllTenantNames().contains(tenant1));
     }
     
     @Test
@@ -120,14 +120,14 @@ public class TenantsTestCase extends TestWithCurator {
         newTenants.add(tenant2);
         newTenants.add(defaultTenant);
         tenants.tenantsChanged(newTenants);
-        Set<TenantName> allTenants = tenants.getAllTenants();
+        Set<TenantName> allTenants = tenants.getAllTenantNames();
         assertTrue(allTenants.contains(tenant2));
         assertEquals("default", defaultTenant.value());
         assertTrue(allTenants.contains(defaultTenant));
         assertFalse(allTenants.contains(tenant1));
         newTenants.clear();
         tenants.tenantsChanged(newTenants);
-        allTenants = tenants.getAllTenants();
+        allTenants = tenants.getAllTenantNames();
         assertFalse(allTenants.contains(tenant1));
         assertFalse(allTenants.contains(tenant2));
         assertFalse(allTenants.contains(defaultTenant));
@@ -138,7 +138,7 @@ public class TenantsTestCase extends TestWithCurator {
         newTenants.add(foo);
         newTenants.add(bar);
         tenants.tenantsChanged(newTenants);
-        allTenants = tenants.getAllTenants();
+        allTenants = tenants.getAllTenantNames();
         assertTrue(allTenants.contains(tenant2));
         assertTrue(allTenants.contains(foo));
         assertTrue(allTenants.contains(bar));
@@ -149,13 +149,13 @@ public class TenantsTestCase extends TestWithCurator {
         TestComponentRegistry reg = new TestComponentRegistry.Builder().curator(curator).build();
         Tenants t = new Tenants(reg, Metrics.createTestMetrics());
         try {
-            assertTrue(t.getAllTenants().contains(TenantName.defaultName()));
+            assertTrue(t.getAllTenantNames().contains(TenantName.defaultName()));
             reg.getCurator().framework().create().forPath(tenants.tenantZkPath(TenantName.from("newTenant")));
             // Poll for the watcher to pick up the tenant from zk, and add it
             int tries=0;
             while(true) {
                 if (tries > 500) fail("Didn't react on watch");
-                if (t.getAllTenants().contains(TenantName.from("newTenant"))) {
+                if (t.getAllTenantNames().contains(TenantName.from("newTenant"))) {
                     return;
                 }
                 tries++;
@@ -166,14 +166,5 @@ public class TenantsTestCase extends TestWithCurator {
         }
     }
 
-    @Test
-    public void testTenantRedeployment() throws Exception {
-        MockDeployer deployer = new MockDeployer();
-        Tenant tenant = tenants.getTenant(tenant1);
-        ApplicationId id = ApplicationId.from(tenant1, ApplicationName.defaultName(), InstanceName.defaultName());
-        tenant.getApplicationRepo().createPutApplicationTransaction(id, 3).commit();
-        tenants.redeployApplications(deployer);
-        assertThat(deployer.lastDeployed, is(id));
-    }
 
 }
diff --git a/dist/post_install.sh b/dist/post_install.sh
index ddaa5f0ff00..a5cb48c2d9e 100755
--- a/dist/post_install.sh
+++ b/dist/post_install.sh
@@ -67,4 +67,39 @@ ln -s $PREFIX/lib/jars/orchestrator-jar-with-dependencies.jar $INSTALLPATH/conf/
 ln -s $PREFIX/lib/jars/node-repository-jar-with-dependencies.jar $INSTALLPATH/conf/configserver-app/components/node-repository.jar
 ln -s $PREFIX/lib/jars/zkfacade-jar-with-dependencies.jar $INSTALLPATH/conf/configserver-app/components/zkfacade.jar
 ln -s $PREFIX/conf/configserver-app/components $INSTALLPATH/lib/jars/config-models
-ln -s storaged-bin $INSTALLPATH/sbin/distributord-bin
+ln -s vespa-storaged-bin $INSTALLPATH/sbin/vespa-distributord-bin
+
+# Temporary when renaming binaries in fnet
+ln -s vespa-rpc-info $INSTALLPATH/bin/rpc_info
+ln -s vespa-rpc-invoke $INSTALLPATH/bin/rpc_invoke
+ln -s vespa-rpc-proxy $INSTALLPATH/bin/rpc_proxy
+
+# Temporary when renaming binaries in fsa
+ln -s vespa-fsadump $INSTALLPATH/bin/fsadump
+ln -s vespa-fsainfo $INSTALLPATH/bin/fsainfo
+ln -s vespa-makefsa $INSTALLPATH/bin/makefsa
+
+# Temporary when renaming binaries in searchcore
+ln -s vespa-proton-bin $INSTALLPATH/sbin/proton-bin
+ln -s vespa-fdispatch-bin $INSTALLPATH/sbin/fdispatch-bin
+ln -s vespa-verify-ranksetup-bin $INSTALLPATH/bin/verify_ranksetup-bin
+ln -s vespa-proton $INSTALLPATH/sbin/proton
+ln -s vespa-fdispatch $INSTALLPATH/sbin/fdispatch
+ln -s vespa-verify-ranksetup $INSTALLPATH/bin/verify_ranksetup
+
+# Temporary when renaming binaries in storageserver
+ln -s vespa-storaged-bin $INSTALLPATH/sbin/storaged-bin
+ln -s vespa-distributord-bin $INSTALLPATH/sbin/distributord-bin
+ln -s vespa-storaged $INSTALLPATH/sbin/storaged
+ln -s vespa-distributord $INSTALLPATH/sbin/distributord
+
+# Temporary when renaming binaries in vespalog
+ln -s vespa-log-conv $INSTALLPATH/bin/vlogconv
+ln -s vespa-logctl $INSTALLPATH/bin/logctl
+ln -s vespa-logfmt $INSTALLPATH/bin/logfmt
+ln -s vespa-log-replay $INSTALLPATH/bin/vlogreplay
+
+# Temporary when renaming binaries in memfilepersistence
+ln -s vespa-dump-slotfile $INSTALLPATH/bin/dumpslotfile
+ln -s vespa-vds-disktool-bin $INSTALLPATH/bin/vdsdisktool-bin
+ln -s vespa-vds-disktool $INSTALLPATH/bin/vdsdisktool
diff --git a/docker/Dockerfile.build b/docker/Dockerfile.build
index 75dee2bd0a7..a0b935093eb 100644
--- a/docker/Dockerfile.build
+++ b/docker/Dockerfile.build
@@ -25,6 +25,7 @@ RUN yum-config-manager --add-repo https://copr.fedorainfracloud.org/coprs/g/vesp
                    perl \
                    perl-Env \
                    perl-JSON \
+                   perl-IO-Socket-IP \
                    sudo \
                    vespa-boost-devel \
                    vespa-libtorrent-devel \
diff --git a/docker/Dockerfile.run b/docker/Dockerfile.run
index 0f57da4b111..5f0dd1fd0cc 100644
--- a/docker/Dockerfile.run
+++ b/docker/Dockerfile.run
@@ -15,6 +15,7 @@ RUN yum-config-manager --add-repo https://copr.fedorainfracloud.org/coprs/g/vesp
                    perl \
                    perl-Env \
                    perl-JSON \
+                   perl-IO-Socket-IP \
                    libatomic \
                    net-tools \
                    less \
diff --git a/docker/README.md b/docker/README.md
index 04c45470017..31766259365 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -6,17 +6,12 @@
 
 *On Linux, the default storage device is devicemapper with loopback device and max 10GB container size. This size is too small for a full build. Please see [here](http://www.projectatomic.io/blog/2016/03/daemon_option_basedevicesize/) and [here](http://www.projectatomic.io/blog/2015/06/notes-on-fedora-centos-and-docker-storage-drivers/) to overcome this limitation.*
 
+
 ## Building the Vespa RPM
 Execute ```./build-vespa.sh <Vespa version number>``` to build Vespa from this source code.
 
 The produced rpms will be available in this folder after compiliation.
 
-### Troubleshooting
-If the build fails, start from scratch: ```docker rmi -f vesparun vespabuild``` - then build again. Clean local docker if docker image disk full:
-- ```docker rm -v $(docker ps -a -q -f status=exited)```
-- ```docker rmi $(docker images -f "dangling=true" -q)```
-
-_Directory renamed before its status could be extracted_ can be caused by [1219](https://github.com/docker/for-mac/issues/1219)
 
 ## Running Vespa
 Execute ```./run-vespa.sh <Vespa version number>``` to start Vespa.
@@ -25,8 +20,18 @@ This will create a Docker image which has the rpms from the build step (or downl
 
 *On OS X, the container runs inside the Docker VM. Execute ```docker-machine ssh vespa-docker-machine``` to enter the VM. The services can also be reached directly from the host on the IP given by ```docker-machine ip vespa-docker-machine```*
 
+
 ## Building Vespa inside a Docker container
 Execute ```./enter-build-container.sh``` to enter the Vespa build environment inside a Docker container.
 
 The container is entered at the root of the Vespa source repository. Follow the build sections in [README.md](https://github.com/yahoo/vespa/blob/master/README.md) to build and test.
 
+
+## Troubleshooting
+- Use ```docker logs CONTAINER``` for output - useful if the commands above fail
+
+- If the build fails, start from scratch: ```docker rmi -f vesparun vespabuild``` - then build again. Clean local docker if docker image disk full:
+    - ```docker rm -v $(docker ps -a -q -f status=exited)```
+    - ```docker rmi $(docker images -f "dangling=true" -q)```
+
+- _Directory renamed before its status could be extracted_ can be caused by [1219](https://github.com/docker/for-mac/issues/1219) - workaround (from the issue): "It may be an overlay storage driver issue - you can add ```{"storage-driver":"aufs"}``` in the advanced daemon preferences pane and see if that makes a difference."
diff --git a/document/src/tests/arrayfieldvaluetest.cpp b/document/src/tests/arrayfieldvaluetest.cpp
index 7f31e0c3b3a..0535edc495b 100644
--- a/document/src/tests/arrayfieldvaluetest.cpp
+++ b/document/src/tests/arrayfieldvaluetest.cpp
@@ -4,6 +4,7 @@
 #include <vespa/document/serialization/vespadocumentdeserializer.h>
 #include <vespa/vdstestlib/cppunit/macros.h>
 #include <vespa/vespalib/objects/nbostream.h>
+#include <vespa/document/util/bytebuffer.h>
 
 using vespalib::nbostream;
 
diff --git a/document/src/tests/base/documentid_test.cpp b/document/src/tests/base/documentid_test.cpp
index adda311e01a..7f17685f84e 100644
--- a/document/src/tests/base/documentid_test.cpp
+++ b/document/src/tests/base/documentid_test.cpp
@@ -2,6 +2,7 @@
 // Unit tests for documentid.
 
 #include <vespa/document/base/documentid.h>
+#include <vespa/document/base/idstringexception.h>
 #include <vespa/vespalib/testkit/testapp.h>
 
 using namespace document;
diff --git a/document/src/tests/documentcalculatortestcase.cpp b/document/src/tests/documentcalculatortestcase.cpp
index a292302ad61..bc112c80a8c 100644
--- a/document/src/tests/documentcalculatortestcase.cpp
+++ b/document/src/tests/documentcalculatortestcase.cpp
@@ -9,6 +9,7 @@
 #include <vespa/document/fieldvalue/intfieldvalue.h>
 #include <vespa/document/fieldvalue/longfieldvalue.h>
 #include <vespa/document/fieldvalue/floatfieldvalue.h>
+#include <vespa/document/select/variablemap.h>
 #include <vespa/vespalib/util/exceptions.h>
 
 namespace document {
@@ -52,46 +53,42 @@ CPPUNIT_TEST_SUITE_REGISTRATION(DocumentCalculatorTest);
 
 void
 DocumentCalculatorTest::testConstant() {
-    DocumentCalculator::VariableMap variables;
+    auto variables = std::make_unique<select::VariableMap>();
     DocumentCalculator calc(getRepo(), "4.0");
 
-    Document doc(*_testRepo.getDocumentType("testdoctype1"),
-                 DocumentId("doc:test:foo"));
+    Document doc(*_testRepo.getDocumentType("testdoctype1"), DocumentId("doc:test:foo"));
     CPPUNIT_ASSERT_EQUAL(4.0, calc.evaluate(doc, std::move(variables)));
 }
 
 void
 DocumentCalculatorTest::testSimple() {
-    DocumentCalculator::VariableMap variables;
+    auto variables = std::make_unique<select::VariableMap>();
     DocumentCalculator calc(getRepo(), "(3 + 5) / 2");
 
-    Document doc(*_testRepo.getDocumentType("testdoctype1"),
-                 DocumentId("doc:test:foo"));
+    Document doc(*_testRepo.getDocumentType("testdoctype1"), DocumentId("doc:test:foo"));
     CPPUNIT_ASSERT_EQUAL(4.0, calc.evaluate(doc, std::move(variables)));
 }
 
 void
 DocumentCalculatorTest::testVariables() {
-    DocumentCalculator::VariableMap variables;
-    variables["x"] = 3.0;
-    variables["y"] = 5.0;
+    auto variables = std::make_unique<select::VariableMap>();
+    (*variables)["x"] = 3.0;
+    (*variables)["y"] = 5.0;
     DocumentCalculator calc(getRepo(), "($x + $y) / 2");
 
-    Document doc(*_testRepo.getDocumentType("testdoctype1"),
-                 DocumentId("doc:test:foo"));
+    Document doc(*_testRepo.getDocumentType("testdoctype1"), DocumentId("doc:test:foo"));
     CPPUNIT_ASSERT_EQUAL(4.0, calc.evaluate(doc, std::move(variables)));
 }
 
 void
 DocumentCalculatorTest::testFields() {
-    DocumentCalculator::VariableMap variables;
-    variables["x"] = 3.0;
-    variables["y"] = 5.0;
+    auto variables = std::make_unique<select::VariableMap>();
+    (*variables)["x"] = 3.0;
+    (*variables)["y"] = 5.0;
     DocumentCalculator calc(getRepo(), "(testdoctype1.headerval + testdoctype1"
                             ".hfloatval) / testdoctype1.headerlongval");
 
-    Document doc(*_testRepo.getDocumentType("testdoctype1"),
-                 DocumentId("doc:test:foo"));
+    Document doc(*_testRepo.getDocumentType("testdoctype1"), DocumentId("doc:test:foo"));
     doc.setValue(doc.getField("headerval"), IntFieldValue(5));
     doc.setValue(doc.getField("hfloatval"), FloatFieldValue(3.0));
     doc.setValue(doc.getField("headerlongval"), LongFieldValue(2));
@@ -100,14 +97,13 @@ DocumentCalculatorTest::testFields() {
 
 void
 DocumentCalculatorTest::testFieldsDivZero() {
-    DocumentCalculator::VariableMap variables;
-    variables["x"] = 3.0;
-    variables["y"] = 5.0;
+    auto variables = std::make_unique<select::VariableMap>();
+    (*variables)["x"] = 3.0;
+    (*variables)["y"] = 5.0;
     DocumentCalculator calc(getRepo(), "(testdoctype1.headerval + testdoctype1"
                             ".hfloatval) / testdoctype1.headerlongval");
 
-    Document doc(*_testRepo.getDocumentType("testdoctype1"),
-                 DocumentId("doc:test:foo"));
+    Document doc(*_testRepo.getDocumentType("testdoctype1"), DocumentId("doc:test:foo"));
     doc.setValue(doc.getField("headerval"), IntFieldValue(5));
     doc.setValue(doc.getField("hfloatval"), FloatFieldValue(3.0));
     doc.setValue(doc.getField("headerlongval"), LongFieldValue(0));
@@ -121,11 +117,10 @@ DocumentCalculatorTest::testFieldsDivZero() {
 
 void
 DocumentCalculatorTest::testDivideByZero() {
-    DocumentCalculator::VariableMap variables;
+    auto variables = std::make_unique<select::VariableMap>();
     DocumentCalculator calc(getRepo(), "(3 + 5) / 0");
 
-    Document doc(*_testRepo.getDocumentType("testdoctype1"),
-                 DocumentId("doc:test:foo"));
+    Document doc(*_testRepo.getDocumentType("testdoctype1"), DocumentId("doc:test:foo"));
     try {
         calc.evaluate(doc, std::move(variables));
         CPPUNIT_ASSERT(false);
@@ -136,11 +131,10 @@ DocumentCalculatorTest::testDivideByZero() {
 
 void
 DocumentCalculatorTest::testModByZero() {
-    DocumentCalculator::VariableMap variables;
+    auto variables = std::make_unique<select::VariableMap>();
     DocumentCalculator calc(getRepo(), "(3 + 5) % 0");
 
-    Document doc(*_testRepo.getDocumentType("testdoctype1"),
-                 DocumentId("doc:test:foo"));
+    Document doc(*_testRepo.getDocumentType("testdoctype1"), DocumentId("doc:test:foo"));
     try {
         calc.evaluate(doc, std::move(variables));
         CPPUNIT_ASSERT(false);
@@ -151,12 +145,11 @@ DocumentCalculatorTest::testModByZero() {
 
 void
 DocumentCalculatorTest::testFieldNotSet() {
-    DocumentCalculator::VariableMap variables;
+    auto variables = std::make_unique<select::VariableMap>();
     DocumentCalculator calc(getRepo(), "(testdoctype1.headerval + testdoctype1"
                             ".hfloatval) / testdoctype1.headerlongval");
 
-    Document doc(*_testRepo.getDocumentType("testdoctype1"),
-                 DocumentId("doc:test:foo"));
+    Document doc(*_testRepo.getDocumentType("testdoctype1"), DocumentId("doc:test:foo"));
     doc.setValue(doc.getField("hfloatval"), FloatFieldValue(3.0));
     doc.setValue(doc.getField("headerlongval"), LongFieldValue(2));
     try {
@@ -169,13 +162,12 @@ DocumentCalculatorTest::testFieldNotSet() {
 
 void
 DocumentCalculatorTest::testFieldNotFound() {
-    DocumentCalculator::VariableMap variables;
+    auto variables = std::make_unique<select::VariableMap>();
     DocumentCalculator calc(getRepo(),
                             "(testdoctype1.mynotfoundfield + testdoctype1"
                             ".hfloatval) / testdoctype1.headerlongval");
 
-    Document doc(*_testRepo.getDocumentType("testdoctype1"),
-                 DocumentId("doc:test:foo"));
+    Document doc(*_testRepo.getDocumentType("testdoctype1"), DocumentId("doc:test:foo"));
     doc.setValue(doc.getField("hfloatval"), FloatFieldValue(3.0));
     doc.setValue(doc.getField("headerlongval"), LongFieldValue(2));
     try {
@@ -188,11 +180,10 @@ DocumentCalculatorTest::testFieldNotFound() {
 
 void
 DocumentCalculatorTest::testByteSubtractionZeroResult() {
-    DocumentCalculator::VariableMap variables;
+    auto variables = std::make_unique<select::VariableMap>();
     DocumentCalculator calc(getRepo(), "testdoctype1.byteval - 3");
 
-    Document doc(*_testRepo.getDocumentType("testdoctype1"),
-                 DocumentId("doc:test:foo"));
+    Document doc(*_testRepo.getDocumentType("testdoctype1"), DocumentId("doc:test:foo"));
     doc.setValue(doc.getField("byteval"), ByteFieldValue(3));
     CPPUNIT_ASSERT_EQUAL(0.0, calc.evaluate(doc, std::move(variables)));
 }
diff --git a/document/src/tests/documentselectparsertest.cpp b/document/src/tests/documentselectparsertest.cpp
index 6957634cbde..7fb7703c04a 100644
--- a/document/src/tests/documentselectparsertest.cpp
+++ b/document/src/tests/documentselectparsertest.cpp
@@ -6,6 +6,7 @@
 #include <vespa/document/repo/configbuilder.h>
 #include <vespa/document/repo/documenttyperepo.h>
 #include <vespa/document/update/assignvalueupdate.h>
+#include <vespa/document/update/documentupdate.h>
 #include <vespa/document/base/testdocman.h>
 #include <vespa/document/select/parser.h>
 #include <vespa/document/select/visitor.h>
diff --git a/document/src/tests/documenttestcase.cpp b/document/src/tests/documenttestcase.cpp
index b100487a94e..1aa6f5fcaa2 100644
--- a/document/src/tests/documenttestcase.cpp
+++ b/document/src/tests/documenttestcase.cpp
@@ -5,12 +5,14 @@
 #include <vespa/vdstestlib/cppunit/macros.h>
 
 #include <vespa/document/datatype/annotationreferencedatatype.h>
+#include <vespa/document/fieldvalue/iteratorhandler.h>
 #include <vespa/document/repo/configbuilder.h>
 #include <vespa/document/serialization/vespadocumentdeserializer.h>
 #include <vespa/document/serialization/vespadocumentserializer.h>
 #include <vespa/vespalib/objects/nbostream.h>
 #include <vespa/vespalib/testkit/test_kit.h>
 #include <vespa/document/util/serializableexceptions.h>
+#include <vespa/document/util/bytebuffer.h>
 
 using vespalib::nbostream;
 
@@ -18,6 +20,8 @@ using namespace document::config_builder;
 
 namespace document {
 
+using namespace fieldvalue;
+
 struct DocumentTest : public CppUnit::TestFixture {
     void testTraversing();
     void testFieldPath();
@@ -75,7 +79,7 @@ void DocumentTest::testSizeOf()
     CPPUNIT_ASSERT_EQUAL(120ul, sizeof(Document));
     CPPUNIT_ASSERT_EQUAL(64ul, sizeof(StructFieldValue));
     CPPUNIT_ASSERT_EQUAL(16ul, sizeof(StructuredFieldValue));
-    CPPUNIT_ASSERT_EQUAL(120ul, sizeof(SerializableArray));
+    CPPUNIT_ASSERT_EQUAL(64ul, sizeof(SerializableArray));
 }
 
 void DocumentTest::testFieldPath()
@@ -104,7 +108,7 @@ void DocumentTest::testFieldPath()
     }
 }
 
-class Handler : public FieldValue::IteratorHandler {
+class Handler : public fieldvalue::IteratorHandler {
 public:
     Handler();
     ~Handler();
@@ -184,7 +188,7 @@ void DocumentTest::testTraversing()
                          std::string("<P<P<PP[PPP][<PP><PP>]>>>"));
 }
 
-class VariableIteratorHandler : public FieldValue::IteratorHandler {
+class VariableIteratorHandler : public IteratorHandler {
 public:
     VariableIteratorHandler();
     ~VariableIteratorHandler();
@@ -267,16 +271,16 @@ DocumentTest::testVariables()
 
 }
 
-class ModifyIteratorHandler : public FieldValue::IteratorHandler {
+class ModifyIteratorHandler : public IteratorHandler {
 public:
     ModificationStatus doModify(FieldValue& fv) override {
         StringFieldValue* sfv = dynamic_cast<StringFieldValue*>(&fv);
         if (sfv != NULL) {
             *sfv = std::string("newvalue");
-            return MODIFIED;
+            return ModificationStatus::MODIFIED;
         }
 
-        return NOT_MODIFIED;
+        return ModificationStatus::NOT_MODIFIED;
     };
 
     bool onComplex(const Content&) override {
diff --git a/document/src/tests/documentupdatetestcase.cpp b/document/src/tests/documentupdatetestcase.cpp
index baa777ceca1..e3cab489aab 100644
--- a/document/src/tests/documentupdatetestcase.cpp
+++ b/document/src/tests/documentupdatetestcase.cpp
@@ -24,6 +24,7 @@
 #include <vespa/eval/tensor/tensor_factory.h>
 #include <vespa/vespalib/testkit/testapp.h>
 #include <vespa/vespalib/objects/nbostream.h>
+#include <vespa/vespalib/util/exception.h>
 
 using namespace document::config_builder;
 using vespalib::tensor::Tensor;
diff --git a/document/src/tests/fieldpathupdatetestcase.cpp b/document/src/tests/fieldpathupdatetestcase.cpp
index 4a23f828921..2cf29e4cc53 100644
--- a/document/src/tests/fieldpathupdatetestcase.cpp
+++ b/document/src/tests/fieldpathupdatetestcase.cpp
@@ -1,6 +1,7 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 #include <vespa/document/base/testdocman.h>
 #include <vespa/document/fieldvalue/fieldvalues.h>
+#include <vespa/document/fieldvalue/iteratorhandler.h>
 #include <vespa/document/select/node.h>
 #include <vespa/vespalib/io/fileutil.h>
 #include <vespa/vdstestlib/cppunit/macros.h>
@@ -21,6 +22,8 @@ using namespace document::config_builder;
 
 namespace document {
 
+using namespace fieldvalue;
+
 struct FieldPathUpdateTestCase : public CppUnit::TestFixture {
     DocumentTypeRepo::SP _repo;
     DocumentType _foobar_type;
@@ -259,7 +262,7 @@ void testSerialize(const DocumentTypeRepo& repo, const DocumentUpdate& a) {
 
 struct TestFieldPathUpdate : FieldPathUpdate
 {
-    struct TestIteratorHandler : FieldValue::IteratorHandler
+    struct TestIteratorHandler : fieldvalue::IteratorHandler
     {
         TestIteratorHandler(std::string& str)
             : _str(str) {}
@@ -272,7 +275,7 @@ struct TestFieldPathUpdate : FieldPathUpdate
                 _str += ';';
             }
             _str += ss.str();
-            return NOT_MODIFIED;
+            return ModificationStatus::NOT_MODIFIED;
         }
 
         bool onComplex(const Content&) override { return false; }
@@ -288,9 +291,9 @@ struct TestFieldPathUpdate : FieldPathUpdate
 
     TestFieldPathUpdate(const TestFieldPathUpdate& other);
 
-    std::unique_ptr<FieldValue::IteratorHandler> getIteratorHandler(Document&) const override
+    std::unique_ptr<IteratorHandler> getIteratorHandler(Document&) const override
     {
-        return std::unique_ptr<FieldValue::IteratorHandler>(
+        return std::unique_ptr<IteratorHandler>(
                 new TestIteratorHandler(_str));
     }
 
diff --git a/document/src/tests/primitivefieldvaluetest.cpp b/document/src/tests/primitivefieldvaluetest.cpp
index 15fd6e00959..018408a2d02 100644
--- a/document/src/tests/primitivefieldvaluetest.cpp
+++ b/document/src/tests/primitivefieldvaluetest.cpp
@@ -4,6 +4,7 @@
 #include <vespa/document/serialization/vespadocumentdeserializer.h>
 #include <vespa/vdstestlib/cppunit/macros.h>
 #include <vespa/vespalib/objects/nbostream.h>
+#include <vespa/document/util/bytebuffer.h>
 
 using vespalib::nbostream;
 
diff --git a/document/src/tests/structfieldvaluetest.cpp b/document/src/tests/structfieldvaluetest.cpp
index d6e585e0987..1b2fa17a79a 100644
--- a/document/src/tests/structfieldvaluetest.cpp
+++ b/document/src/tests/structfieldvaluetest.cpp
@@ -1,10 +1,12 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include <vespa/document/fieldvalue/fieldvalues.h>
+#include <vespa/document/datatype/documenttype.h>
 #include <vespa/document/repo/configbuilder.h>
 #include <vespa/document/serialization/vespadocumentdeserializer.h>
 #include <vespa/vdstestlib/cppunit/macros.h>
 #include <vespa/vespalib/objects/nbostream.h>
+#include <vespa/document/util/bytebuffer.h>
 
 using vespalib::nbostream;
 using document::config_builder::Struct;
diff --git a/document/src/tests/testxml.cpp b/document/src/tests/testxml.cpp
index da328c7a63f..a9abae531fa 100644
--- a/document/src/tests/testxml.cpp
+++ b/document/src/tests/testxml.cpp
@@ -1,6 +1,5 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
-/* $Id$*/
 
 #include <vespa/document/base/testdocrepo.h>
 #include <cppunit/extensions/HelperMacros.h>
@@ -8,6 +7,7 @@
 #include <vespa/document/update/addvalueupdate.h>
 #include <vespa/document/update/assignvalueupdate.h>
 #include <vespa/document/update/removevalueupdate.h>
+#include <vespa/document/datatype/documenttype.h>
 #include <vespa/document/fieldvalue/fieldvalues.h>
 #include <vespa/vespalib/text/stringtokenizer.h>
 #include <vespa/vespalib/testkit/test_kit.h>
diff --git a/document/src/tests/weightedsetfieldvaluetest.cpp b/document/src/tests/weightedsetfieldvaluetest.cpp
index 81b98da297f..68b94cd2f16 100644
--- a/document/src/tests/weightedsetfieldvaluetest.cpp
+++ b/document/src/tests/weightedsetfieldvaluetest.cpp
@@ -4,6 +4,7 @@
 #include <vespa/document/serialization/vespadocumentdeserializer.h>
 #include <vespa/vdstestlib/cppunit/macros.h>
 #include <vespa/vespalib/objects/nbostream.h>
+#include <vespa/document/util/bytebuffer.h>
 
 using vespalib::nbostream;
 
diff --git a/document/src/vespa/document/annotation/spantree.cpp b/document/src/vespa/document/annotation/spantree.cpp
index 199040ea8f6..61c7ac48817 100644
--- a/document/src/vespa/document/annotation/spantree.cpp
+++ b/document/src/vespa/document/annotation/spantree.cpp
@@ -1,9 +1,7 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "spantree.h"
-#include "annotation.h"
 #include "spannode.h"
-#include <vespa/vespalib/stllike/string.h>
 #include <vespa/vespalib/stllike/asciistream.h>
 
 using std::unique_ptr;
diff --git a/document/src/vespa/document/annotation/spantree.h b/document/src/vespa/document/annotation/spantree.h
index d53e9002598..06558cf6887 100644
--- a/document/src/vespa/document/annotation/spantree.h
+++ b/document/src/vespa/document/annotation/spantree.h
@@ -4,6 +4,7 @@
 
 #include <vespa/document/annotation/annotation.h>
 #include <vector>
+#include <cassert>
 
 namespace document {
 class SpanNode;
diff --git a/document/src/vespa/document/base/documentcalculator.cpp b/document/src/vespa/document/base/documentcalculator.cpp
index 7a83d80764c..24afdee3bd1 100644
--- a/document/src/vespa/document/base/documentcalculator.cpp
+++ b/document/src/vespa/document/base/documentcalculator.cpp
@@ -5,6 +5,7 @@
 #include <vespa/document/select/compare.h>
 #include <vespa/document/select/parser.h>
 #include <vespa/document/select/valuenode.h>
+#include <vespa/document/select/variablemap.h>
 #include <vespa/vespalib/util/exceptions.h>
 
 namespace document {
@@ -21,13 +22,13 @@ DocumentCalculator::DocumentCalculator(
 DocumentCalculator::~DocumentCalculator() { }
 
 double
-DocumentCalculator::evaluate(const Document& doc, VariableMap && variables)
+DocumentCalculator::evaluate(const Document& doc, std::unique_ptr<select::VariableMap> variables)
 {
     select::Compare& compare(static_cast<select::Compare&>(*_selectionNode));
     const select::ValueNode& left = compare.getLeft();
 
     select::Context context(doc);
-    context._variables = std::move(variables);
+    context.setVariableMap(std::move(variables));
     std::unique_ptr<select::Value> value = left.getValue(context);
 
     select::NumberValue* num = dynamic_cast<select::NumberValue*>(value.get());
diff --git a/document/src/vespa/document/base/documentcalculator.h b/document/src/vespa/document/base/documentcalculator.h
index 7e1feaf5688..fc9eb73dea2 100644
--- a/document/src/vespa/document/base/documentcalculator.h
+++ b/document/src/vespa/document/base/documentcalculator.h
@@ -2,18 +2,18 @@
 #pragma once
 
 #include <vespa/document/select/node.h>
-#include <vespa/vespalib/stllike/hash_map.h>
 
 namespace document {
+
+namespace select { class VariableMap; }
+
 class DocumentTypeRepo;
 
 class DocumentCalculator {
 public:
-    using VariableMap = vespalib::hash_map<vespalib::string, double>;
-
     DocumentCalculator(const DocumentTypeRepo& repo, const vespalib::string& expression);
     ~DocumentCalculator();
-    double evaluate(const Document& doc, VariableMap && variables);
+    double evaluate(const Document& doc, std::unique_ptr<select::VariableMap> variables);
 
 private:
     std::unique_ptr<select::Node> _selectionNode;
diff --git a/document/src/vespa/document/base/exceptions.cpp b/document/src/vespa/document/base/exceptions.cpp
index 3e5c464b8b4..dbd3f2bd998 100644
--- a/document/src/vespa/document/base/exceptions.cpp
+++ b/document/src/vespa/document/base/exceptions.cpp
@@ -3,6 +3,7 @@
 #include "exceptions.h"
 #include <vespa/document/datatype/datatype.h>
 #include <vespa/document/fieldvalue/document.h>
+#include <vespa/vespalib/util/stringfmt.h>
 
 namespace document {
 
diff --git a/document/src/vespa/document/base/field.cpp b/document/src/vespa/document/base/field.cpp
index d62623b64b9..d00fd42081c 100644
--- a/document/src/vespa/document/base/field.cpp
+++ b/document/src/vespa/document/base/field.cpp
@@ -4,6 +4,7 @@
 #include <vespa/document/fieldvalue/fieldvalue.h>
 #include <vespa/vespalib/util/exceptions.h>
 #include <vespa/vespalib/stllike/asciistream.h>
+#include <vespa/vespalib/util/stringfmt.h>
 #include <vespa/vespalib/util/bobhash.h>
 
 namespace document {
diff --git a/document/src/vespa/document/base/forcelink.cpp b/document/src/vespa/document/base/forcelink.cpp
index b03433081d6..f17c8f582b3 100644
--- a/document/src/vespa/document/base/forcelink.cpp
+++ b/document/src/vespa/document/base/forcelink.cpp
@@ -3,6 +3,8 @@
 #include "forcelink.h"
 #include <vespa/document/update/updates.h>
 #include <vespa/document/fieldvalue/fieldvalues.h>
+#include <vespa/document/datatype/documenttype.h>
+
 
 namespace document {
 
diff --git a/document/src/vespa/document/base/globalid.cpp b/document/src/vespa/document/base/globalid.cpp
index 7fa22d27971..ab8736999aa 100644
--- a/document/src/vespa/document/base/globalid.cpp
+++ b/document/src/vespa/document/base/globalid.cpp
@@ -3,6 +3,7 @@
 #include <vespa/document/base/globalid.h>
 #include <vespa/vespalib/util/exceptions.h>
 #include <vespa/vespalib/stllike/asciistream.h>
+#include <vespa/vespalib/util/stringfmt.h>
 #include <vespa/vespalib/stllike/hash_set.hpp>
 #include <cassert>
 
diff --git a/document/src/vespa/document/base/idstring.cpp b/document/src/vespa/document/base/idstring.cpp
index f5f9f7377b5..4bdecf38996 100644
--- a/document/src/vespa/document/base/idstring.cpp
+++ b/document/src/vespa/document/base/idstring.cpp
@@ -1,8 +1,10 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "idstring.h"
+#include "idstringexception.h"
 #include <vespa/document/bucket/bucketid.h>
 #include <vespa/vespalib/util/md5.h>
+#include <vespa/vespalib/util/stringfmt.h>
 
 using vespalib::string;
 using vespalib::stringref;
diff --git a/document/src/vespa/document/base/idstring.h b/document/src/vespa/document/base/idstring.h
index f1a96974cc9..5e09b9060b8 100644
--- a/document/src/vespa/document/base/idstring.h
+++ b/document/src/vespa/document/base/idstring.h
@@ -1,29 +1,15 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-/**
- * @file idstring.h
- *
- * Contains the various URI schemes accepted in document identifiers.
- */
 
 #pragma once
 
-#include <vespa/vespalib/util/exception.h>
 #include <vespa/vespalib/util/memory.h>
 #include <vespa/vespalib/objects/cloneable.h>
 #include <vespa/vespalib/stllike/string.h>
+#include <cstdint>
 
 namespace document {
 
 /**
- * \class document::IdParseException
- * \ingroup base
- *
- * \brief Exception used to indicate failure to parse a %document identifier
- * URI.
- */
-VESPA_DEFINE_EXCEPTION(IdParseException, vespalib::Exception);
-
-/**
  * \class document::IdString
  * \ingroup base
  *
diff --git a/document/src/vespa/document/base/idstringexception.h b/document/src/vespa/document/base/idstringexception.h
new file mode 100644
index 00000000000..922876a8d6e
--- /dev/null
+++ b/document/src/vespa/document/base/idstringexception.h
@@ -0,0 +1,19 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/vespalib/util/exception.h>
+
+namespace document {
+
+/**
+ * \class document::IdParseException
+ * \ingroup base
+ *
+ * \brief Exception used to indicate failure to parse a %document identifier
+ * URI.
+ */
+
+VESPA_DEFINE_EXCEPTION(IdParseException, vespalib::Exception);
+
+}
diff --git a/document/src/vespa/document/bucket/CMakeLists.txt b/document/src/vespa/document/bucket/CMakeLists.txt
index 757884f0050..7dfeaa1920f 100644
--- a/document/src/vespa/document/bucket/CMakeLists.txt
+++ b/document/src/vespa/document/bucket/CMakeLists.txt
@@ -5,6 +5,7 @@ vespa_add_library(document_bucket OBJECT
     bucketdistribution.cpp
     bucketid.cpp
     bucketidfactory.cpp
+    bucketidlist.cpp
     bucketselector.cpp
     bucketspace.cpp
     DEPENDS
diff --git a/document/src/vespa/document/bucket/bucketdistribution.h b/document/src/vespa/document/bucket/bucketdistribution.h
index cc7232c81e6..aabc615bdda 100644
--- a/document/src/vespa/document/bucket/bucketdistribution.h
+++ b/document/src/vespa/document/bucket/bucketdistribution.h
@@ -12,8 +12,8 @@
  */
 #pragma once
 
+#include "bucketid.h"
 #include <vector>
-#include <vespa/document/bucket/bucketid.h>
 #include <vespa/vespalib/util/sync.h>
 
 namespace document {
diff --git a/document/src/vespa/document/bucket/bucketid.cpp b/document/src/vespa/document/bucket/bucketid.cpp
index 8554450fb22..c9efe7cd152 100644
--- a/document/src/vespa/document/bucket/bucketid.cpp
+++ b/document/src/vespa/document/bucket/bucketid.cpp
@@ -6,6 +6,7 @@
 #include <vespa/vespalib/objects/nbostream.h>
 #include <vespa/vespalib/stllike/asciistream.h>
 #include <vespa/vespalib/stllike/hash_set.hpp>
+#include <vespa/vespalib/util/stringfmt.h>
 
 using vespalib::nbostream;
 using vespalib::asciistream;
diff --git a/document/src/vespa/document/bucket/bucketid.h b/document/src/vespa/document/bucket/bucketid.h
index 63b513dd5a2..79da6c2d11b 100644
--- a/document/src/vespa/document/bucket/bucketid.h
+++ b/document/src/vespa/document/bucket/bucketid.h
@@ -22,7 +22,6 @@
 #pragma once
 
 #include <vespa/vespalib/stllike/string.h>
-#include <vespa/vespalib/util/array.h>
 
 namespace vespalib {
     class nbostream;
@@ -33,6 +32,8 @@ namespace document {
 
 extern const unsigned char reverseBitTable[256];
 
+namespace bucket { class BucketIdList; }
+
 class BucketId
 {
 public:
@@ -47,7 +48,7 @@ public:
      * typedef when needed we can alter this later with less code changes.
      */
     using Type = uint64_t;
-    using List = vespalib::Array<BucketId>;
+    using List = bucket::BucketIdList;
     /** Create an initially unset bucket id. */
     BucketId() : _id(0) {}
     /** Create a bucket id with the given raw unchecked content. */
diff --git a/document/src/vespa/document/bucket/bucketidlist.cpp b/document/src/vespa/document/bucket/bucketidlist.cpp
new file mode 100644
index 00000000000..6c9f7250b8e
--- /dev/null
+++ b/document/src/vespa/document/bucket/bucketidlist.cpp
@@ -0,0 +1,12 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "bucketidlist.h"
+
+namespace document::bucket {
+
+BucketIdList::BucketIdList() { }
+BucketIdList::BucketIdList(const BucketIdList & rhs) = default;
+BucketIdList & BucketIdList::operator = (const BucketIdList &) = default;
+BucketIdList::~BucketIdList() { }
+
+}
diff --git a/document/src/vespa/document/bucket/bucketidlist.h b/document/src/vespa/document/bucket/bucketidlist.h
new file mode 100644
index 00000000000..f53301d1b44
--- /dev/null
+++ b/document/src/vespa/document/bucket/bucketidlist.h
@@ -0,0 +1,22 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "bucketid.h"
+#include <vespa/vespalib/util/array.h>
+
+namespace document::bucket {
+
+using BucketIdListT = vespalib::Array<BucketId>;
+
+class BucketIdList : public BucketIdListT {
+public:
+    BucketIdList();
+    BucketIdList(BucketIdList && rhs) = default;
+    BucketIdList & operator = (BucketIdList &&) = default;
+    BucketIdList(const BucketIdList & rhs);
+    BucketIdList & operator = (const BucketIdList &);
+    ~BucketIdList();
+};
+
+}
diff --git a/document/src/vespa/document/bucket/bucketselector.cpp b/document/src/vespa/document/bucket/bucketselector.cpp
index 8b5e7ada91b..7eb3821e283 100644
--- a/document/src/vespa/document/bucket/bucketselector.cpp
+++ b/document/src/vespa/document/bucket/bucketselector.cpp
@@ -4,10 +4,11 @@
 #include "bucketidfactory.h"
 #include <vespa/document/base/documentid.h>
 #include <vespa/document/select/node.h>
-#include <vespa/document/select/valuenode.h>
+#include <vespa/document/select/valuenodes.h>
 #include <vespa/document/select/visitor.h>
 #include <vespa/document/select/branch.h>
 #include <vespa/document/select/compare.h>
+#include <vespa/vespalib/util/stringfmt.h>
 
 namespace document {
 
diff --git a/document/src/vespa/document/datatype/annotationreferencedatatype.cpp b/document/src/vespa/document/datatype/annotationreferencedatatype.cpp
index 2591cdbf37c..344fc31ed33 100644
--- a/document/src/vespa/document/datatype/annotationreferencedatatype.cpp
+++ b/document/src/vespa/document/datatype/annotationreferencedatatype.cpp
@@ -3,6 +3,7 @@
 #include "annotationreferencedatatype.h"
 #include <vespa/document/fieldvalue/annotationreferencefieldvalue.h>
 #include <ostream>
+#include <cassert>
 
 using std::unique_ptr;
 using std::ostream;
diff --git a/document/src/vespa/document/datatype/documenttype.cpp b/document/src/vespa/document/datatype/documenttype.cpp
index 883c4df8f9b..dcf28a9fabf 100644
--- a/document/src/vespa/document/datatype/documenttype.cpp
+++ b/document/src/vespa/document/datatype/documenttype.cpp
@@ -3,6 +3,7 @@
 #include <vespa/document/datatype/documenttype.h>
 #include <vespa/document/fieldvalue/document.h>
 #include <vespa/vespalib/util/exceptions.h>
+#include <vespa/vespalib/util/stringfmt.h>
 #include <iomanip>
 #include <vespa/log/log.h>
 
diff --git a/document/src/vespa/document/datatype/primitivedatatype.cpp b/document/src/vespa/document/datatype/primitivedatatype.cpp
index 821a22aad4a..afc32e291ea 100644
--- a/document/src/vespa/document/datatype/primitivedatatype.cpp
+++ b/document/src/vespa/document/datatype/primitivedatatype.cpp
@@ -3,6 +3,7 @@
 #include "primitivedatatype.h"
 #include <vespa/document/fieldvalue/fieldvalues.h>
 #include <vespa/vespalib/util/exceptions.h>
+#include <vespa/vespalib/util/stringfmt.h>
 #include <sstream>
 
 namespace document {
diff --git a/document/src/vespa/document/datatype/structdatatype.cpp b/document/src/vespa/document/datatype/structdatatype.cpp
index 6cf9e18656b..954bd9c742f 100644
--- a/document/src/vespa/document/datatype/structdatatype.cpp
+++ b/document/src/vespa/document/datatype/structdatatype.cpp
@@ -1,12 +1,14 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "structdatatype.h"
-
-#include <iomanip>
 #include <vespa/document/base/exceptions.h>
 #include <vespa/document/fieldvalue/structfieldvalue.h>
 #include <vespa/document/fieldvalue/document.h>
+#include <vespa/vespalib/util/stringfmt.h>
 #include <vespa/vespalib/stllike/hash_map.hpp>
+#include <iomanip>
+#include <cassert>
+
 #include <vespa/log/log.h>
 LOG_SETUP(".document.datatype.struct");
 
diff --git a/document/src/vespa/document/fieldset/fieldsets.cpp b/document/src/vespa/document/fieldset/fieldsets.cpp
index 4ac09820979..51166adec86 100644
--- a/document/src/vespa/document/fieldset/fieldsets.cpp
+++ b/document/src/vespa/document/fieldset/fieldsets.cpp
@@ -2,6 +2,7 @@
 
 #include "fieldsets.h"
 #include <vespa/document/fieldvalue/document.h>
+#include <vespa/document/datatype/documenttype.h>
 
 namespace document {
 
diff --git a/document/src/vespa/document/fieldvalue/CMakeLists.txt b/document/src/vespa/document/fieldvalue/CMakeLists.txt
index 8c76db54d5f..18ea4fc9368 100644
--- a/document/src/vespa/document/fieldvalue/CMakeLists.txt
+++ b/document/src/vespa/document/fieldvalue/CMakeLists.txt
@@ -10,6 +10,7 @@ vespa_add_library(document_fieldvalues OBJECT
     fieldvalue.cpp
     floatfieldvalue.cpp
     intfieldvalue.cpp
+    iteratorhandler.cpp
     literalfieldvalue.cpp
     longfieldvalue.cpp
     mapfieldvalue.cpp
@@ -22,6 +23,7 @@ vespa_add_library(document_fieldvalues OBJECT
     structfieldvalue.cpp
     structuredfieldvalue.cpp
     tensorfieldvalue.cpp
+    variablemap.cpp
     weightedsetfieldvalue.cpp
     referencefieldvalue.cpp
     DEPENDS
diff --git a/document/src/vespa/document/fieldvalue/annotationreferencefieldvalue.cpp b/document/src/vespa/document/fieldvalue/annotationreferencefieldvalue.cpp
index 9863ee1c18a..cd6d43d2f4d 100644
--- a/document/src/vespa/document/fieldvalue/annotationreferencefieldvalue.cpp
+++ b/document/src/vespa/document/fieldvalue/annotationreferencefieldvalue.cpp
@@ -1,6 +1,7 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "annotationreferencefieldvalue.h"
+#include <vespa/vespalib/util/xmlstream.h>
 #include <ostream>
 
 using std::ostream;
@@ -23,8 +24,7 @@ int AnnotationReferenceFieldValue::compare(const FieldValue &other) const {
     return (getDataType()->getId() - other.getDataType()->getId());
 }
 
-void AnnotationReferenceFieldValue::print(ostream &out, bool,
-                                          const string &) const {
+void AnnotationReferenceFieldValue::print(ostream &out, bool, const string &) const {
     out << "AnnotationReferenceFieldValue(n)";
 }
 
diff --git a/document/src/vespa/document/fieldvalue/arrayfieldvalue.cpp b/document/src/vespa/document/fieldvalue/arrayfieldvalue.cpp
index 866c82a63c9..b5b031d31da 100644
--- a/document/src/vespa/document/fieldvalue/arrayfieldvalue.cpp
+++ b/document/src/vespa/document/fieldvalue/arrayfieldvalue.cpp
@@ -3,14 +3,23 @@
 #include "intfieldvalue.h"
 #include "stringfieldvalue.h"
 #include "predicatefieldvalue.h"
+#include "iteratorhandler.h"
 #include <vespa/document/util/serializableexceptions.h>
+#include <vespa/vespalib/util/polymorphicarrays.h>
+#include <vespa/vespalib/util/xmlstream.h>
 #include <vespa/log/log.h>
 
 LOG_SETUP(".document.fieldvalue.array");
 
+using namespace vespalib::xml;
+
 namespace document {
 
 using vespalib::IllegalArgumentException;
+using fieldvalue::IndexValue;
+using fieldvalue::ModificationStatus;
+using fieldvalue::IteratorHandler;
+using fieldvalue::VariableMap;
 
 IMPLEMENT_IDENTIFIABLE_ABSTRACT(ArrayFieldValue, CollectionFieldValue);
 
@@ -23,7 +32,7 @@ ArrayFieldValue::ArrayFieldValue(const DataType &type)
                 "Cannot generate an array value with non-array type "
                 + type.toString() + ".", VESPA_STRLOC);
     }
-    _array = createArray(getNestedType());
+    _array.reset(static_cast<IArray *>(createArray(getNestedType()).release()));
 }
 
 ArrayFieldValue::ArrayFieldValue(const ArrayFieldValue& other)
@@ -175,14 +184,13 @@ ArrayFieldValue::hasChanged() const
     return false;
 }
 
-FieldValue::IteratorHandler::ModificationStatus
+fieldvalue::ModificationStatus
 ArrayFieldValue::iterateSubset(int startPos, int endPos,
                                const vespalib::stringref & variable,
                                PathRange nested,
-                               IteratorHandler& handler) const
+                               fieldvalue::IteratorHandler& handler) const
 {
-    FieldValue::IteratorHandler::ModificationStatus
-        retVal = FieldValue::IteratorHandler::NOT_MODIFIED;
+    fieldvalue::ModificationStatus retVal = ModificationStatus::NOT_MODIFIED;
 
     LOG(spam, "iterateSubset(start=%d, end=%d, variable='%s')",
         startPos, endPos, variable.c_str());
@@ -191,16 +199,15 @@ ArrayFieldValue::iterateSubset(int startPos, int endPos,
 
     for (int i = startPos; i <= endPos && i < static_cast<int>(_array->size()); ++i) {
         if (!variable.empty()) {
-            handler.getVariables()[variable] = IteratorHandler::IndexValue(i);
+            handler.getVariables()[variable] = IndexValue(i);
         }
 
-        FieldValue::IteratorHandler::ModificationStatus
-            status = array()[i].iterateNested(nested, handler);
+        ModificationStatus status = array()[i].iterateNested(nested, handler);
 
-        if (status == FieldValue::IteratorHandler::REMOVED) {
+        if (status == ModificationStatus::REMOVED) {
             indicesToRemove.push_back(i);
-            retVal = FieldValue::IteratorHandler::MODIFIED;
-        } else if (status == FieldValue::IteratorHandler::MODIFIED) {
+            retVal = ModificationStatus::MODIFIED;
+        } else if (status == ModificationStatus::MODIFIED) {
             retVal = status;
         }
     }
@@ -218,7 +225,7 @@ ArrayFieldValue::iterateSubset(int startPos, int endPos,
     return retVal;
 }
 
-FieldValue::IteratorHandler::ModificationStatus
+fieldvalue::ModificationStatus
 ArrayFieldValue::onIterateNested(PathRange nested, IteratorHandler & handler) const
 {
     IteratorHandler::CollectionScope autoScope(handler, *this);
@@ -234,7 +241,7 @@ ArrayFieldValue::onIterateNested(PathRange nested, IteratorHandler & handler) co
         case FieldPathEntry::VARIABLE:
         {
             LOG(spam, "VARIABLE");
-            IteratorHandler::VariableMap::iterator iter = handler.getVariables().find(fpe.getVariableName());
+            VariableMap::iterator iter = handler.getVariables().find(fpe.getVariableName());
             if (iter != handler.getVariables().end()) {
                 int idx = iter->second.index;
 
@@ -258,17 +265,17 @@ ArrayFieldValue::onIterateNested(PathRange nested, IteratorHandler & handler) co
         }
         return iterateSubset(0, static_cast<int>(_array->size()) - 1, "", nested, handler);
     } else {
-        IteratorHandler::ModificationStatus status = handler.modify(const_cast<ArrayFieldValue&>(*this));
+        ModificationStatus status = handler.modify(const_cast<ArrayFieldValue&>(*this));
 
-        if (status == FieldValue::IteratorHandler::REMOVED) {
+        if (status == ModificationStatus::REMOVED) {
             return status;
         }
 
         if (handler.handleComplex(*this)) {
-            if (iterateSubset(0, static_cast<int>(_array->size()) - 1, "",
-                              nested, handler) != FieldValue::IteratorHandler::NOT_MODIFIED)
+            if (iterateSubset(0, static_cast<int>(_array->size()) - 1, "", nested, handler)
+                != ModificationStatus::NOT_MODIFIED)
             {
-                status = FieldValue::IteratorHandler::MODIFIED;
+                status = ModificationStatus::MODIFIED;
             }
         }
 
diff --git a/document/src/vespa/document/fieldvalue/arrayfieldvalue.h b/document/src/vespa/document/fieldvalue/arrayfieldvalue.h
index 33e07e18588..d61acb8605c 100644
--- a/document/src/vespa/document/fieldvalue/arrayfieldvalue.h
+++ b/document/src/vespa/document/fieldvalue/arrayfieldvalue.h
@@ -14,25 +14,27 @@
 
 #include "collectionfieldvalue.h"
 #include <vespa/document/datatype/arraydatatype.h>
+#include <vespa/vespalib/util/polymorphicarray.h>
 
 namespace document {
 
 class ArrayFieldValue : public CollectionFieldValue {
 private:
-    IArray::UP _array;
+    using IArray = vespalib::IArrayT<FieldValue>;
+    std::unique_ptr<IArray> _array;
 
     bool addValue(const FieldValue&) override;
     bool containsValue(const FieldValue& val) const override;
     bool removeValue(const FieldValue& val) override;
-    IteratorHandler::ModificationStatus iterateSubset(
+    fieldvalue::ModificationStatus iterateSubset(
             int startPos, int endPos, const vespalib::stringref & variable,
             PathRange nested,
-            IteratorHandler& handler) const;
-    IteratorHandler::ModificationStatus onIterateNested(PathRange nested, IteratorHandler & handler) const override;
+            fieldvalue::IteratorHandler& handler) const;
+    fieldvalue::ModificationStatus onIterateNested(PathRange nested, fieldvalue::IteratorHandler & handler) const override;
 public:
-    typedef IArray::const_iterator const_iterator;
-    typedef IArray::iterator iterator;
-    typedef std::unique_ptr<ArrayFieldValue> UP;
+    using const_iterator = IArray::const_iterator;
+    using iterator = IArray::iterator;
+    using UP = std::unique_ptr<ArrayFieldValue>;
 
     /**
      * @param arrayType Type of the array. Must be an ArrayDataType, but does
diff --git a/document/src/vespa/document/fieldvalue/document.cpp b/document/src/vespa/document/fieldvalue/document.cpp
index f8358cfb544..a8336519e16 100644
--- a/document/src/vespa/document/fieldvalue/document.cpp
+++ b/document/src/vespa/document/fieldvalue/document.cpp
@@ -1,6 +1,7 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "document.h"
+#include <vespa/document/datatype/documenttype.h>
 #include <vespa/vespalib/util/crc.h>
 #include <vespa/document/repo/fixedtyperepo.h>
 #include <vespa/document/serialization/vespadocumentdeserializer.h>
@@ -8,13 +9,15 @@
 #include <vespa/vespalib/objects/nbostream.h>
 #include <vespa/document/util/serializableexceptions.h>
 #include <vespa/document/base/exceptions.h>
+#include <vespa/document/util/bytebuffer.h>
+#include <vespa/vespalib/util/xmlstream.h>
 #include <sstream>
 
 using vespalib::nbostream;
 using vespalib::make_string;
 using vespalib::IllegalArgumentException;
 using vespalib::IllegalStateException;
-
+using namespace vespalib::xml;
 
 namespace document {
 namespace {
@@ -153,6 +156,11 @@ Document::swap(Document & rhs)
     std::swap(_lastModified, rhs._lastModified);
 }
 
+const DocumentType&
+Document::getType() const {
+    return static_cast<const DocumentType &>(StructuredFieldValue::getType());
+}
+
 Document& Document::operator=(const Document& doc)
 {
     StructuredFieldValue::operator=(doc);
diff --git a/document/src/vespa/document/fieldvalue/document.h b/document/src/vespa/document/fieldvalue/document.h
index f159e9be1d5..02b97238cb2 100644
--- a/document/src/vespa/document/fieldvalue/document.h
+++ b/document/src/vespa/document/fieldvalue/document.h
@@ -18,7 +18,6 @@
 #include "structfieldvalue.h"
 #include <vespa/document/base/documentid.h>
 #include <vespa/document/base/field.h>
-#include <vespa/document/datatype/documenttype.h>
 
 namespace document {
 
@@ -71,10 +70,7 @@ public:
     void accept(FieldValueVisitor &visitor) override { visitor.visit(*this); }
     void accept(ConstFieldValueVisitor &visitor) const override { visitor.visit(*this); }
 
-    const DocumentType& getType() const {
-        return static_cast<const DocumentType &>(StructuredFieldValue::getType());
-    }
-
+    const DocumentType& getType() const;
     const DocumentId& getId() const { return _id; }
     DocumentId & getId() { return _id; }
 
diff --git a/document/src/vespa/document/fieldvalue/fieldvalue.cpp b/document/src/vespa/document/fieldvalue/fieldvalue.cpp
index fd3093488ea..c137811514c 100644
--- a/document/src/vespa/document/fieldvalue/fieldvalue.cpp
+++ b/document/src/vespa/document/fieldvalue/fieldvalue.cpp
@@ -10,17 +10,22 @@
 #include "doublefieldvalue.h"
 #include "bytefieldvalue.h"
 #include "predicatefieldvalue.h"
-
+#include "iteratorhandler.h"
+#include <vespa/document/util/bytebuffer.h>
 #include <vespa/document/base/exceptions.h>
 #include <vespa/document/serialization/vespadocumentserializer.h>
 #include <vespa/vespalib/objects/nbostream.h>
+#include <vespa/vespalib/util/polymorphicarrays.h>
+#include <vespa/vespalib/util/xmlstream.h>
 #include <sstream>
 
 using vespalib::FieldBase;
 using vespalib::nbostream;
-
+using namespace vespalib::xml;
 namespace document {
 
+using namespace fieldvalue;
+
 IMPLEMENT_IDENTIFIABLE_ABSTRACT(FieldValue, vespalib::Identifiable);
 
 void FieldValue::serialize(nbostream &stream) const {
@@ -51,6 +56,21 @@ FieldValue::hash() const
     return vespalib::hashValue(os.c_str(), os.size()) ;
 }
 
+bool
+FieldValue::isA(const FieldValue& other) const {
+    return (getDataType()->isA(*other.getDataType()));
+}
+int
+FieldValue::compare(const FieldValue& other) const {
+    const DataType & a = *getDataType();
+    const DataType & b = *other.getDataType();
+    return (a < b)
+           ? -1
+           : (b < a)
+             ? 1
+             : 0;
+}
+
 FieldValue&
 FieldValue::assign(const FieldValue& value)
 {
@@ -168,13 +188,13 @@ FieldValue::onGetNestedFieldValue(PathRange nested) const
     return FieldValue::UP();
 }
 
-FieldValue::IteratorHandler::ModificationStatus
+ModificationStatus
 FieldValue::iterateNested(PathRange nested, IteratorHandler & handler) const
 {
     return onIterateNested(nested, handler);
 }
 
-FieldValue::IteratorHandler::ModificationStatus
+ModificationStatus
 FieldValue::onIterateNested(PathRange nested, IteratorHandler & handler) const
 {
     if (nested.atEnd()) {
@@ -185,78 +205,6 @@ FieldValue::onIterateNested(PathRange nested, IteratorHandler & handler) const
     }
 }
 
-FieldValue::IteratorHandler::~IteratorHandler() { }
-
-bool
-FieldValue::IteratorHandler::IndexValue::operator==(const FieldValue::IteratorHandler::IndexValue& other) const {
-    if (key.get() != NULL) {
-        if (other.key.get() != NULL && *key == *other.key) {
-            return true;
-        }
-        return false;
-    }
-
-    return index == other.index;
-}
-
-FieldValue::IteratorHandler::IndexValue::IndexValue(const FieldValue& key_)
-    : index(-1),
-      key(FieldValue::CP(key_.clone()))
-{ }
-
-FieldValue::IteratorHandler::IndexValue::~IndexValue() { }
-
-vespalib::string
-FieldValue::IteratorHandler::IndexValue::toString() const {
-    if (key.get() != NULL) {
-        return key->toString();
-    } else {
-        return vespalib::make_string("%d", index);
-    }
-}
-
-void
-FieldValue::IteratorHandler::handlePrimitive(uint32_t fid, const FieldValue & fv) {
-    onPrimitive(fid, Content(fv, getWeight()));
-}
-bool
-FieldValue::IteratorHandler::handleComplex(const FieldValue & fv) {
-    return onComplex(Content(fv, getWeight()));
-}
-void
-FieldValue::IteratorHandler::handleCollectionStart(const FieldValue & fv) {
-    onCollectionStart(Content(fv, getWeight()));
-}
-void
-FieldValue::IteratorHandler::handleCollectionEnd(const FieldValue & fv) {
-    onCollectionEnd(Content(fv, getWeight()));
-}
-void
-FieldValue::IteratorHandler::handleStructStart(const FieldValue & fv) {
-    onStructStart(Content(fv, getWeight()));
-}
-void
-FieldValue::IteratorHandler::handleStructEnd(const FieldValue & fv) {
-    onStructEnd(Content(fv, getWeight()));
-}
-
-void
-FieldValue::IteratorHandler::onPrimitive(uint32_t fid, const Content & fv) {
-    (void) fid;
-    (void) fv;
-}
-
-std::string
-FieldValue::IteratorHandler::toString(const VariableMap& vars) {
-    std::ostringstream out;
-    out << "[ ";
-    for (const auto & entry : vars) {
-        out << entry.first << "=" << entry.second.toString() << " ";
-    }
-    out << "]";
-    return out.str();
-}
-
 std::string
 FieldValue::toString(bool verbose, const std::string& indent) const
 {
@@ -280,26 +228,26 @@ private:
 };
 }
 
-FieldValue::IArray::UP
+std::unique_ptr<vespalib::IArrayBase>
 FieldValue::createArray(const DataType & baseType)
 {
     switch(baseType.getId()) {
     case DataType::T_INT:
-        return IArray::UP(new PrimitiveArrayT<IntFieldValue, FieldValue>());
+        return std::make_unique<PrimitiveArrayT<IntFieldValue, FieldValue>>();
     case DataType::T_FLOAT:
-        return IArray::UP(new PrimitiveArrayT<FloatFieldValue, FieldValue>());
+        return std::make_unique<PrimitiveArrayT<FloatFieldValue, FieldValue>>();
     case DataType::T_STRING:
-        return IArray::UP(new PrimitiveArrayT<StringFieldValue, FieldValue>());
+        return std::make_unique<PrimitiveArrayT<StringFieldValue, FieldValue>>();
     case DataType::T_RAW:
-        return IArray::UP(new PrimitiveArrayT<RawFieldValue, FieldValue>());
+        return std::make_unique<PrimitiveArrayT<RawFieldValue, FieldValue>>();
     case DataType::T_LONG:
-        return IArray::UP(new PrimitiveArrayT<LongFieldValue, FieldValue>());
+        return std::make_unique<PrimitiveArrayT<LongFieldValue, FieldValue>>();
     case DataType::T_DOUBLE:
-        return IArray::UP(new PrimitiveArrayT<DoubleFieldValue, FieldValue>());
+        return std::make_unique<PrimitiveArrayT<DoubleFieldValue, FieldValue>>();
     case DataType::T_BYTE:
-        return IArray::UP(new PrimitiveArrayT<ByteFieldValue, FieldValue>());
+        return std::make_unique<PrimitiveArrayT<ByteFieldValue, FieldValue>>();
     default:
-        return IArray::UP(new ComplexArrayT<FieldValue>(FieldValueFactory::UP(new FieldValueFactory(DataType::UP(baseType.clone())))));
+        return std::make_unique<ComplexArrayT<FieldValue>>(std::make_unique<FieldValueFactory>(DataType::UP(baseType.clone())));
     }
 }
 
diff --git a/document/src/vespa/document/fieldvalue/fieldvalue.h b/document/src/vespa/document/fieldvalue/fieldvalue.h
index bb2a10a3f53..a96628514c3 100644
--- a/document/src/vespa/document/fieldvalue/fieldvalue.h
+++ b/document/src/vespa/document/fieldvalue/fieldvalue.h
@@ -12,12 +12,12 @@
 #pragma once
 
 #include "fieldvaluevisitor.h"
-#include <vespa/document/datatype/datatype.h>
+#include "modificationstatus.h"
 #include <vespa/document/util/xmlserializable.h>
-#include <vespa/vespalib/util/polymorphicarrays.h>
-#include <vespa/document/util/bytebuffer.h>
+#include <vespa/document/base/fieldpath.h>
 #include <vespa/vespalib/objects/cloneable.h>
-#include <map>
+#include <vespa/vespalib/objects/identifiable.h>
+#include <vespa/vespalib/util/polymorphicarraybase.h>
 
 namespace vespalib {
     class nbostream;
@@ -25,13 +25,19 @@ namespace vespalib {
 
 namespace document {
 
+namespace fieldvalue {
+    class IteratorHandler;
+}
+
+class ByteBuffer;
+class DataType;
+
 class FieldValue : public vespalib::Identifiable
 {
 protected:
     FieldValue(const FieldValue&) = default;
     FieldValue& operator=(const FieldValue&) = default;
-    using IArray = vespalib::IArrayT<FieldValue>;
-    static IArray::UP createArray(const DataType & baseType);
+    static std::unique_ptr<vespalib::IArrayBase> createArray(const DataType & baseType);
 
 public:
     using PathRange = FieldPath::Range<FieldPath::const_iterator>;
@@ -39,110 +45,6 @@ public:
     using SP = std::shared_ptr<FieldValue>;
     using CP = vespalib::CloneablePtr<FieldValue>;
 
-    class IteratorHandler {
-    public:
-        class CollectionScope {
-        public:
-            CollectionScope(IteratorHandler& handler, const FieldValue& value)
-                : _handler(handler), _value(value)
-            {
-                _handler.handleCollectionStart(_value);
-            }
-
-            ~CollectionScope() {
-                _handler.handleCollectionEnd(_value);
-            }
-        private:
-            IteratorHandler& _handler;
-            const FieldValue& _value;
-        };
-
-        class StructScope {
-        public:
-            StructScope(IteratorHandler& handler, const FieldValue& value)
-                : _handler(handler), _value(value)
-            {
-                _handler.handleStructStart(_value);
-            }
-
-            ~StructScope() {
-                _handler.handleStructEnd(_value);
-            }
-        private:
-            IteratorHandler& _handler;
-            const FieldValue& _value;
-        };
-
-        class IndexValue {
-        public:
-            IndexValue() : index(-1), key() {}
-            IndexValue(int index_) : index(index_), key() {}
-            IndexValue(const FieldValue& key_);
-            ~IndexValue();
-
-            vespalib::string toString() const;
-
-            bool operator==(const IndexValue& other) const;
-
-            int index; // For array
-            FieldValue::CP key; // For map/wset
-        };
-
-        enum ModificationStatus {
-            MODIFIED,
-            REMOVED,
-            NOT_MODIFIED
-        };
-
-        typedef std::map<vespalib::string, IndexValue> VariableMap;
-    protected:
-        class Content {
-        public:
-            Content(const FieldValue & fv, int weight=1) : _fieldValue(fv), _weight(weight) { }
-            int getWeight()               const { return _weight; }
-            const FieldValue & getValue() const { return _fieldValue; }
-        private:
-            const FieldValue & _fieldValue;
-            int                _weight;
-        };
-        IteratorHandler() : _weight(1) { }
-    public:
-        virtual ~IteratorHandler();
-
-        void handlePrimitive(uint32_t fid, const FieldValue & fv);
-
-        /**
-           Handles a complex type (struct/array/map etc) that is at the end of the
-           field path.
-           @return Return true if you want to recurse into the members.
-        */
-        bool handleComplex(const FieldValue& fv);
-        void handleCollectionStart(const FieldValue & fv);
-        void handleCollectionEnd(const FieldValue & fv);
-        void handleStructStart(const FieldValue & fv);
-        void handleStructEnd(const FieldValue & fv);
-        void setWeight(int weight) { _weight = weight; }
-        ModificationStatus modify(FieldValue& fv) { return doModify(fv); }
-
-        VariableMap& getVariables() { return _variables; }
-        void setVariables(const VariableMap& vars) { _variables = vars; }
-        static std::string toString(const VariableMap& vars);
-        virtual bool createMissingPath() const { return false; }
-    private:
-        virtual bool onComplex(const Content& fv) { (void) fv; return true; }
-        virtual void onPrimitive(uint32_t fid, const Content & fv);
-        virtual void onCollectionStart(const Content & fv) { (void) fv; }
-        virtual void onCollectionEnd(const Content & fv) { (void) fv; }
-        virtual void onStructStart(const Content & fv) { (void) fv; }
-        virtual void onStructEnd(const Content & fv) { (void) fv; }
-        virtual ModificationStatus doModify(FieldValue&) { return NOT_MODIFIED; };
-
-        // Scratchpad to store pass on weight.
-        int getWeight() const  { return _weight; }
-        int _weight;
-        VariableMap _variables;
-    };
-
     DECLARE_IDENTIFIABLE_ABSTRACT(FieldValue);
 
     FieldValue() {}
@@ -168,27 +70,18 @@ public:
     virtual const DataType *getDataType() const = 0;
 
     /** Wrapper for datatypes isA() function. See DataType. */
-    virtual bool isA(const FieldValue& other) const
-        { return (getDataType()->isA(*other.getDataType())); }
+    virtual bool isA(const FieldValue& other) const;
 
     void serialize(vespalib::nbostream &stream) const;
     void serialize(ByteBuffer& buffer) const;
-    ByteBuffer::UP serialize() const;
+    std::unique_ptr<ByteBuffer> serialize() const;
 
     /**
      * Compares this fieldvalue with another fieldvalue.
      * Should return 0 if the two are equal, <0 if this object is "less" than
      * the other, and >0 if this object is more than the other.
      */
-    virtual int compare(const FieldValue& other) const {
-        const DataType & a = *getDataType();
-        const DataType & b = *other.getDataType();
-        return (a < b)
-                   ? -1
-                   : (b < a)
-                       ? 1
-                       : 0;
-    }
+    virtual int compare(const FieldValue& other) const;
 
     /**
      * Returns true if this object have been altered since last
@@ -276,9 +169,9 @@ public:
      * invocations of the before mentioned methods and the additional
      * onPrimitive.
      */
-    IteratorHandler::ModificationStatus iterateNested(PathRange nested, IteratorHandler & handler) const;
+    fieldvalue::ModificationStatus iterateNested(PathRange nested, fieldvalue::IteratorHandler & handler) const;
 
-    IteratorHandler::ModificationStatus iterateNested(const FieldPath& fieldPath, IteratorHandler& handler) const {
+    fieldvalue::ModificationStatus iterateNested(const FieldPath& fieldPath, fieldvalue::IteratorHandler& handler) const {
         return iterateNested(fieldPath.begin(), fieldPath.end(), handler);
     }
 
@@ -292,12 +185,12 @@ public:
     virtual void printXml(XmlOutputStream& out) const = 0;
 
 private:
-    IteratorHandler::ModificationStatus
-    iterateNested(FieldPath::const_iterator start, FieldPath::const_iterator end, IteratorHandler & handler) const {
+    fieldvalue::ModificationStatus
+    iterateNested(FieldPath::const_iterator start, FieldPath::const_iterator end, fieldvalue::IteratorHandler & handler) const {
         return iterateNested(PathRange(start, end), handler);
     }
     virtual FieldValue::UP onGetNestedFieldValue(PathRange nested) const;
-    virtual IteratorHandler::ModificationStatus onIterateNested(PathRange nested, IteratorHandler & handler) const;
+    virtual fieldvalue::ModificationStatus onIterateNested(PathRange nested, fieldvalue::IteratorHandler & handler) const;
 };
 
 std::ostream& operator<<(std::ostream& out, const FieldValue & p);
diff --git a/document/src/vespa/document/fieldvalue/iteratorhandler.cpp b/document/src/vespa/document/fieldvalue/iteratorhandler.cpp
new file mode 100644
index 00000000000..394979f31cb
--- /dev/null
+++ b/document/src/vespa/document/fieldvalue/iteratorhandler.cpp
@@ -0,0 +1,43 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+
+#include "iteratorhandler.h"
+
+namespace document::fieldvalue {
+
+IteratorHandler::~IteratorHandler() { }
+
+
+void
+IteratorHandler::handlePrimitive(uint32_t fid, const FieldValue & fv) {
+    onPrimitive(fid, Content(fv, getWeight()));
+}
+bool
+IteratorHandler::handleComplex(const FieldValue & fv) {
+    return onComplex(Content(fv, getWeight()));
+}
+void
+IteratorHandler::handleCollectionStart(const FieldValue & fv) {
+    onCollectionStart(Content(fv, getWeight()));
+}
+void
+IteratorHandler::handleCollectionEnd(const FieldValue & fv) {
+    onCollectionEnd(Content(fv, getWeight()));
+}
+void
+IteratorHandler::handleStructStart(const FieldValue & fv) {
+    onStructStart(Content(fv, getWeight()));
+}
+void
+IteratorHandler::handleStructEnd(const FieldValue & fv) {
+    onStructEnd(Content(fv, getWeight()));
+}
+
+void
+IteratorHandler::onPrimitive(uint32_t fid, const Content & fv) {
+    (void) fid;
+    (void) fv;
+}
+
+}
+
diff --git a/document/src/vespa/document/fieldvalue/iteratorhandler.h b/document/src/vespa/document/fieldvalue/iteratorhandler.h
new file mode 100644
index 00000000000..96290c1ec56
--- /dev/null
+++ b/document/src/vespa/document/fieldvalue/iteratorhandler.h
@@ -0,0 +1,99 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "variablemap.h"
+#include "modificationstatus.h"
+
+namespace document::fieldvalue {
+
+class IteratorHandler {
+public:
+    class CollectionScope {
+    public:
+        CollectionScope(IteratorHandler &handler, const FieldValue &value)
+                : _handler(handler), _value(value) {
+            _handler.handleCollectionStart(_value);
+        }
+
+        ~CollectionScope() {
+            _handler.handleCollectionEnd(_value);
+        }
+
+    private:
+        IteratorHandler &_handler;
+        const FieldValue &_value;
+    };
+
+    class StructScope {
+    public:
+        StructScope(IteratorHandler &handler, const FieldValue &value)
+                : _handler(handler), _value(value) {
+            _handler.handleStructStart(_value);
+        }
+
+        ~StructScope() {
+            _handler.handleStructEnd(_value);
+        }
+
+    private:
+        IteratorHandler &_handler;
+        const FieldValue &_value;
+    };
+
+protected:
+    class Content {
+    public:
+        Content(const FieldValue &fv, int weight = 1) : _fieldValue(fv), _weight(weight) {}
+
+        int getWeight() const { return _weight; }
+
+        const FieldValue &getValue() const { return _fieldValue; }
+
+    private:
+        const FieldValue &_fieldValue;
+        int _weight;
+    };
+
+    IteratorHandler() : _weight(1) {}
+
+public:
+    virtual ~IteratorHandler();
+    void handlePrimitive(uint32_t fid, const FieldValue &fv);
+
+    /**
+       Handles a complex type (struct/array/map etc) that is at the end of the
+       field path.
+       @return Return true if you want to recurse into the members.
+    */
+    bool handleComplex(const FieldValue &fv);
+    void handleCollectionStart(const FieldValue &fv);
+    void handleCollectionEnd(const FieldValue &fv);
+    void handleStructStart(const FieldValue &fv);
+    void handleStructEnd(const FieldValue &fv);
+    void setWeight(int weight) { _weight = weight; }
+    ModificationStatus modify(FieldValue &fv) { return doModify(fv); }
+    fieldvalue::VariableMap &getVariables() { return _variables; }
+    void setVariables(const fieldvalue::VariableMap &vars) { _variables = vars; }
+    virtual bool createMissingPath() const { return false; }
+private:
+    virtual bool onComplex(const Content &fv) {
+        (void) fv;
+        return true;
+    }
+
+    virtual void onPrimitive(uint32_t fid, const Content &fv);
+    virtual void onCollectionStart(const Content &fv) { (void) fv; }
+    virtual void onCollectionEnd(const Content &fv) { (void) fv; }
+    virtual void onStructStart(const Content &fv) { (void) fv; }
+    virtual void onStructEnd(const Content &fv) { (void) fv; }
+    virtual ModificationStatus doModify(FieldValue &) { return ModificationStatus::NOT_MODIFIED; };
+
+    // Scratchpad to store pass on weight.
+    int getWeight() const { return _weight; }
+
+    int _weight;
+    fieldvalue::VariableMap _variables;
+};
+
+}
diff --git a/document/src/vespa/document/fieldvalue/literalfieldvalue.cpp b/document/src/vespa/document/fieldvalue/literalfieldvalue.cpp
index 4d044219d4c..50c6c901808 100644
--- a/document/src/vespa/document/fieldvalue/literalfieldvalue.cpp
+++ b/document/src/vespa/document/fieldvalue/literalfieldvalue.cpp
@@ -1,9 +1,12 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
-#include <vespa/document/fieldvalue/literalfieldvalue.h>
+#include "literalfieldvalue.h"
 #include <vespa/document/util/stringutil.h>
+#include <vespa/vespalib/util/xmlstream.h>
 #include <sstream>
 
+using namespace vespalib::xml;
+
 namespace document {
 
 IMPLEMENT_IDENTIFIABLE_ABSTRACT(LiteralFieldValueB, FieldValue);
diff --git a/document/src/vespa/document/fieldvalue/mapfieldvalue.cpp b/document/src/vespa/document/fieldvalue/mapfieldvalue.cpp
index 2186ff28f60..2b37c43be55 100644
--- a/document/src/vespa/document/fieldvalue/mapfieldvalue.cpp
+++ b/document/src/vespa/document/fieldvalue/mapfieldvalue.cpp
@@ -2,12 +2,16 @@
 
 #include "mapfieldvalue.h"
 #include "weightedsetfieldvalue.h"
+#include "iteratorhandler.h"
 #include <vespa/document/base/exceptions.h>
+#include <vespa/vespalib/util/xmlstream.h>
+#include <ostream>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".document.fieldvalue.map");
 
 using vespalib::Identifiable;
+using namespace vespalib::xml;
 
 /// \todo TODO (was warning):
 // Find a way to search through internal map without
@@ -15,6 +19,8 @@ using vespalib::Identifiable;
 
 namespace document {
 
+using namespace fieldvalue;
+
 IMPLEMENT_IDENTIFIABLE_ABSTRACT(MapFieldValue, FieldValue);
 
 namespace {
@@ -31,8 +37,8 @@ const MapDataType *verifyMapType(const DataType& type) {
 MapFieldValue::MapFieldValue(const DataType &mapType)
     : FieldValue(),
       _type(verifyMapType(mapType)),
-      _keys(createArray(getMapType().getKeyType())),
-      _values(createArray(getMapType().getValueType())),
+      _keys(static_cast<IArray *>(createArray(getMapType().getKeyType()).release())),
+      _values(static_cast<IArray *>(createArray(getMapType().getValueType()).release())),
       _altered(true)
 {
 }
@@ -203,8 +209,7 @@ MapFieldValue::compare(const FieldValue& other) const
 }
 
 void
-MapFieldValue::print(std::ostream& out, bool verbose,
-                       const std::string& indent) const
+MapFieldValue::print(std::ostream& out, bool verbose, const std::string& indent) const
 {
     out << "Map(";
 
@@ -268,22 +273,22 @@ MapFieldValue::find(const FieldValue& key)
 }
 bool
 MapFieldValue::checkAndRemove(const FieldValue& key,
-                              FieldValue::IteratorHandler::ModificationStatus status,
+                             ModificationStatus status,
                               bool wasModified,
                               std::vector<const FieldValue*>& keysToRemove) const
 {
-    if (status == FieldValue::IteratorHandler::REMOVED) {
+    if (status == ModificationStatus::REMOVED) {
         LOG(spam, "will remove: %s", key.toString().c_str());
         keysToRemove.push_back(&key);
         return true;
-    } else if (status == FieldValue::IteratorHandler::MODIFIED) {
+    } else if (status == ModificationStatus::MODIFIED) {
         return true;
     }
 
     return wasModified;
 }
 
-FieldValue::IteratorHandler::ModificationStatus
+ModificationStatus
 MapFieldValue::iterateNestedImpl(PathRange nested,
                                  IteratorHandler & handler,
                                  const FieldValue& complexFieldValue) const
@@ -307,10 +312,9 @@ MapFieldValue::iterateNestedImpl(PathRange nested,
                         wasModified, keysToRemove);
             } else if (handler.createMissingPath()) {
                 LOG(spam, "creating missing path");
-                FieldValue::UP val =
-                    getMapType().getValueType().createFieldValue();
-                IteratorHandler::ModificationStatus status = val->iterateNested(nested.next(), handler);
-                if (status == IteratorHandler::MODIFIED) {
+                FieldValue::UP val = getMapType().getValueType().createFieldValue();
+                ModificationStatus status = val->iterateNested(nested.next(), handler);
+                if (status == ModificationStatus::MODIFIED) {
                     const_cast<MapFieldValue&>(*this).put(FieldValue::UP(fpe.getLookupKey()->clone()), std::move(val));
                     return status;
                 }
@@ -339,8 +343,7 @@ MapFieldValue::iterateNestedImpl(PathRange nested,
         case FieldPathEntry::VARIABLE:
         {
             LOG(spam, "VARIABLE");
-            IteratorHandler::VariableMap::iterator
-                iter = handler.getVariables().find(fpe.getVariableName());
+            VariableMap::iterator iter = handler.getVariables().find(fpe.getVariableName());
             if (iter != handler.getVariables().end()) {
                 LOG(spam, "variable key = %s", iter->second.key->toString().c_str());
                 const_iterator found = find(*iter->second.key);
@@ -353,12 +356,9 @@ MapFieldValue::iterateNestedImpl(PathRange nested,
                 PathRange next = nested.next();
                 for (const_iterator it(begin()), mt(end()); it != mt; it++) {
                     LOG(spam, "key is '%s'", it->first->toString().c_str());
-                    handler.getVariables()[fpe.getVariableName()]
-                        = IteratorHandler::IndexValue(*it->first);
-                    LOG(spam, "vars at this time = %s",
-                        FieldValue::IteratorHandler::toString(handler.getVariables()).c_str());
-                    wasModified = checkAndRemove(*it->first,
-                                                 it->second->iterateNested(next, handler),
+                    handler.getVariables()[fpe.getVariableName()] = IndexValue(*it->first);
+                    LOG(spam, "vars at this time = %s", handler.getVariables().toString().c_str());
+                    wasModified = checkAndRemove(*it->first, it->second->iterateNested(next, handler),
                                                  wasModified, keysToRemove);
                 }
                 handler.getVariables().erase(fpe.getVariableName());
@@ -371,8 +371,7 @@ MapFieldValue::iterateNestedImpl(PathRange nested,
                 if (isWSet) {
                     handler.setWeight(static_cast<const IntFieldValue &>(*it->second).getValue());
                 }
-                wasModified = checkAndRemove(*it->first,
-                                             it->first->iterateNested(nested, handler),
+                wasModified = checkAndRemove(*it->first, it->first->iterateNested(nested, handler),
                                              wasModified, keysToRemove);
                 // Don't iterate over values
                 /*wasModified = checkAndRemove(*it->second,
@@ -383,13 +382,12 @@ MapFieldValue::iterateNestedImpl(PathRange nested,
         }
     } else {
         LOG(spam, "at end of field path");
-        IteratorHandler::ModificationStatus
-            status = handler.modify(const_cast<FieldValue&>(complexFieldValue));
+        ModificationStatus status = handler.modify(const_cast<FieldValue&>(complexFieldValue));
 
-        if (status == IteratorHandler::REMOVED) {
+        if (status == ModificationStatus::REMOVED) {
             LOG(spam, "status = REMOVED");
             return status;
-        } else if (status == IteratorHandler::MODIFIED) {
+        } else if (status == ModificationStatus::MODIFIED) {
             LOG(spam, "status = MODIFIED");
             wasModified = true;
         }
@@ -419,10 +417,10 @@ MapFieldValue::iterateNestedImpl(PathRange nested,
         LOG(spam, "erasing map entry with key %s", (*i)->toString().c_str());
         const_cast<MapFieldValue&>(*this).erase(**i);
     }
-    return wasModified ? IteratorHandler::MODIFIED : IteratorHandler::NOT_MODIFIED;
+    return wasModified ? ModificationStatus::MODIFIED : ModificationStatus::NOT_MODIFIED;
 }
 
-FieldValue::IteratorHandler::ModificationStatus
+ModificationStatus
 MapFieldValue::onIterateNested(PathRange nested, IteratorHandler & handler) const
 {
     LOG(spam, "iterating over MapFieldValue");
diff --git a/document/src/vespa/document/fieldvalue/mapfieldvalue.h b/document/src/vespa/document/fieldvalue/mapfieldvalue.h
index 8b480cbb7f9..edc20fea777 100644
--- a/document/src/vespa/document/fieldvalue/mapfieldvalue.h
+++ b/document/src/vespa/document/fieldvalue/mapfieldvalue.h
@@ -26,10 +26,10 @@ private:
     virtual bool containsValue(const FieldValue& fv) const { return contains(fv); }
     virtual bool removeValue(const FieldValue& fv) { return erase(fv); }
     bool checkAndRemove(const FieldValue& key,
-                        FieldValue::IteratorHandler::ModificationStatus status,
+                        fieldvalue::ModificationStatus status,
                         bool wasModified,
                         std::vector<const FieldValue*>& keysToRemove) const;
-    IteratorHandler::ModificationStatus onIterateNested(PathRange nested, IteratorHandler & handler) const override;
+    fieldvalue::ModificationStatus onIterateNested(PathRange nested, fieldvalue::IteratorHandler & handler) const override;
     // Utility method to avoid constant explicit casting
     const MapDataType& getMapType() const { return *_type; }
 
@@ -118,8 +118,8 @@ public:
     void reserve(size_t sz) { _keys->reserve(sz); _values->reserve(sz); }
     void resize(size_t sz) { _keys->resize(sz); _values->resize(sz); }
 
-    IteratorHandler::ModificationStatus iterateNestedImpl(PathRange nested, IteratorHandler & handler,
-                                                          const FieldValue& complexFieldValue) const;
+    fieldvalue::ModificationStatus iterateNestedImpl(PathRange nested, fieldvalue::IteratorHandler & handler,
+                                                     const FieldValue& complexFieldValue) const;
 
     // FieldValue implementation
     FieldValue& assign(const FieldValue&) override;
diff --git a/document/src/vespa/document/fieldvalue/modificationstatus.h b/document/src/vespa/document/fieldvalue/modificationstatus.h
new file mode 100644
index 00000000000..9ea0c2fa3ae
--- /dev/null
+++ b/document/src/vespa/document/fieldvalue/modificationstatus.h
@@ -0,0 +1,13 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+namespace document::fieldvalue {
+
+enum class ModificationStatus {
+    MODIFIED,
+    REMOVED,
+    NOT_MODIFIED
+};
+
+}
diff --git a/document/src/vespa/document/fieldvalue/numericfieldvalue.cpp b/document/src/vespa/document/fieldvalue/numericfieldvalue.cpp
index 3d39544b3d0..b897a8db930 100644
--- a/document/src/vespa/document/fieldvalue/numericfieldvalue.cpp
+++ b/document/src/vespa/document/fieldvalue/numericfieldvalue.cpp
@@ -1,6 +1,7 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "numericfieldvalue.h"
+#include <vespa/vespalib/util/xmlstream.h>
 
 namespace document {
 
@@ -9,7 +10,7 @@ IMPLEMENT_IDENTIFIABLE_ABSTRACT(NumericFieldValueBase, FieldValue);
 void
 NumericFieldValueBase::printXml(XmlOutputStream& out) const
 {
-    out << XmlContent(getAsString());
+    out << vespalib::xml::XmlContent(getAsString());
 }
 
 
diff --git a/document/src/vespa/document/fieldvalue/predicatefieldvalue.cpp b/document/src/vespa/document/fieldvalue/predicatefieldvalue.cpp
index 8e98ebc4c4e..a3394d8fcf4 100644
--- a/document/src/vespa/document/fieldvalue/predicatefieldvalue.cpp
+++ b/document/src/vespa/document/fieldvalue/predicatefieldvalue.cpp
@@ -1,13 +1,16 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "predicatefieldvalue.h"
+#include <vespa/document/datatype/datatype.h>
 #include <vespa/document/predicate/predicate.h>
 #include <vespa/document/predicate/predicate_printer.h>
 #include <vespa/vespalib/data/slime/inserter.h>
 #include <vespa/vespalib/data/slime/slime.h>
+#include <vespa/vespalib/util/xmlstream.h>
 
 using vespalib::Slime;
 using vespalib::slime::SlimeInserter;
+using namespace vespalib::xml;
 
 namespace document {
 
@@ -67,4 +70,19 @@ void PredicateFieldValue::print(std::ostream& out, bool, const std::string&) con
     out << PredicatePrinter::print(*_slime) << "\n";
 }
 
+const DataType *
+PredicateFieldValue::getDataType() const {
+    return DataType::PREDICATE;
+}
+
+bool
+PredicateFieldValue::hasChanged() const {
+    return _altered;
+}
+
+FieldValue *
+PredicateFieldValue::clone() const {
+    return new PredicateFieldValue(*this);
+}
+
 }  // namespace document
diff --git a/document/src/vespa/document/fieldvalue/predicatefieldvalue.h b/document/src/vespa/document/fieldvalue/predicatefieldvalue.h
index b38f9e5dcc1..74dc0d11322 100644
--- a/document/src/vespa/document/fieldvalue/predicatefieldvalue.h
+++ b/document/src/vespa/document/fieldvalue/predicatefieldvalue.h
@@ -2,7 +2,6 @@
 
 #pragma once
 
-#include <memory>
 #include "fieldvalue.h"
 
 namespace vespalib {
@@ -25,20 +24,20 @@ public:
     void accept(FieldValueVisitor &visitor) override { visitor.visit(*this); }
     void accept(ConstFieldValueVisitor &visitor) const override { visitor.visit(*this); }
 
-    virtual FieldValue *clone() const override { return new PredicateFieldValue(*this); }
-    virtual int compare(const FieldValue &rhs) const override;
+    FieldValue *clone() const override;
+    int compare(const FieldValue &rhs) const override;
 
-    virtual void printXml(XmlOutputStream &out) const override;
-    virtual void print(std::ostream &out, bool verbose, const std::string &indent) const override;
+    void printXml(XmlOutputStream &out) const override;
+    void print(std::ostream &out, bool verbose, const std::string &indent) const override;
 
-    virtual const DataType *getDataType() const override { return DataType::PREDICATE; }
-    virtual bool hasChanged() const override { return _altered; }
+    const DataType *getDataType() const override;
+    bool hasChanged() const override;
 
     const vespalib::Slime &getSlime() const { return *_slime; }
 
-    virtual FieldValue &assign(const FieldValue &rhs) override;
+    FieldValue &assign(const FieldValue &rhs) override;
 
-DECLARE_IDENTIFIABLE(PredicateFieldValue);
+    DECLARE_IDENTIFIABLE(PredicateFieldValue);
 };
 
 }  // namespace document
diff --git a/document/src/vespa/document/fieldvalue/rawfieldvalue.cpp b/document/src/vespa/document/fieldvalue/rawfieldvalue.cpp
index 96701832b5e..c9950036435 100644
--- a/document/src/vespa/document/fieldvalue/rawfieldvalue.cpp
+++ b/document/src/vespa/document/fieldvalue/rawfieldvalue.cpp
@@ -3,6 +3,9 @@
 #include "rawfieldvalue.h"
 #include "literalfieldvalue.hpp"
 #include <vespa/document/util/stringutil.h>
+#include <vespa/vespalib/util/xmlstream.h>
+
+using namespace vespalib::xml;
 
 namespace document {
 
@@ -18,8 +21,7 @@ RawFieldValue::printXml(XmlOutputStream& out) const
 void
 RawFieldValue::print(std::ostream& out, bool, const std::string&) const
 {
-    StringUtil::printAsHex(out, _value.c_str(),
-                                _value.size());
+    StringUtil::printAsHex(out, _value.c_str(), _value.size());
 }
 
 } // document
diff --git a/document/src/vespa/document/fieldvalue/serializablearray.cpp b/document/src/vespa/document/fieldvalue/serializablearray.cpp
index 7a174f1ab52..c8beea5e79c 100644
--- a/document/src/vespa/document/fieldvalue/serializablearray.cpp
+++ b/document/src/vespa/document/fieldvalue/serializablearray.cpp
@@ -1,16 +1,29 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 #include "serializablearray.h"
 #include <vespa/document/util/serializableexceptions.h>
+#include <vespa/document/util/bytebuffer.h>
 #include <vespa/vespalib/stllike/hash_map.hpp>
+#include <vespa/vespalib/data/databuffer.h>
+#include <vespa/document/util/compressor.h>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".document.serializable-array");
 
-
 using std::vector;
 
 namespace document {
 
+namespace serializablearray {
+
+using BufferMapT = vespalib::hash_map<int, ByteBuffer::UP>;
+
+class BufferMap : public BufferMapT {
+public:
+    using BufferMapT::BufferMapT;
+};
+
+}
+
 SerializableArray::Statistics SerializableArray::_stats;
 
 SerializableArray::SerializableArray()
@@ -19,6 +32,13 @@ SerializableArray::SerializableArray()
 {
 }
 
+serializablearray::BufferMap & ensure(std::unique_ptr<serializablearray::BufferMap> & owned) {
+    if (!owned) {
+        owned = std::make_unique<serializablearray::BufferMap>();
+    }
+    return *owned;
+}
+
 SerializableArray::SerializableArray(const SerializableArray& other)
     : Cloneable(),
       _entries(other._entries),
@@ -34,7 +54,7 @@ SerializableArray::SerializableArray(const SerializableArray& other)
             // Pointing to a buffer in the _owned structure.
             ByteBuffer::UP buf(ByteBuffer::copyBuffer(e.getBuffer(_uncompSerData.get()), e.size()));
             e.setBuffer(buf->getBuffer());
-            _owned[e.id()] = std::move(buf);
+            ensure(_owned)[e.id()] = std::move(buf);
         } else {
             // If not it is relative to the buffer _uncompSerData, and hence it is valid as is.
         }
@@ -79,7 +99,7 @@ SerializableArray::set(int id, ByteBuffer::UP buffer)
 {
     maybeDecompress();
     Entry e(id, buffer->getRemaining(), buffer->getBuffer());
-    _owned[id] = std::move(buffer);
+    ensure(_owned)[id] = std::move(buffer);
     EntryMap::iterator it = find(id);
     if (it == _entries.end()) {
         _entries.push_back(e);
@@ -149,7 +169,9 @@ SerializableArray::clear(int id)
     EntryMap::iterator it = find(id);
     if (it != _entries.end()) {
         _entries.erase(it);
-        _owned.erase(id);
+        if (_owned) {
+            _owned->erase(id);
+        }
         invalidate();
     }
 }
@@ -215,4 +237,14 @@ void SerializableArray::assign(EntryMap & entries,
     }
 }
 
+CompressionInfo
+SerializableArray::getCompressionInfo() const {
+    return CompressionInfo(_uncompressedLength, _compSerData->getRemaining());
+}
+
+const char *
+SerializableArray::Entry::getBuffer(const ByteBuffer * readOnlyBuffer) const {
+    return hasBuffer() ? _data._buffer : readOnlyBuffer->getBuffer() + getOffset();
+}
+
 } // document
diff --git a/document/src/vespa/document/fieldvalue/serializablearray.h b/document/src/vespa/document/fieldvalue/serializablearray.h
index 9c0978f1476..1d211a50ab3 100644
--- a/document/src/vespa/document/fieldvalue/serializablearray.h
+++ b/document/src/vespa/document/fieldvalue/serializablearray.h
@@ -16,20 +16,22 @@
 
 #pragma once
 
-#include <vespa/document/util/bytebuffer.h>
 #include <vespa/document/util/compressionconfig.h>
-#include <vespa/document/util/compressor.h>
-#include <vespa/document/util/serializable.h>
-#include <vector>
 #include <vespa/vespalib/objects/cloneable.h>
-#include <vespa/vespalib/stllike/hash_map.h>
 #include <vespa/vespalib/util/buffer.h>
-#include <vespa/fastos/dynamiclibrary.h>
+#include <vespa/vespalib/util/memory.h>
+#include <vector>
 
-namespace document
-{
+#define VESPA_DLL_LOCAL  __attribute__ ((visibility("hidden")))
+
+namespace document {
 
 class SerializableArrayIterator;
+class ByteBuffer;
+
+namespace serializablearray {
+    class BufferMap;
+}
 
 class SerializableArray : public vespalib::Cloneable
 {
@@ -71,7 +73,7 @@ public:
         bool operator < (const Entry & e) const { return cmp(e) < 0; }
         int cmp(const Entry & e) const { return _id - e._id; }
         void setBuffer(const char * buffer) { _data._buffer = buffer; _sz |= BUFFER_MASK; }
-        const char * getBuffer(const ByteBuffer * readOnlyBuffer) const { return hasBuffer() ? _data._buffer : readOnlyBuffer->getBuffer() + getOffset(); }
+        VESPA_DLL_LOCAL const char * getBuffer(const ByteBuffer * readOnlyBuffer) const;
     private:
         uint32_t getOffset() const { return _data._offset; }
         enum { BUFFER_MASK=0x80000000 };
@@ -100,12 +102,11 @@ public:
 private:
     static Statistics _stats;
 
-    typedef vespalib::hash_map<int, uint32_t> HashMap;
-
 public:
     static Statistics& getStatistics() { return _stats; }
-    typedef vespalib::CloneablePtr<SerializableArray> CP;
-    typedef std::unique_ptr<SerializableArray> UP;
+    using CP = vespalib::CloneablePtr<SerializableArray>;
+    using UP = std::unique_ptr<SerializableArray>;
+    using ByteBufferUP = std::unique_ptr<ByteBuffer>;
 
     SerializableArray();
     virtual ~SerializableArray();
@@ -122,7 +123,7 @@ public:
     void set(int id, const char* value, int len);
 
     /** Stores a value in the array. */
-    void set(int id, std::unique_ptr<ByteBuffer> buffer);
+    void set(int id, ByteBufferUP buffer);
 
     /**
      * Gets a value from the array. This is the faster version of the above.
@@ -152,19 +153,17 @@ public:
     void clear();
 
     CompressionConfig::Type getCompression() const { return _serializedCompression; }
-    CompressionInfo getCompressionInfo() const {
-        return CompressionInfo(_uncompressedLength, _compSerData->getRemaining());
-    }
+    CompressionInfo getCompressionInfo() const;
 
     /**
      * Sets the serialized data that is the basis for this object's
      * content. This is used by deserialization. Any existing entries
      * are cleared.
      */
-    VESPA_DLL_LOCAL void assign(EntryMap &entries,
-                                ByteBuffer::UP buffer,
-                                CompressionConfig::Type comp_type,
-                                uint32_t uncompressed_length);
+    void assign(EntryMap &entries,
+                ByteBufferUP buffer,
+                CompressionConfig::Type comp_type,
+                uint32_t uncompressed_length);
 
     bool empty() const { return _entries.empty(); }
 
@@ -189,22 +188,22 @@ private:
         return false;
     }
 
-    VESPA_DLL_LOCAL bool deCompressAndCatch() const;
+    bool deCompressAndCatch() const;
     void maybeDecompress() const {
         if ( shouldDecompress() ) {
             const_cast<SerializableArray *>(this)->deCompress();
         }
     }
-    VESPA_DLL_LOCAL void deCompress(); // throw (DeserializeException);
+    void deCompress(); // throw (DeserializeException);
 
     /** Contains the stored attributes, with reference to the real data.. */
     EntryMap _entries;
     /** The buffers we own. */
-    vespalib::hash_map<int, ByteBuffer::UP > _owned;
+    std::unique_ptr<serializablearray::BufferMap> _owned;
 
     /** Data we deserialized from, if applicable. */
-    ByteBuffer::UP           _uncompSerData;
-    ByteBuffer::UP           _compSerData;
+    ByteBufferUP             _uncompSerData;
+    ByteBufferUP             _compSerData;
     CompressionConfig::Type  _serializedCompression;
 
     uint32_t     _uncompressedLength;
@@ -215,4 +214,3 @@ private:
 };
 
 } // document
-
diff --git a/document/src/vespa/document/fieldvalue/structfieldvalue.cpp b/document/src/vespa/document/fieldvalue/structfieldvalue.cpp
index 079b730c8cc..cdfba106c49 100644
--- a/document/src/vespa/document/fieldvalue/structfieldvalue.cpp
+++ b/document/src/vespa/document/fieldvalue/structfieldvalue.cpp
@@ -8,9 +8,10 @@
 #include <vespa/vespalib/objects/nbostream.h>
 #include <vespa/vespalib/util/crc.h>
 #include <vespa/document/datatype/positiondatatype.h>
-#include <vespa/vespalib/util/vstringfmt.h>
 #include <vespa/document/util/serializableexceptions.h>
 #include <vespa/document/base/exceptions.h>
+#include <vespa/document/util/bytebuffer.h>
+#include <vespa/vespalib/util/xmlstream.h>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".document.structfieldvalue");
@@ -19,6 +20,7 @@ using std::vector;
 using vespalib::nbostream;
 using vespalib::nbostream_longlivedbuf;
 using vespalib::make_string;
+using namespace vespalib::xml;
 
 namespace document {
 
@@ -43,6 +45,26 @@ StructFieldValue::~StructFieldValue() { }
 StructFieldValue::Chunks::~Chunks() { }
 
 void
+StructFieldValue::Chunks::push_back(SerializableArray::UP item) {
+    assert(_sz < 2);
+    _chunks[_sz++].reset(item.release());
+}
+
+void
+StructFieldValue::Chunks::clear() {
+    _chunks[0].reset();
+    _chunks[1].reset();
+    _sz = 0;
+}
+
+void
+StructFieldValue::Chunks::swap(Chunks & rhs) {
+    _chunks[0].swap(rhs._chunks[0]);
+    _chunks[1].swap(rhs._chunks[1]);
+    std::swap(_sz, rhs._sz);
+}
+
+void
 StructFieldValue::swap(StructFieldValue & rhs)
 {
     StructuredFieldValue::swap(rhs);
@@ -53,6 +75,16 @@ StructFieldValue::swap(StructFieldValue & rhs)
     std::swap(_version, _version);
 }
 
+const StructDataType &
+StructFieldValue::getStructType() const {
+    return static_cast<const StructDataType &>(getType());
+}
+
+const CompressionConfig &
+StructFieldValue::getCompressionConfig() const {
+    return getStructType().getCompressionConfig();
+}
+
 void
 StructFieldValue::lazyDeserialize(const FixedTypeRepo &repo,
                                   uint16_t version,
diff --git a/document/src/vespa/document/fieldvalue/structfieldvalue.h b/document/src/vespa/document/fieldvalue/structfieldvalue.h
index a46a36c477d..01a0e732e43 100644
--- a/document/src/vespa/document/fieldvalue/structfieldvalue.h
+++ b/document/src/vespa/document/fieldvalue/structfieldvalue.h
@@ -11,17 +11,16 @@
 
 #include "structuredfieldvalue.h"
 #include "serializablearray.h"
-#include <vespa/document/util/compressionconfig.h>
-#include <vespa/document/datatype/structdatatype.h>
-#include <vector>
 
 namespace document {
+
 class Document;
 class DocumentType;
 class DocumentTypeRepo;
 class FieldValueWriter;
 class FixedTypeRepo;
 class FieldSet;
+class StructDataType;
 
 class StructFieldValue : public StructuredFieldValue
 {
@@ -32,24 +31,13 @@ public:
         ~Chunks();
         SerializableArray & operator [] (size_t i) { return *_chunks[i]; }
         const SerializableArray & operator [] (size_t i) const { return *_chunks[i]; }
-        void push_back(SerializableArray::UP item) {
-            assert(_sz < 2);
-            _chunks[_sz++].reset(item.release());
-        }
+        VESPA_DLL_LOCAL void push_back(SerializableArray::UP item);
         SerializableArray & back() { return *_chunks[_sz-1]; }
         const SerializableArray & back() const { return *_chunks[_sz-1]; }
         size_t size() const { return _sz; }
         bool empty() const { return _sz == 0; }
-        void clear() {
-            _chunks[0].reset();
-            _chunks[1].reset();
-            _sz = 0;
-        }
-        void swap(Chunks & rhs) {
-            _chunks[0].swap(rhs._chunks[0]);
-            _chunks[1].swap(rhs._chunks[1]);
-            std::swap(_sz, rhs._sz);
-        }
+        VESPA_DLL_LOCAL void clear();
+        VESPA_DLL_LOCAL void swap(Chunks & rhs);
     private:
         SerializableArray::CP _chunks[2];
         size_t _sz;
@@ -73,12 +61,12 @@ public:
     const DocumentTypeRepo * getRepo() const { return _repo; }
     void setDocumentType(const DocumentType & docType) { _doc_type = & docType; }
 
-    const StructDataType & getStructType() const { return static_cast<const StructDataType &>(getType()); }
+    const StructDataType & getStructType() const;
 
     void lazyDeserialize(const FixedTypeRepo &repo,
                          uint16_t version,
                          SerializableArray::EntryMap && fields,
-                         ByteBuffer::UP buffer,
+                         std::unique_ptr<ByteBuffer> buffer,
                          CompressionConfig::Type comp_type,
                          int32_t uncompressed_length);
 
@@ -99,8 +87,7 @@ public:
     const Field& getField(const vespalib::stringref & name) const override;
     void clear() override;
 
-    const CompressionConfig &getCompressionConfig() const
-    { return getStructType().getCompressionConfig(); }
+    const CompressionConfig &getCompressionConfig() const;
 
     // FieldValue implementation.
     FieldValue& assign(const FieldValue&) override;
@@ -146,4 +133,3 @@ private:
 };
 
 } // document
-
diff --git a/document/src/vespa/document/fieldvalue/structuredfieldvalue.cpp b/document/src/vespa/document/fieldvalue/structuredfieldvalue.cpp
index 6d77d101ff2..8e30cba2052 100644
--- a/document/src/vespa/document/fieldvalue/structuredfieldvalue.cpp
+++ b/document/src/vespa/document/fieldvalue/structuredfieldvalue.cpp
@@ -1,13 +1,19 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "structuredfieldvalue.hpp"
-#include "fieldvalues.h"
+#include "iteratorhandler.h"
+#include "weightedsetfieldvalue.h"
+#include "arrayfieldvalue.h"
 
 #include <vespa/log/log.h>
 LOG_SETUP(".document.fieldvalue.structured");
 
+using vespalib::IllegalArgumentException;
+
 namespace document {
 
+using namespace fieldvalue;
+
 IMPLEMENT_IDENTIFIABLE_ABSTRACT(StructuredFieldValue, FieldValue);
 
 StructuredFieldValue::Iterator::Iterator()
@@ -59,7 +65,7 @@ void StructuredFieldValue::setFieldValue(const Field & field, const FieldValue &
     if (!field.getDataType().isValueType(value) &&
         !value.getDataType()->isA(field.getDataType()))
     {
-        throw vespalib::IllegalArgumentException(
+        throw IllegalArgumentException(
                 "Cannot assign value of type " + value.getDataType()->toString()
                 + "with value : '" + value.toString()
                 + "' to field " + field.getName().c_str() + " of type "
@@ -81,7 +87,7 @@ StructuredFieldValue::onGetNestedFieldValue(PathRange nested) const
     return fv;
 }
 
-FieldValue::IteratorHandler::ModificationStatus
+ModificationStatus
 StructuredFieldValue::onIterateNested(PathRange nested, IteratorHandler & handler) const
 {
     IteratorHandler::StructScope autoScope(handler, *this);
@@ -93,38 +99,35 @@ StructuredFieldValue::onIterateNested(PathRange nested, IteratorHandler & handle
             LOG(spam, "fieldRef = %s", fpe.getFieldRef().toString().c_str());
             LOG(spam, "fieldValueToSet = %s", fpe.getFieldValueToSet().toString().c_str());
             if (exists) {
-                IteratorHandler::ModificationStatus status = fpe.getFieldValueToSet().iterateNested(nested.next(), handler);
-                if (status == IteratorHandler::REMOVED) {
+                ModificationStatus status = fpe.getFieldValueToSet().iterateNested(nested.next(), handler);
+                if (status == ModificationStatus::REMOVED) {
                     LOG(spam, "field exists, status = REMOVED");
                     const_cast<StructuredFieldValue&>(*this).remove(fpe.getFieldRef());
-                    return IteratorHandler::MODIFIED;
-                } else if (status == IteratorHandler::MODIFIED) {
+                    return ModificationStatus::MODIFIED;
+                } else if (status == ModificationStatus::MODIFIED) {
                     LOG(spam, "field exists, status = MODIFIED");
                     const_cast<StructuredFieldValue&>(*this).setFieldValue(fpe.getFieldRef(), fpe.getFieldValueToSet());
-                    return IteratorHandler::MODIFIED;
+                    return ModificationStatus::MODIFIED;
                 } else {
-                    LOG(spam, "field exists, status = %u", status);
                     return status;
                 }
             } else if (handler.createMissingPath()) {
                 LOG(spam, "createMissingPath is true");
-                IteratorHandler::ModificationStatus status
-                    = fpe.getFieldValueToSet().iterateNested(nested.next(), handler);
-                if (status == IteratorHandler::MODIFIED) {
+                ModificationStatus status = fpe.getFieldValueToSet().iterateNested(nested.next(), handler);
+                if (status == ModificationStatus::MODIFIED) {
                     LOG(spam, "field did not exist, status = MODIFIED");
                     const_cast<StructuredFieldValue&>(*this).setFieldValue(fpe.getFieldRef(), fpe.getFieldValueToSet());
                     return status;
                 }
             }
             LOG(spam, "field did not exist, returning NOT_MODIFIED");
-            return IteratorHandler::NOT_MODIFIED;
+            return ModificationStatus::NOT_MODIFIED;
         } else {
-            throw vespalib::IllegalArgumentException("Illegal field path for struct value");
+            throw IllegalArgumentException("Illegal field path for struct value");
         }
     } else {
-        IteratorHandler::ModificationStatus
-            status = handler.modify(const_cast<StructuredFieldValue&>(*this));
-        if (status == IteratorHandler::REMOVED) {
+        ModificationStatus status = handler.modify(const_cast<StructuredFieldValue&>(*this));
+        if (status == ModificationStatus::REMOVED) {
             LOG(spam, "field REMOVED");
             return status;
         }
@@ -133,11 +136,11 @@ StructuredFieldValue::onIterateNested(PathRange nested, IteratorHandler & handle
             LOG(spam, "handleComplex");
             std::vector<const Field*> fieldsToRemove;
             for (const_iterator it(begin()), mt(end()); it != mt; ++it) {
-                IteratorHandler::ModificationStatus currStatus = getValue(it.field())->iterateNested(nested, handler);
-                if (currStatus == IteratorHandler::REMOVED) {
+                ModificationStatus currStatus = getValue(it.field())->iterateNested(nested, handler);
+                if (currStatus == ModificationStatus::REMOVED) {
                     fieldsToRemove.push_back(&it.field());
-                    status = IteratorHandler::MODIFIED;
-                } else if (currStatus == IteratorHandler::MODIFIED) {
+                    status = ModificationStatus::MODIFIED;
+                } else if (currStatus == ModificationStatus::MODIFIED) {
                     status = currStatus;
                 }
             }
diff --git a/document/src/vespa/document/fieldvalue/structuredfieldvalue.h b/document/src/vespa/document/fieldvalue/structuredfieldvalue.h
index 4cccbf05de0..eb5912734c5 100644
--- a/document/src/vespa/document/fieldvalue/structuredfieldvalue.h
+++ b/document/src/vespa/document/fieldvalue/structuredfieldvalue.h
@@ -92,8 +92,8 @@ protected:
     virtual void setFieldValue(const Field&, FieldValue::UP value) = 0;
     void setFieldValue(const Field & field, const FieldValue & value);
 
-    IteratorHandler::ModificationStatus
-    onIterateNested(PathRange nested, IteratorHandler & handler) const override;
+    fieldvalue::ModificationStatus
+    onIterateNested(PathRange nested, fieldvalue::IteratorHandler & handler) const override;
 
 public:
     DECLARE_IDENTIFIABLE_ABSTRACT(StructuredFieldValue);
diff --git a/document/src/vespa/document/fieldvalue/tensorfieldvalue.cpp b/document/src/vespa/document/fieldvalue/tensorfieldvalue.cpp
index d6396c43a46..d7d7721e833 100644
--- a/document/src/vespa/document/fieldvalue/tensorfieldvalue.cpp
+++ b/document/src/vespa/document/fieldvalue/tensorfieldvalue.cpp
@@ -1,10 +1,14 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "tensorfieldvalue.h"
+#include <vespa/document/datatype/datatype.h>
+#include <vespa/vespalib/util/xmlstream.h>
 #include <vespa/eval/tensor/tensor.h>
 #include <ostream>
+#include <cassert>
 
 using vespalib::tensor::Tensor;
+using namespace vespalib::xml;
 
 namespace document {
 
@@ -164,7 +168,6 @@ TensorFieldValue::compare(const FieldValue &other) const
     return ((_tensor.get()  < rhs._tensor.get()) ? -1 : 1);
 }
 
-
 IMPLEMENT_IDENTIFIABLE(TensorFieldValue, FieldValue);
 
 } // document
diff --git a/document/src/vespa/document/fieldvalue/variablemap.cpp b/document/src/vespa/document/fieldvalue/variablemap.cpp
new file mode 100644
index 00000000000..b38b5baeec2
--- /dev/null
+++ b/document/src/vespa/document/fieldvalue/variablemap.cpp
@@ -0,0 +1,59 @@
+#include "variablemap.h"
+#include "fieldvalue.h"
+#include <vespa/vespalib/util/stringfmt.h>
+#include <vespa/vespalib/stllike/asciistream.h>
+
+namespace document::fieldvalue {
+
+IndexValue::IndexValue() : index(-1), key() {}
+IndexValue::IndexValue(int index_) : index(index_), key() {}
+
+bool
+IndexValue::operator==(const IndexValue& other) const {
+    if (key.get() != NULL) {
+        if (other.key.get() != NULL && *key == *other.key) {
+            return true;
+        }
+        return false;
+    }
+
+    return index == other.index;
+}
+
+IndexValue::IndexValue(const FieldValue& key_)
+        : index(-1),
+          key(FieldValue::CP(key_.clone()))
+{ }
+
+IndexValue::IndexValue(const IndexValue & rhs) = default;
+IndexValue & IndexValue::operator = (const IndexValue & rhs) = default;
+
+IndexValue::~IndexValue() { }
+
+vespalib::string
+IndexValue::toString() const {
+    if (key.get() != NULL) {
+        return key->toString();
+    } else {
+        return vespalib::make_string("%d", index);
+    }
+}
+
+VariableMap::VariableMap() {}
+VariableMap::~VariableMap() {}
+VariableMap::VariableMap(const VariableMap & rhs) = default;
+VariableMap & VariableMap::operator = (const VariableMap & rhs) = default;
+
+vespalib::string
+VariableMap::toString() const {
+    vespalib::asciistream out;
+    out << "[ ";
+    for (const auto & entry : *this) {
+        out << entry.first << "=" << entry.second.toString() << " ";
+    }
+    out << "]";
+    return out.str();
+}
+
+}
+
diff --git a/document/src/vespa/document/fieldvalue/variablemap.h b/document/src/vespa/document/fieldvalue/variablemap.h
new file mode 100644
index 00000000000..06679682e0c
--- /dev/null
+++ b/document/src/vespa/document/fieldvalue/variablemap.h
@@ -0,0 +1,47 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/vespalib/util/memory.h>
+#include <vespa/vespalib/stllike/string.h>
+#include <map>
+
+namespace document {
+    class FieldValue;
+}
+
+namespace document::fieldvalue {
+
+class IndexValue {
+public:
+    IndexValue();
+    IndexValue(int index_);
+    IndexValue(const FieldValue& key_);
+    IndexValue(IndexValue && rhs) = default;
+    IndexValue & operator = (IndexValue && rhs) = default;
+    IndexValue(const IndexValue & rhs);
+    IndexValue & operator = (const IndexValue & rhs);
+
+    ~IndexValue();
+
+    vespalib::string toString() const;
+    bool operator==(const IndexValue& other) const;
+
+    int index; // For array
+    vespalib::CloneablePtr<FieldValue> key; // For map/wset
+};
+
+using VariableMapT = std::map<vespalib::string, IndexValue>;
+
+class VariableMap : public VariableMapT {
+public:
+    VariableMap();
+    VariableMap(VariableMap && rhs) = default;
+    VariableMap & operator = (VariableMap && rhs) = default;
+    VariableMap(const VariableMap & rhs);
+    VariableMap & operator = (const VariableMap & rhs);
+    ~VariableMap();
+    vespalib::string toString() const;
+};
+
+}
+\ No newline at end of file
diff --git a/document/src/vespa/document/fieldvalue/weightedsetfieldvalue.cpp b/document/src/vespa/document/fieldvalue/weightedsetfieldvalue.cpp
index 7bf5f5350d3..67e0c8327e6 100644
--- a/document/src/vespa/document/fieldvalue/weightedsetfieldvalue.cpp
+++ b/document/src/vespa/document/fieldvalue/weightedsetfieldvalue.cpp
@@ -2,14 +2,20 @@
 
 #include "weightedsetfieldvalue.h"
 #include <vespa/document/base/exceptions.h>
+#include <vespa/vespalib/util/xmlstream.h>
 #include <ostream>
 
 using vespalib::Identifiable;
+using vespalib::IllegalArgumentException;
+using vespalib::IllegalStateException;
+using namespace vespalib::xml;
 
 /// \todo TODO (was warning):  Find a way to search through internal map without duplicating keys to create shared pointers.
 
 namespace document {
 
+using namespace fieldvalue;
+
 IMPLEMENT_IDENTIFIABLE_ABSTRACT(WeightedSetFieldValue, CollectionFieldValue);
 
 namespace {
@@ -17,9 +23,8 @@ const DataType &getKeyType(const DataType &type) {
     const WeightedSetDataType *wtype =
         Identifiable::cast<const WeightedSetDataType *>(&type);
     if (!wtype) {
-        throw vespalib::IllegalArgumentException(
-                "Cannot generate a weighted set value with non-weighted set "
-                "type " + type.toString() + ".", VESPA_STRLOC);
+        throw IllegalArgumentException("Cannot generate a weighted set value with non-weighted set "
+                                       "type " + type.toString() + ".", VESPA_STRLOC);
     }
     return wtype->getNestedType();
 }
@@ -39,8 +44,7 @@ WeightedSetFieldValue::~WeightedSetFieldValue() { }
 void WeightedSetFieldValue::verifyKey(const FieldValue & v)
 {
     if (!getNestedType().isValueType(v)) {
-        throw InvalidDataTypeException(*v.getDataType(), getNestedType(),
-                                       VESPA_STRLOC);
+        throw InvalidDataTypeException(*v.getDataType(), getNestedType(), VESPA_STRLOC);
     }
 }
 
@@ -92,9 +96,7 @@ WeightedSetFieldValue::increment(const FieldValue& key, int val)
         }
     } else {
         if (it == _map.end()) {
-           throw vespalib::IllegalStateException("Cannot modify non-existing "
-                    "entry in weightedset without createIfNonExistent set",
-                    VESPA_STRLOC);
+           throw IllegalStateException("Cannot modify non-existing entry in weightedset without createIfNonExistent set", VESPA_STRLOC);
         }
         IntFieldValue& fv = static_cast<IntFieldValue&>(*it->second);
         fv.setValue(fv.getValue() + val);
@@ -203,7 +205,7 @@ WeightedSetFieldValue::find(const FieldValue& key)
     return _map.find(key);
 }
 
-FieldValue::IteratorHandler::ModificationStatus
+ModificationStatus
 WeightedSetFieldValue::onIterateNested(PathRange nested, IteratorHandler & handler) const
 {
     return _map.iterateNestedImpl(nested, handler, *this);
diff --git a/document/src/vespa/document/fieldvalue/weightedsetfieldvalue.h b/document/src/vespa/document/fieldvalue/weightedsetfieldvalue.h
index 4d6be1fd35a..5de0bf91614 100644
--- a/document/src/vespa/document/fieldvalue/weightedsetfieldvalue.h
+++ b/document/src/vespa/document/fieldvalue/weightedsetfieldvalue.h
@@ -7,9 +7,9 @@
  */
 #pragma once
 
-#include <vespa/document/fieldvalue/collectionfieldvalue.h>
-#include <vespa/document/fieldvalue/mapfieldvalue.h>
-#include <vespa/document/fieldvalue/intfieldvalue.h>
+#include "collectionfieldvalue.h"
+#include "mapfieldvalue.h"
+#include "intfieldvalue.h"
 #include <vespa/document/datatype/weightedsetdatatype.h>
 #include <map>
 
@@ -33,7 +33,7 @@ private:
     bool addValue(const FieldValue& fval) override { return add(fval, 1); }
     bool containsValue(const FieldValue& val) const override;
     bool removeValue(const FieldValue& val) override;
-    IteratorHandler::ModificationStatus onIterateNested(PathRange nested, IteratorHandler& handler) const override;
+    fieldvalue::ModificationStatus onIterateNested(PathRange nested, fieldvalue::IteratorHandler& handler) const override;
 public:
     typedef std::unique_ptr<WeightedSetFieldValue> UP;
 
diff --git a/document/src/vespa/document/repo/documenttyperepo.cpp b/document/src/vespa/document/repo/documenttyperepo.cpp
index 78d668a4886..93adce76efd 100644
--- a/document/src/vespa/document/repo/documenttyperepo.cpp
+++ b/document/src/vespa/document/repo/documenttyperepo.cpp
@@ -3,7 +3,6 @@
 #include "documenttyperepo.h"
 
 #include <vespa/document/datatype/annotationreferencedatatype.h>
-#include <vespa/document/datatype/annotationtype.h>
 #include <vespa/document/datatype/arraydatatype.h>
 #include <vespa/document/datatype/documenttype.h>
 #include <vespa/document/datatype/mapdatatype.h>
@@ -11,15 +10,10 @@
 #include <vespa/document/datatype/urldatatype.h>
 #include <vespa/document/datatype/weightedsetdatatype.h>
 #include <vespa/document/datatype/referencedatatype.h>
-#include <vespa/vespalib/objects/identifiable.h>
 #include <vespa/vespalib/stllike/hash_map.hpp>
-#include <vespa/vespalib/util/closure.h>
 #include <vespa/vespalib/util/exceptions.h>
-#include <vespa/vespalib/util/stringfmt.h>
 #include <vespa/document/config/config-documenttypes.h>
 #include <fstream>
-#include <memory>
-#include <utility>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".documenttyperepo");
@@ -39,6 +33,20 @@ using vespalib::stringref;
 
 namespace document {
 
+namespace internal {
+
+using DocumentTypeMapT = vespalib::hash_map<int32_t, DataTypeRepo *>;
+
+class DocumentTypeMap : public DocumentTypeMapT
+{
+public:
+    using DocumentTypeMapT::DocumentTypeMapT;
+};
+
+}
+
+using DocumentTypeMap = internal::DocumentTypeMap;
+
 namespace {
 template <typename Container>
 void DeleteContent(Container &c) {
@@ -359,7 +367,6 @@ void addDataTypes(const vector<Datatype> &types, Repo &repo,
     }
 }
 
-typedef hash_map<int32_t, DataTypeRepo *> DocumentTypeMap;
 void addDocumentTypes(const DocumentTypeMap &type_map, Repo &repo) {
     for (DocumentTypeMap::const_iterator
              it = type_map.begin(); it != type_map.end(); ++it) {
@@ -515,49 +522,55 @@ void configureAllRepos(const DocumenttypesConfig::DocumenttypeVector &t,
 
 }  // namespace
 
-DocumentTypeRepo::DocumentTypeRepo() {
-    addDefaultDocument(_doc_types);
+DocumentTypeRepo::DocumentTypeRepo() :
+    _doc_types(std::make_unique<internal::DocumentTypeMap>())
+{
+    addDefaultDocument(*_doc_types);
 }
 
-DocumentTypeRepo::DocumentTypeRepo(const DocumentType & type) {
-    addDefaultDocument(_doc_types);
+DocumentTypeRepo::DocumentTypeRepo(const DocumentType & type) :
+    _doc_types(std::make_unique<internal::DocumentTypeMap>())
+{
+    addDefaultDocument(*_doc_types);
     try {
-        addDataTypeRepo(makeDataTypeRepo(type, _doc_types), _doc_types);
+        addDataTypeRepo(makeDataTypeRepo(type, *_doc_types), *_doc_types);
     } catch (...) {
-        DeleteMapContent(_doc_types);
+        DeleteMapContent(*_doc_types);
         throw;
     }
 }
 
-DocumentTypeRepo::DocumentTypeRepo(const DocumenttypesConfig &config) {
-    addDefaultDocument(_doc_types);
+DocumentTypeRepo::DocumentTypeRepo(const DocumenttypesConfig &config) :
+    _doc_types(std::make_unique<internal::DocumentTypeMap>())
+{
+    addDefaultDocument(*_doc_types);
     try {
-        createAllDocumentTypes(config.documenttype, _doc_types);
-        addAllDocumentTypesToRepos(_doc_types);
-        configureAllRepos(config.documenttype, _doc_types);
+        createAllDocumentTypes(config.documenttype, *_doc_types);
+        addAllDocumentTypesToRepos(*_doc_types);
+        configureAllRepos(config.documenttype, *_doc_types);
     } catch (...) {
-        DeleteMapContent(_doc_types);
+        DeleteMapContent(*_doc_types);
         throw;
     }
 }
 
 DocumentTypeRepo::~DocumentTypeRepo() {
-    DeleteMapContent(_doc_types);
+    DeleteMapContent(*_doc_types);
 }
 
 const DocumentType *DocumentTypeRepo::getDocumentType(int32_t type_id) const {
-    const DataTypeRepo *repo = FindPtr(_doc_types, type_id);
+    const DataTypeRepo *repo = FindPtr(*_doc_types, type_id);
     return repo ? repo->doc_type : nullptr;
 }
 
 const DocumentType *DocumentTypeRepo::getDocumentType(const stringref &name) const {
     DocumentTypeMap::const_iterator it =
-        _doc_types.find(DocumentType::createId(name));
+        _doc_types->find(DocumentType::createId(name));
 
-    if (it != _doc_types.end() && it->second->doc_type->getName() == name) {
+    if (it != _doc_types->end() && it->second->doc_type->getName() == name) {
         return it->second->doc_type;
     }
-    for (it = _doc_types.begin(); it != _doc_types.end(); ++it) {
+    for (it = _doc_types->begin(); it != _doc_types->end(); ++it) {
         if (it->second->doc_type->getName() == name) {
             return it->second->doc_type;
         }
@@ -567,27 +580,27 @@ const DocumentType *DocumentTypeRepo::getDocumentType(const stringref &name) con
 
 const DataType *
 DocumentTypeRepo::getDataType(const DocumentType &doc_type, int32_t id) const {
-    const DataTypeRepo *dt_repo = FindPtr(_doc_types, doc_type.getId());
+    const DataTypeRepo *dt_repo = FindPtr(*_doc_types, doc_type.getId());
     return dt_repo ? dt_repo->repo.lookup(id) : nullptr;
 }
 
 const DataType *
 DocumentTypeRepo::getDataType(
         const DocumentType &doc_type, const stringref &name) const {
-    const DataTypeRepo *dt_repo = FindPtr(_doc_types, doc_type.getId());
+    const DataTypeRepo *dt_repo = FindPtr(*_doc_types, doc_type.getId());
     return dt_repo ? dt_repo->repo.lookup(name) : nullptr;
 }
 
 const AnnotationType *DocumentTypeRepo::getAnnotationType(
         const DocumentType &doc_type, int32_t id) const {
-    const DataTypeRepo *dt_repo = FindPtr(_doc_types, doc_type.getId());
+    const DataTypeRepo *dt_repo = FindPtr(*_doc_types, doc_type.getId());
     return dt_repo ? dt_repo->annotations.lookup(id) : nullptr;
 }
 
 void DocumentTypeRepo::forEachDocumentType(
         Closure1<const DocumentType &> &c) const {
     for (DocumentTypeMap::const_iterator
-             it = _doc_types.begin(); it != _doc_types.end(); ++it) {
+             it = _doc_types->begin(); it != _doc_types->end(); ++it) {
         c.call(*it->second->doc_type);
     }
 }
diff --git a/document/src/vespa/document/repo/documenttyperepo.h b/document/src/vespa/document/repo/documenttyperepo.h
index 55655fe169e..4955f3562db 100644
--- a/document/src/vespa/document/repo/documenttyperepo.h
+++ b/document/src/vespa/document/repo/documenttyperepo.h
@@ -3,7 +3,6 @@
 #pragma once
 
 #include <memory>
-#include <vespa/vespalib/stllike/hash_map.h>
 #include <vespa/vespalib/stllike/string.h>
 #include <vespa/vespalib/util/closure.h>
 
@@ -11,6 +10,7 @@ namespace document {
 
 namespace internal {
     class InternalDocumenttypesType;
+    class DocumentTypeMap;
 }
 
 class AnnotationType;
@@ -19,9 +19,7 @@ class DataTypeRepo;
 class DocumentType;
 
 class DocumentTypeRepo {
-    typedef vespalib::hash_map<int32_t, DataTypeRepo *> DocumentTypeMap;
-
-    DocumentTypeMap _doc_types;
+    std::unique_ptr<internal::DocumentTypeMap> _doc_types;
 
 public:
     using DocumenttypesConfig = const internal::InternalDocumenttypesType;
diff --git a/document/src/vespa/document/select/CMakeLists.txt b/document/src/vespa/document/select/CMakeLists.txt
index 8521c525171..5077c9f0494 100644
--- a/document/src/vespa/document/select/CMakeLists.txt
+++ b/document/src/vespa/document/select/CMakeLists.txt
@@ -1,26 +1,27 @@
 # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 vespa_add_library(document_select OBJECT
     SOURCES
+    bodyfielddetector.cpp
     branch.cpp
+    cloningvisitor.cpp
     compare.cpp
     constant.cpp
+    context.cpp
     doctype.cpp
+    gid_filter.cpp
+    invalidconstant.cpp
     operator.cpp
-    parser.cpp
-    result.cpp
-    value.cpp
-    valuenode.cpp
     orderingselector.cpp
     orderingspecification.cpp
+    parser.cpp
+    result.cpp
+    resultset.cpp
     resultlist.cpp
-    invalidconstant.cpp
-    bodyfielddetector.cpp
     simpleparser.cpp
-    resultset.cpp
     traversingvisitor.cpp
-    cloningvisitor.cpp
-    context.cpp
-    gid_filter.cpp
+    value.cpp
+    valuenode.cpp
+    valuenodes.cpp
     AFTER
     document_documentconfig
 )
diff --git a/document/src/vespa/document/select/bodyfielddetector.cpp b/document/src/vespa/document/select/bodyfielddetector.cpp
index d0ddd0043ef..2b4c9890e45 100644
--- a/document/src/vespa/document/select/bodyfielddetector.cpp
+++ b/document/src/vespa/document/select/bodyfielddetector.cpp
@@ -1,15 +1,12 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "bodyfielddetector.h"
+#include "valuenodes.h"
 #include <vespa/document/base/exceptions.h>
 #include <vespa/document/repo/documenttyperepo.h>
 #include <vespa/document/datatype/documenttype.h>
-#include <vespa/vespalib/util/closure.h>
-#include "valuenode.h"
 
-namespace document {
-
-namespace select {
+namespace document::select {
 
 void
 BodyFieldDetector::detectFieldType(const FieldValueNode *expr,
@@ -35,10 +32,8 @@ BodyFieldDetector::detectFieldType(const FieldValueNode *expr,
 void
 BodyFieldDetector::visitFieldValueNode(const FieldValueNode& expr)
 {
-    _repo.forEachDocumentType(
-            *makeClosure(this, &BodyFieldDetector::detectFieldType, &expr));
+    _repo.forEachDocumentType(*makeClosure(this, &BodyFieldDetector::detectFieldType, &expr));
 }
 
 
-}  // namespace select
-}  // namespace document
+}
diff --git a/document/src/vespa/document/select/branch.cpp b/document/src/vespa/document/select/branch.cpp
index ddbad69b14d..ea0d0ef5646 100644
--- a/document/src/vespa/document/select/branch.cpp
+++ b/document/src/vespa/document/select/branch.cpp
@@ -2,6 +2,8 @@
 
 #include "branch.h"
 #include "visitor.h"
+#include <cassert>
+#include <ostream>
 
 namespace document {
 namespace select {
diff --git a/document/src/vespa/document/select/cloningvisitor.cpp b/document/src/vespa/document/select/cloningvisitor.cpp
index 84188788416..c0a45b758d5 100644
--- a/document/src/vespa/document/select/cloningvisitor.cpp
+++ b/document/src/vespa/document/select/cloningvisitor.cpp
@@ -1,7 +1,7 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "cloningvisitor.h"
-#include "valuenode.h"
+#include "valuenodes.h"
 #include "branch.h"
 #include "compare.h" 
 #include "constant.h"
diff --git a/document/src/vespa/document/select/constant.cpp b/document/src/vespa/document/select/constant.cpp
index 686ab05055c..0e2fdc2ecec 100644
--- a/document/src/vespa/document/select/constant.cpp
+++ b/document/src/vespa/document/select/constant.cpp
@@ -2,9 +2,10 @@
 
 #include "constant.h"
 #include "visitor.h"
+#include <cassert>
+#include <ostream>
 
-namespace document {
-namespace select {
+namespace document::select {
 
 Constant::Constant(const vespalib::stringref & value)
     : Node(value),
@@ -46,13 +47,11 @@ Constant::visit(Visitor &v) const
 
 
 void
-Constant::print(std::ostream& out, bool,
-                const std::string&) const
+Constant::print(std::ostream& out, bool, const std::string&) const
 {
     if (_parentheses) out << '(';
     out << _name;
     if (_parentheses) out << ')';
 }
 
-} // select
-} // document
+}
diff --git a/document/src/vespa/document/select/context.cpp b/document/src/vespa/document/select/context.cpp
index 44d005e9db2..ef29770ddc8 100644
--- a/document/src/vespa/document/select/context.cpp
+++ b/document/src/vespa/document/select/context.cpp
@@ -1,11 +1,12 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "context.h"
+#include "variablemap.h"
+#include <vespa/document/select/value.h>
 
-namespace document {
-namespace select {
+namespace document::select {
 
-Context::Context(void)
+Context::Context()
     : _doc(NULL),
       _docId(NULL),
       _docUpdate(NULL),
@@ -35,5 +36,20 @@ Context::Context(const DocumentUpdate& docUpdate)
 
 Context::~Context() { }
 
-} // select
-} // document
+std::unique_ptr<Value>
+Context::getValue(const vespalib::string & value) const {
+    VariableMap::const_iterator iter = _variables->find(value);
+
+    if (iter != _variables->end()) {
+        return std::make_unique<FloatValue>(iter->second);
+    } else {
+        return std::make_unique<FloatValue>(0.0);
+    }
+}
+
+void
+Context::setVariableMap(std::unique_ptr<VariableMap> map) {
+    _variables = std::move(map);
+}
+
+}
diff --git a/document/src/vespa/document/select/context.h b/document/src/vespa/document/select/context.h
index 11a3a2be51c..0e347e22bb5 100644
--- a/document/src/vespa/document/select/context.h
+++ b/document/src/vespa/document/select/context.h
@@ -2,9 +2,9 @@
 #pragma once
 
 #include <vespa/vespalib/stllike/string.h>
-#include <vespa/vespalib/stllike/hash_map.h>
+#include <memory>
 
-namespace document{
+namespace document {
 
 class Document;
 class DocumentId;
@@ -12,24 +12,26 @@ class DocumentUpdate;
 
 namespace select {
 
-class Context
-{
-public:
-    typedef vespalib::hash_map<vespalib::string, double> VariableMap;
+class Value;
+class VariableMap;
 
+class Context {
+public:
     Context();
-    Context(const Document& doc);
-    Context(const DocumentId& docId);
-    Context(const DocumentUpdate& docUpdate);
+    Context(const Document & doc);
+    Context(const DocumentId & docId);
+    Context(const DocumentUpdate & docUpdate);
     virtual ~Context();
 
-    const Document       * _doc;
-    const DocumentId     * _docId;
-    const DocumentUpdate * _docUpdate;
-    VariableMap            _variables;
-};
-
-} // select
-} // document
+    void setVariableMap(std::unique_ptr<VariableMap> map);
+    std::unique_ptr<Value> getValue(const vespalib::string & value) const;
 
+    const Document *_doc;
+    const DocumentId *_docId;
+    const DocumentUpdate *_docUpdate;
+private:
+    std::unique_ptr<VariableMap> _variables;
+};
 
+}
+}
diff --git a/document/src/vespa/document/select/doctype.cpp b/document/src/vespa/document/select/doctype.cpp
index 3c23b2e5ba1..117f97ea1bc 100644
--- a/document/src/vespa/document/select/doctype.cpp
+++ b/document/src/vespa/document/select/doctype.cpp
@@ -5,9 +5,9 @@
 
 #include <vespa/document/update/documentupdate.h>
 #include <vespa/document/fieldvalue/document.h>
+#include <vespa/document/datatype/documenttype.h>
 
-namespace document {
-namespace select {
+namespace document::select {
 
 namespace {
     bool documentTypeEqualsName(const DocumentType& type,
@@ -88,5 +88,4 @@ DocType::print(std::ostream& out, bool verbose,
     if (_parentheses) out << ')';
 }
 
-} // select
-} // document
+}
diff --git a/document/src/vespa/document/select/gid_filter.cpp b/document/src/vespa/document/select/gid_filter.cpp
index cb194452ead..7630053c16f 100644
--- a/document/src/vespa/document/select/gid_filter.cpp
+++ b/document/src/vespa/document/select/gid_filter.cpp
@@ -1,15 +1,13 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 #include "gid_filter.h"
 #include "node.h"
-#include "parser.h"
 #include "visitor.h"
-#include "valuenode.h"
+#include "valuenodes.h"
 #include "compare.h"
 #include "branch.h"
 #include <vespa/document/base/idstring.h>
 
-namespace document {
-namespace select {
+namespace document::select {
 
 namespace {
 
@@ -156,5 +154,4 @@ GidFilter::GidFilter(const Node& ast_root)
 {
 }
 
-} // select
-} // document
+}
diff --git a/document/src/vespa/document/select/invalidconstant.cpp b/document/src/vespa/document/select/invalidconstant.cpp
index 793e34c7bc5..53e3a6e1647 100644
--- a/document/src/vespa/document/select/invalidconstant.cpp
+++ b/document/src/vespa/document/select/invalidconstant.cpp
@@ -2,9 +2,9 @@
 
 #include "invalidconstant.h"
 #include "visitor.h"
+#include <ostream>
 
-namespace document {
-namespace select {
+namespace document::select {
 
 InvalidConstant::InvalidConstant(const vespalib::stringref & value)
     : Node(value)
@@ -26,13 +26,11 @@ InvalidConstant::visit(Visitor &v) const
 
 
 void
-InvalidConstant::print(std::ostream& out, bool,
-                       const std::string&) const
+InvalidConstant::print(std::ostream& out, bool, const std::string&) const
 {
     if (_parentheses) out << '(';
     out << _name;
     if (_parentheses) out << ')';
 }
 
-} // select
-} // document
+}
diff --git a/document/src/vespa/document/select/node.h b/document/src/vespa/document/select/node.h
index c56203da76c..6ccc1fa5dec 100644
--- a/document/src/vespa/document/select/node.h
+++ b/document/src/vespa/document/select/node.h
@@ -12,7 +12,6 @@
 
 #pragma once
 
-#include <string>
 #include "resultlist.h"
 #include "context.h"
 
@@ -33,15 +32,11 @@ public:
     typedef std::shared_ptr<Node> SP;
 
     Node(const vespalib::stringref & name) : _name(name), _parentheses(false) {}
-    virtual ~Node() {}
+    ~Node() override {}
 
     void setParentheses() { _parentheses = true; }
 
-    void
-    clearParentheses()
-    {
-        _parentheses = false;
-    }
+    void clearParentheses() { _parentheses = false; }
 
     bool hadParentheses() const { return _parentheses; }
 
diff --git a/document/src/vespa/document/select/operator.cpp b/document/src/vespa/document/select/operator.cpp
index b127e29c1ad..3c09a482cf0 100644
--- a/document/src/vespa/document/select/operator.cpp
+++ b/document/src/vespa/document/select/operator.cpp
@@ -4,9 +4,9 @@
 #include <vespa/vespalib/util/regexp.h>
 #include <vespa/vespalib/stllike/asciistream.h>
 #include <vespa/vespalib/stllike/hash_map.hpp>
+#include <cassert>
 
-namespace document {
-namespace select {
+namespace document::select {
 
 Operator::OperatorMap Operator::_operators;
 
@@ -228,5 +228,4 @@ GlobOperator::containsVariables(const vespalib::stringref & expression)
 
 const GlobOperator GlobOperator::GLOB("=");
 
-} // select
-} // document
+}
diff --git a/document/src/vespa/document/select/orderingselector.cpp b/document/src/vespa/document/select/orderingselector.cpp
index 32a1f0a8adf..4195a6c28a2 100644
--- a/document/src/vespa/document/select/orderingselector.cpp
+++ b/document/src/vespa/document/select/orderingselector.cpp
@@ -2,18 +2,16 @@
 
 #include "orderingselector.h"
 #include "node.h"
-#include "valuenode.h"
+#include "valuenodes.h"
 #include "visitor.h"
 #include "compare.h"
 #include "branch.h"
 
-#include <algorithm>
 #include <vespa/document/base/documentid.h>
-#include <vespa/document/base/idstring.h>
 
 namespace document {
 
-using namespace document::select;
+using namespace select;
 
 namespace {
     /**
diff --git a/document/src/vespa/document/select/parser.h b/document/src/vespa/document/select/parser.h
index 61cc6e05876..b25ec951968 100644
--- a/document/src/vespa/document/select/parser.h
+++ b/document/src/vespa/document/select/parser.h
@@ -2,10 +2,8 @@
 
 #pragma once
 
-#include <memory>
-#include <string>
-#include <vespa/document/bucket/bucketidfactory.h>
 #include "node.h"
+#include <vespa/document/bucket/bucketidfactory.h>
 #include <vespa/vespalib/util/exception.h>
 #include <vespa/vespalib/util/sync.h>
 
diff --git a/document/src/vespa/document/select/result.h b/document/src/vespa/document/select/result.h
index ae7b0fdcabf..fbaa8e35eb6 100644
--- a/document/src/vespa/document/select/result.h
+++ b/document/src/vespa/document/select/result.h
@@ -19,11 +19,9 @@
 
 #pragma once
 
-#include <string>
 #include <vespa/document/util/printable.h>
 
-namespace document {
-namespace select {
+namespace document::select {
 
 class Result : public Printable {
 public:
@@ -41,16 +39,9 @@ public:
     const Result& operator!() const;
 
     static const Result& get(bool b) { return (b ? True : False); }
+    static constexpr uint32_t enumRange = 3u;
 
-    static uint32_t
-    enumRange(void)
-    {
-        return 3u;
-    }
-
-    uint32_t
-    toEnum(void) const
-    {
+    uint32_t toEnum() const {
         if (this == &Result::Invalid)
             return 0u;
         if (this == &Result::False)
@@ -60,9 +51,7 @@ public:
         abort();
     }
 
-    static const Result &
-    fromEnum(uint32_t val)
-    {
+    static const Result &fromEnum(uint32_t val) {
         if (val == 0u)
             return Result::Invalid;
         if (val == 1u)
@@ -74,12 +63,9 @@ public:
 
 private:
     Result();
-
-        // Singletons are not copyable
-    Result(const Result&);
-    Result& operator=(const Result&);
+    // Singletons are not copyable
+    Result(const Result&) = delete;
+    Result& operator=(const Result&) = delete;
 };
 
-} // select
-} // document
-
+}
diff --git a/document/src/vespa/document/select/resultlist.cpp b/document/src/vespa/document/select/resultlist.cpp
index f8224b7a158..2cde06fa2f2 100644
--- a/document/src/vespa/document/select/resultlist.cpp
+++ b/document/src/vespa/document/select/resultlist.cpp
@@ -15,8 +15,7 @@ ResultList::ResultList(const Result& result) {
 }
 
 void
-ResultList::add(const FieldValue::IteratorHandler::VariableMap& variables,
-                const Result& result)
+ResultList::add(const fieldvalue::VariableMap& variables, const Result& result)
 {
     _results.push_back(ResultPair(variables, &result));
 }
@@ -27,8 +26,7 @@ ResultList::print(std::ostream& out, bool, const std::string&) const
     out << "ResultList(";
     for (uint32_t i = 0; i < _results.size(); i++) {
         if (!_results[i].first.empty()) {
-            out << FieldValue::IteratorHandler::toString(_results[i].first)
-                << " => ";
+            out << _results[i].first.toString() << " => ";
         }
         out << _results[i].second->toString() << " ";
     }
@@ -66,15 +64,12 @@ ResultList::combineResults() const {
 
 bool
 ResultList::combineVariables(
-        FieldValue::IteratorHandler::VariableMap& output,
-        const FieldValue::IteratorHandler::VariableMap& input) const
+        fieldvalue::VariableMap& output,
+        const fieldvalue::VariableMap& input) const
 {
     // First, verify that all variables are overlapping
-    for (FieldValue::IteratorHandler::VariableMap::const_iterator iter
-            = output.begin(); iter != output.end(); iter++)
-    {
-        FieldValue::IteratorHandler::VariableMap::const_iterator found(
-                input.find(iter->first));
+    for (fieldvalue::VariableMap::const_iterator iter = output.begin(); iter != output.end(); iter++) {
+        fieldvalue::VariableMap::const_iterator found(input.find(iter->first));
 
         if (found != input.end()) {
             if (!(found->second == iter->second)) {
@@ -83,11 +78,8 @@ ResultList::combineVariables(
         }
     }
 
-    for (FieldValue::IteratorHandler::VariableMap::const_iterator iter
-            = input.begin(); iter != input.end(); iter++)
-    {
-        FieldValue::IteratorHandler::VariableMap::const_iterator found(
-                output.find(iter->first));
+    for (fieldvalue::VariableMap::const_iterator iter = input.begin(); iter != input.end(); iter++) {
+        fieldvalue::VariableMap::const_iterator found(output.find(iter->first));
         if (found != output.end()) {
             if (!(found->second == iter->second)) {
                 return false;
@@ -95,9 +87,7 @@ ResultList::combineVariables(
         }
     }
     // Ok, variables are overlapping. Add all variables from input to output.
-    for (FieldValue::IteratorHandler::VariableMap::const_iterator iter
-            = input.begin(); iter != input.end(); iter++)
-    {
+    for (fieldvalue::VariableMap::const_iterator iter = input.begin(); iter != input.end(); iter++) {
         output[iter->first] = iter->second;
     }
 
@@ -112,7 +102,7 @@ ResultList::operator&&(const ResultList& other) const
     // TODO: optimize
     for (const auto & it : _results) {
         for (const auto & it2 : other._results) {
-            FieldValue::IteratorHandler::VariableMap vars = it.first;
+            fieldvalue::VariableMap vars = it.first;
 
             if (combineVariables(vars, it2.first)) {
                 result.add(vars, *it.second && *it2.second);
@@ -131,7 +121,7 @@ ResultList::operator||(const ResultList& other) const
     // TODO: optimize
     for (const auto & it : _results) {
         for (const auto & it2 : other._results) {
-            FieldValue::IteratorHandler::VariableMap vars = it.first;
+            fieldvalue::VariableMap vars = it.first;
 
             if (combineVariables(vars, it2.first)) {
                 result.add(vars, *it.second || *it2.second);
diff --git a/document/src/vespa/document/select/resultlist.h b/document/src/vespa/document/select/resultlist.h
index 6a3cc1f78d1..a00419d9196 100644
--- a/document/src/vespa/document/select/resultlist.h
+++ b/document/src/vespa/document/select/resultlist.h
@@ -1,16 +1,15 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 #pragma once
 
-#include <vector>
-#include <vespa/document/fieldvalue/fieldvalue.h>
 #include "result.h"
+#include <vector>
+#include <vespa/document/fieldvalue/variablemap.h>
 
-namespace document {
-namespace select {
+namespace document::select {
 
 class ResultList : public Printable {
 public:
-    typedef FieldValue::IteratorHandler::VariableMap VariableMap;
+    using VariableMap = fieldvalue::VariableMap;
     typedef std::pair<VariableMap, const Result*> ResultPair;
     typedef std::vector<ResultPair> Results;
     typedef Results::iterator iterator;
@@ -49,7 +48,7 @@ public:
 
 private:
     Results _results;
-    bool combineVariables(VariableMap& output, const FieldValue::IteratorHandler::VariableMap& input) const;
+    bool combineVariables(VariableMap& output, const VariableMap& input) const;
 };
 
 inline bool operator==(const ResultList& list, const Result& other) {
@@ -61,5 +60,4 @@ inline bool operator!=(const ResultList& list, const Result& other) {
 }
 
 }
-}
 
diff --git a/document/src/vespa/document/select/resultset.cpp b/document/src/vespa/document/select/resultset.cpp
index f7fd3e95fbd..ddde4b366a3 100644
--- a/document/src/vespa/document/select/resultset.cpp
+++ b/document/src/vespa/document/select/resultset.cpp
@@ -2,9 +2,7 @@
 
 #include "resultset.h"
 
-namespace document {
-
-namespace select {
+namespace document::select {
 
 std::vector<ResultSet> ResultSet::_ands;
 std::vector<ResultSet> ResultSet::_ors;
@@ -15,9 +13,9 @@ std::vector<ResultSet> ResultSet::_nots;
  * inputs.
  */
 void
-ResultSet::preCalc(void)
+ResultSet::preCalc()
 {
-    uint32_t erange = Result::enumRange();
+    uint32_t erange = Result::enumRange;
     uint32_t range = illegalMask();
     _ands.resize(range * range);
     _ors.resize(range * range);
@@ -53,13 +51,12 @@ ResultSet::preCalc(void)
 }
 
 
-namespace
-{
+namespace {
 
 class Precalc
 {
 public:
-    Precalc(void)
+    Precalc()
     {
         ResultSet::preCalc();
     }
@@ -70,5 +67,4 @@ Precalc precalc;
 
 }
 
-} // select
-} // document
+}
diff --git a/document/src/vespa/document/select/resultset.h b/document/src/vespa/document/select/resultset.h
index b3e4d8c4066..125582faba3 100644
--- a/document/src/vespa/document/select/resultset.h
+++ b/document/src/vespa/document/select/resultset.h
@@ -1,12 +1,10 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 #pragma once
 
-#include <string>
-#include <vector>
 #include "result.h"
+#include <vector>
 
-namespace document {
-namespace select {
+namespace document::select {
 
 /*
  * Contains possible values of operations returning a Result.
@@ -29,7 +27,7 @@ public:
     }
 
     static uint32_t illegalMask() {
-        return (1u << Result::enumRange());
+        return (1u << Result::enumRange);
     }
 
     void add(const Result &rhs) {
@@ -54,26 +52,24 @@ public:
 
     // calculcate set of results emitted by document selection and operator.
     ResultSet calcAnd(const ResultSet &rhs) const {
-        return _ands[(_val << Result::enumRange()) | rhs._val];
+        return _ands[(_val << Result::enumRange) | rhs._val];
     }
 
     // calculcate set of results emitted by document selection or operator.
     ResultSet calcOr(const ResultSet &rhs) const {
-        return _ors[(_val << Result::enumRange()) | rhs._val];
+        return _ors[(_val << Result::enumRange) | rhs._val];
     }
 
     // calculcate set of results emitted by document selection not operator.
-    ResultSet calcNot(void) const { return _nots[_val]; }
+    ResultSet calcNot() const { return _nots[_val]; }
 
-    void clear(void) { _val = 0; }
-    void fill(void) { _val = illegalMask() - 1; }
-    static void preCalc(void);
+    void clear() { _val = 0; }
+    void fill() { _val = illegalMask() - 1; }
+    static void preCalc();
 private:
     // precalc helper methods
     void pcnext(void) { ++_val; }
     bool pcvalid(void) const { return _val < illegalMask(); }
 };
 
-} // select
-} // document
-
+}
diff --git a/document/src/vespa/document/select/simpleparser.h b/document/src/vespa/document/select/simpleparser.h
index e6457c3dec7..c52aed0f367 100644
--- a/document/src/vespa/document/select/simpleparser.h
+++ b/document/src/vespa/document/select/simpleparser.h
@@ -2,17 +2,11 @@
 
 #pragma once
 
-#include <vespa/vespalib/stllike/string.h>
 #include "node.h"
-#include "valuenode.h"
+#include "valuenodes.h"
 #include "operator.h"
-#include <vespa/document/bucket/bucketidfactory.h>
 
-namespace document {
-
-namespace select {
-
-namespace simple {
+namespace document::select::simple {
 
 class Parser {
 public:
@@ -89,9 +83,4 @@ private:
     const BucketIdFactory & _bucketIdFactory;
 };
 
-
-
-} // simple
-} // select
-} // parser
-
+}
diff --git a/document/src/vespa/document/select/traversingvisitor.cpp b/document/src/vespa/document/select/traversingvisitor.cpp
index 7f85d8b7ebb..2f2468e5afc 100644
--- a/document/src/vespa/document/select/traversingvisitor.cpp
+++ b/document/src/vespa/document/select/traversingvisitor.cpp
@@ -1,13 +1,11 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "traversingvisitor.h"
-#include "valuenode.h"
+#include "valuenodes.h"
 #include "branch.h"
 #include "compare.h" 
 
-namespace document {
-
-namespace select {
+namespace document::select {
 
 void
 TraversingVisitor::visitAndBranch(const And &expr)
@@ -127,5 +125,3 @@ TraversingVisitor::visitInvalidValueNode(const InvalidValueNode &)
 }
 
 }
-
-}
diff --git a/document/src/vespa/document/select/value.cpp b/document/src/vespa/document/select/value.cpp
index 7638ba3c36e..3540935f8c7 100644
--- a/document/src/vespa/document/select/value.cpp
+++ b/document/src/vespa/document/select/value.cpp
@@ -2,7 +2,7 @@
 
 #include "value.h"
 #include "operator.h"
-#include <cstdint>
+#include <vespa/document/fieldvalue/fieldvalue.h>
 
 namespace document {
 namespace select {
diff --git a/document/src/vespa/document/select/value.h b/document/src/vespa/document/select/value.h
index 15856648985..b26e959ea1d 100644
--- a/document/src/vespa/document/select/value.h
+++ b/document/src/vespa/document/select/value.h
@@ -1,4 +1,5 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
 /**
  * @class document::select::Value
  * @ingroup select
@@ -190,7 +191,7 @@ inline ResultList FloatValue::operator==(const FloatValue& value) const
 class ArrayValue : public Value
 {
 public:
-    typedef std::pair<FieldValue::IteratorHandler::VariableMap, Value::SP> VariableValue;
+    using VariableValue = std::pair<fieldvalue::VariableMap, Value::SP>;
 
     ArrayValue(const std::vector<VariableValue>& values);
 
diff --git a/document/src/vespa/document/select/valuenode.cpp b/document/src/vespa/document/select/valuenode.cpp
index ebf0a466ad9..79b795ca76f 100644
--- a/document/src/vespa/document/select/valuenode.cpp
+++ b/document/src/vespa/document/select/valuenode.cpp
@@ -1,1168 +1,20 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 #include "valuenode.h"
-#include "visitor.h"
-#include "parser.h"
-#include <vespa/document/base/exceptions.h>
-#include <vespa/document/fieldvalue/fieldvalues.h>
-#include <vespa/vespalib/util/md5.h>
-#include <vespa/document/util/stringutil.h>
-#include <vespa/vespalib/text/lowercase.h>
-#include <regex>
-#include <iomanip>
-
-
-#include <vespa/log/log.h>
-LOG_SETUP(".document.select.valuenode");
-
-namespace document {
-namespace select {
-
-namespace {
-    static const std::regex FIELD_NAME_REGEX("^([_A-Za-z][_A-Za-z0-9]*).*");
-}
-
-namespace {
-    bool documentTypeEqualsName(const DocumentType& type, const vespalib::stringref& name)
-    {
-        if (type.getName() == name) return true;
-        for (std::vector<const DocumentType *>::const_iterator it
-                = type.getInheritedTypes().begin();
-             it != type.getInheritedTypes().end(); ++it)
-        {
-            if (documentTypeEqualsName(**it, name)) return true;
-        }
-        return false;
-    }
-}
-
-InvalidValueNode::InvalidValueNode(const vespalib::stringref & name)
-    : _name(name)
-{ }
-
-
-void
-InvalidValueNode::visit(Visitor &visitor) const
-{
-    visitor.visitInvalidValueNode(*this);
-}
-
-
-void
-InvalidValueNode::print(std::ostream& out, bool verbose,
-                        const std::string& indent) const
-{
-    (void) verbose; (void) indent;
-    if (hadParentheses()) out << '(';
-    out << _name;
-    if (hadParentheses()) out << ')';
-}
-
-NullValueNode::NullValueNode(const vespalib::stringref & name)
-    : _name(name)
-{ }
-
-
-void
-NullValueNode::visit(Visitor &visitor) const
-{
-    visitor.visitNullValueNode(*this);
-}
-
-
-void
-NullValueNode::print(std::ostream& out, bool verbose,
-                        const std::string& indent) const
-{
-    (void) verbose; (void) indent;
-    if (hadParentheses()) out << '(';
-    out << _name;
-    if (hadParentheses()) out << ')';
-}
-
-StringValueNode::StringValueNode(const vespalib::stringref & val)
-    : _value(val)
-{
-}
-
-
-void
-StringValueNode::visit(Visitor &visitor) const
-{
-    visitor.visitStringValueNode(*this);
-}
-
-
-void
-StringValueNode::print(std::ostream& out, bool verbose,
-                       const std::string& indent) const
-{
-    (void) verbose; (void) indent;
-    if (hadParentheses()) out << '(';
-    out << "\"" << StringUtil::escape(_value) << "\"";
-    if (hadParentheses()) out << ')';
-}
-
-
-void
-IntegerValueNode::visit(Visitor &visitor) const
-{
-    visitor.visitIntegerValueNode(*this);
-}
-
-
-void
-IntegerValueNode::print(std::ostream& out, bool verbose,
-                       const std::string& indent) const
-{
-    (void) verbose; (void) indent;
-    if (hadParentheses()) out << '(';
-    out << _value;
-    if (hadParentheses()) out << ')';
-}
-
-int64_t
-CurrentTimeValueNode::getValue() const
-{
-    struct timeval mytime;
-    gettimeofday(&mytime, 0);
-    return mytime.tv_sec;
-}
-
-
-void
-CurrentTimeValueNode::visit(Visitor &visitor) const
-{
-    visitor.visitCurrentTimeValueNode(*this);
-}
-
-
-void
-CurrentTimeValueNode::print(std::ostream& out, bool verbose,
-                            const std::string& indent) const
-{
-    (void) verbose; (void) indent;
-    out << "now()";
-}
-
-std::unique_ptr<Value>
-VariableValueNode::getValue(const Context& context) const {
-    VariableMap::const_iterator iter = context._variables.find(_value);
-
-    if (iter != context._variables.end()) {
-        return std::unique_ptr<Value>(new FloatValue(iter->second));
-    } else {
-        return std::unique_ptr<Value>(new FloatValue(0.0));
-    }
-}
-
-
-void
-VariableValueNode::visit(Visitor &visitor) const
-{
-    visitor.visitVariableValueNode(*this);
-}
-
-
-void
-VariableValueNode::print(std::ostream& out, bool verbose,
-                       const std::string& indent) const
-{
-    (void) verbose; (void) indent;
-    if (hadParentheses()) out << '(';
-    out << "$" << _value;
-    if (hadParentheses()) out << ')';
-}
-
-
-void
-FloatValueNode::visit(Visitor &visitor) const
-{
-    visitor.visitFloatValueNode(*this);
-}
-
-
-void
-FloatValueNode::print(std::ostream& out, bool verbose,
-                       const std::string& indent) const
-{
-    (void) verbose; (void) indent;
-    if (hadParentheses()) out << '(';
-    out << _value;
-    if (hadParentheses()) out << ')';
-}
-
-FieldValueNode::FieldValueNode(const vespalib::string& doctype,
-                               const vespalib::string& fieldExpression)
-    : _doctype(doctype),
-      _fieldExpression(fieldExpression),
-      _fieldName(extractFieldName(fieldExpression))
-{
-}
-
-FieldValueNode::FieldValueNode(const FieldValueNode &) = default;
-FieldValueNode & FieldValueNode::operator = (const FieldValueNode &) = default;
-
-FieldValueNode::~FieldValueNode() {}
-vespalib::string
-FieldValueNode::extractFieldName(const std::string & fieldExpression) {
-    std::smatch match;
-
-    if (std::regex_match(fieldExpression, match, FIELD_NAME_REGEX) && match[1].matched) {
-        return vespalib::string(match[1].first, match[1].second);
-    }
-
-    throw ParsingFailedException("Fatal: could not extract field name from field expression '" + fieldExpression + "'");
-}
-
-void
-FieldValueNode::initFieldPath(const DocumentType& type) const {
-    if (_fieldPath.size() == 0) {
-        FieldPath::UP path(type.buildFieldPath(_fieldExpression));
-        if (!path.get()) {
-            throw FieldNotFoundException(
-                    vespalib::make_string("Could not create field path for doc type: '%s' field: '%s'",
-                                          type.toString().c_str(), _fieldExpression.c_str()),
-                    VESPA_STRLOC);
-        }
-        _fieldPath = *path;
-    }
-}
-
-std::unique_ptr<Value>
-FieldValueNode::getValue(const Context& context) const
-{
-    if (context._doc == NULL) {
-        return std::unique_ptr<Value>(new InvalidValue());
-    }
-
-    const Document& doc = *context._doc;
-
-    if (!documentTypeEqualsName(doc.getType(), _doctype)) {
-        return std::unique_ptr<Value>(new InvalidValue());
-    }
-    try{
-        initFieldPath(doc.getType());
-
-        IteratorHandler handler;
-        doc.iterateNested(_fieldPath.getFullRange(), handler);
-
-        if (handler.hasSingleValue()) {
-            return handler.getSingleValue();
-        } else {
-            const std::vector<ArrayValue::VariableValue>& values = handler.getValues();
-
-            if (values.size() == 0) {
-                return std::unique_ptr<Value>(new NullValue());
-            } else {
-                return std::unique_ptr<Value>(new ArrayValue(handler.getValues()));
-            }
-        }
-    } catch (vespalib::IllegalArgumentException& e) {
-        LOG(warning, "Caught exception while fetching field from document: %s", e.what());
-        return std::unique_ptr<Value>(new InvalidValue());
-    } catch (FieldNotFoundException& e) {
-        LOG(warning, "Tried to compare to field %s, not found in document type", _fieldExpression.c_str());
-        return std::unique_ptr<Value>(new InvalidValue());
-    }
-}
-
-FieldValueNode::IteratorHandler::IteratorHandler() { }
-FieldValueNode::IteratorHandler::~IteratorHandler() { }
-
-bool
-FieldValueNode::IteratorHandler::hasSingleValue() const {
-    return _firstValue.get() && (_values.size() == 0);
-}
-
-std::unique_ptr<Value>
-FieldValueNode::IteratorHandler::getSingleValue() {
-    return std::move(_firstValue);
-}
-
-const std::vector<ArrayValue::VariableValue>&
-FieldValueNode::IteratorHandler::getValues() {
-    if (_firstValue.get()) {
-        _values.insert(_values.begin(), ArrayValue::VariableValue(FieldValue::IteratorHandler::VariableMap(), Value::SP(_firstValue.release())));
-    }
-
-    return _values;
-}
-
-void
-FieldValueNode::IteratorHandler::onPrimitive(uint32_t fid, const Content& fv) {
-    (void) fid;
-    if (!_firstValue && getVariables().empty()) {
-        _firstValue = getInternalValue(fv.getValue());
-    } else {
-        _values.emplace_back(getVariables(), Value::SP(getInternalValue(fv.getValue()).release()));
-    }
-}
-
-std::unique_ptr<Value>
-FieldValueNode::IteratorHandler::getInternalValue(const FieldValue& fval) const
-{
-    switch(fval.getClass().id()) {
-        case document::IntFieldValue::classId:
-        {
-            const IntFieldValue& val(dynamic_cast<const IntFieldValue&>(fval));
-            return std::make_unique<IntegerValue>(val.getAsInt(), false);
-        }
-        case document::ByteFieldValue::classId:
-        {
-            const ByteFieldValue& val(dynamic_cast<const ByteFieldValue&>(fval));
-            return std::make_unique<IntegerValue>(val.getAsByte(), false);
-        }
-        case LongFieldValue::classId:
-        {
-            const LongFieldValue& val(dynamic_cast<const LongFieldValue&>(fval));
-            return std::make_unique<IntegerValue>(val.getAsLong(), false);
-        }
-        case FloatFieldValue::classId:
-        {
-            const FloatFieldValue& val(dynamic_cast<const FloatFieldValue&>(fval));
-            return std::make_unique<FloatValue>(val.getAsFloat());
-        }
-        case DoubleFieldValue::classId:
-        {
-            const DoubleFieldValue& val(dynamic_cast<const DoubleFieldValue&>(fval));
-            return std::make_unique<FloatValue>(val.getAsDouble());
-        }
-        case StringFieldValue::classId:
-        {
-            const StringFieldValue& val(dynamic_cast<const StringFieldValue&>(fval));
-            return std::make_unique<StringValue>(val.getAsString());
-        }
-        case ArrayFieldValue::classId:
-        {
-            const ArrayFieldValue& val(dynamic_cast<const ArrayFieldValue&>(fval));
-            if (val.size() == 0) {
-                return std::make_unique<NullValue>();
-            } else {
-                std::vector<ArrayValue::VariableValue> values;
-                // TODO: Array comparison.
-                return std::make_unique<ArrayValue>(values);
-            }
-        }
-        case StructFieldValue::classId:
-        {
-            const StructFieldValue& val(dynamic_cast<const StructFieldValue&>(fval));
-            if (val.empty()) {
-                return std::make_unique<NullValue>();
-            } else {
-                StructValue::ValueMap values;
-                for (StructFieldValue::const_iterator it(val.begin()); it != val.end(); ++it) {
-                    FieldValue::UP fv(val.getValue(it.field()));
-                    values[it.field().getName()] = Value::SP(getInternalValue(*fv).release());
-                }
-                return std::make_unique<StructValue>(values);
-            }
-        }
-        case MapFieldValue::classId:
-        {
-            const MapFieldValue& val(static_cast<const MapFieldValue&>(fval));
-            if (val.isEmpty()) {
-                return std::make_unique<NullValue>();
-            } else {
-                std::vector<ArrayValue::VariableValue> values;
-                // TODO: Map comparison
-                return std::make_unique<ArrayValue>(values);
-            }
-        }
-    }
-    LOG(warning, "Tried to use unsupported datatype %s in field comparison",
-                 fval.getDataType()->toString().c_str());
-    return std::make_unique<InvalidValue>();
-}
-
-
-void
-FieldValueNode::visit(Visitor &visitor) const
-{
-    visitor.visitFieldValueNode(*this);
-}
-
-
-void
-FieldValueNode::print(std::ostream& out, bool verbose,
-                       const std::string& indent) const
-{
-    (void) verbose; (void) indent;
-    if (hadParentheses()) out << '(';
-    out << _doctype << "." << _fieldExpression;
-    if (hadParentheses()) out << ')';
-}
-
-
-std::unique_ptr<Value>
-FieldValueNode::traceValue(const Context &context, std::ostream& out) const
-{
-    if (context._doc == NULL) {
-        return defaultTrace(getValue(context), out);
-    }
-    const Document &doc(*context._doc);
-    if (!documentTypeEqualsName(doc.getType(), _doctype)) {
-        out << "Document is of type " << doc.getType() << " which isn't a "
-            << _doctype << " document, thus resolving invalid.\n";
-        return std::unique_ptr<Value>(new InvalidValue());
-    }
-    try{
-        initFieldPath(doc.getType());
-
-        IteratorHandler handler;
-        doc.iterateNested(_fieldPath.getFullRange(), handler);
-
-        if (handler.hasSingleValue()) {
-            return handler.getSingleValue();
-        } else {
-            const std::vector<ArrayValue::VariableValue>& values = handler.getValues();
-
-            if (values.size() == 0) {
-                return std::unique_ptr<Value>(new NullValue());
-            } else {
-                return std::unique_ptr<Value>(new ArrayValue(handler.getValues()));
-            }
-        }
-    } catch (FieldNotFoundException& e) {
-        LOG(warning, "Tried to compare to field %s, not found in document type",
-                     _fieldExpression.c_str());
-        out << "Field not found in document type " << doc.getType()
-            << ". Returning invalid.\n";
-        return std::unique_ptr<Value>(new InvalidValue());
-    }
-}
-
-IdValueNode::IdValueNode(const BucketIdFactory& bucketIdFactory,
-                         const vespalib::stringref & name, const vespalib::stringref & type,
-                         int widthBits, int divisionBits)
-    : _bucketIdFactory(bucketIdFactory),
-      _id(name),
-      _typestring(type),
-      _type(ALL),
-      _widthBits(widthBits),
-      _divisionBits(divisionBits)
-{
-    if (type.length() > 2) switch (type[0]) {
-    case 'b': _type = BUCKET;
-        break;
-    case 'n': _type = NS;
-        break;
-    case 'g':
-        if (type[1] == 'r') {
-            _type = GROUP;
-        } else if (type[1] == 'i') {
-            _type = GID;
-        }
-        break;
-    case 's': {
-        if (type[1] == 'c') { _type = SCHEME; } else { _type = SPEC; }
-        break;
-    }
-    case 't':
-        _type = TYPE;
-        break;
-    case 'u':
-        _type = USER;
-        break;
-    case 'o':
-        _type = ORDER;
-        break;
-    }
-}
-
-
-std::unique_ptr<Value>
-IdValueNode::getValue(const Context& context) const
-{
-    if (context._doc != NULL) {
-        return getValue(context._doc->getId());
-    } else if (context._docId != NULL) {
-        return getValue(*context._docId);
-    } else {
-        return getValue(context._docUpdate->getId());
-    }
-}
-
-
-std::unique_ptr<Value>
-IdValueNode::getValue(const DocumentId& id) const
-{
-    vespalib::string value;
-    switch (_type) {
-    case BUCKET:
-        return std::unique_ptr<Value>(
-            new IntegerValue(
-                _bucketIdFactory.getBucketId(id).getId(), true));
-    case NS:
-        value = id.getScheme().getNamespace(); break;
-    case SCHEME:
-        value = id.getScheme().getTypeName(id.getScheme().getType());
-        break;
-    case TYPE:
-        if (id.getScheme().hasDocType()) {
-            value = id.getScheme().getDocType();
-        } else {
-            return std::unique_ptr<Value>(new InvalidValue);
-        }
-        break;
-    case SPEC:
-        value = id.getScheme().getNamespaceSpecific();
-        break;
-    case ALL:
-        value = id.getScheme().toString();
-        break;
-    case GROUP:
-        if (id.getScheme().hasGroup()) {
-            value = id.getScheme().getGroup();
-        } else {
-            fprintf(stderr, "***** Returning invalid value for %s\n",
-                    id.toString().c_str());
-            return std::unique_ptr<Value>(new InvalidValue);
-        }
-        break;
-    case GID:
-        value = id.getGlobalId().toString();
-        break;
-    case ORDER:
-        if (id.getScheme().getType() == IdString::ORDERDOC) {
-            const OrderDocIdString& ods(
-                    static_cast<const OrderDocIdString&>(id.getScheme()));
-            if (ods.getWidthBits() == _widthBits
-                && ods.getDivisionBits() == _divisionBits)
-            {
-                return std::unique_ptr<Value>(new IntegerValue(
-                        static_cast<const OrderDocIdString&>(id.getScheme())
-                            .getOrdering(), false));
-            }
-        }
-        return std::unique_ptr<Value>(new InvalidValue());
-    case USER:
-        if (id.getScheme().hasNumber()) {
-            return std::unique_ptr<Value>(
-                    new IntegerValue(id.getScheme().getNumber(), false));
-        } else {
-            return std::unique_ptr<Value>(new InvalidValue);
-        }
-    }
-
-    return std::unique_ptr<Value>(new StringValue(value));
-}
-
-
-std::unique_ptr<Value>
-IdValueNode::traceValue(const Context& context,
-                        std::ostream &out) const
-{
-    if (context._doc != NULL) {
-        return traceValue(context._doc->getId(), out);
-    } else if (context._docId != NULL) {
-        return traceValue(*context._docId, out);
-    } else {
-        return traceValue(context._docUpdate->getId(), out);
-    }
-}
-
-
-std::unique_ptr<Value>
-IdValueNode::traceValue(const DocumentId& id, std::ostream& out) const
-{
-    vespalib::string value;
-    switch (_type) {
-    case BUCKET:
-        {
-            document::BucketId bucket(_bucketIdFactory.getBucketId(id));
-            std::unique_ptr<Value> result(
-                new IntegerValue(bucket.getId(), true));
-            out << "Found id.bucket specification. Resolved to "
-                << bucket.toString() << ".\n";
-            return result;
-        }
-    case NS:
-        value = id.getScheme().getNamespace();
-        out << "Resolved id.namespace to value\"" << value << "\".\n";
-        break;
-    case SCHEME:
-        value = id.getScheme().getTypeName(id.getScheme().getType());
-        out << "Resolved id.scheme to value\"" << value << "\".\n";
-        break;
-    case TYPE:
-        if (id.getScheme().hasDocType()) {
-            value = id.getScheme().getDocType();
-            out << "Resolved id.type to value\"" << value << "\".\n";
-        } else {
-            out << "Could not resolve type of doc " << id << ".\n";
-            return std::unique_ptr<Value>(new InvalidValue);
-        }
-        break;
-    case SPEC:
-        value = id.getScheme().getNamespaceSpecific();
-        out << "Resolved id.specific to value\"" << value << "\".\n";
-        break;
-    case ALL:
-        value = id.getScheme().toString();
-        out << "Resolved id to \"" << value << "\".\n";
-        break;
-    case GROUP:
-        if (id.getScheme().hasGroup()) {
-            value = id.getScheme().getGroup();
-            out << "Resolved group of doc (type " << id.getScheme().getType()
-                << ") to \"" << value << "\".\n";
-        } else {
-            out << "Can't resolve group of doc \"" << id << "\".\n";
-            return std::unique_ptr<Value>(new InvalidValue);
-        }
-        break;
-    case GID:
-        value = id.getGlobalId().toString();
-        out << "Resolved gid to \"" << value << "\".\n";
-        break;
-    case ORDER:
-        if (id.getScheme().getType() == IdString::ORDERDOC) {
-            const OrderDocIdString& ods(
-                    static_cast<const OrderDocIdString&>(id.getScheme()));
-            if (ods.getWidthBits() == _widthBits
-                && ods.getDivisionBits() == _divisionBits)
-            {
-                std::unique_ptr<Value> result(new IntegerValue(
-                        static_cast<const OrderDocIdString&>(id.getScheme())
-                            .getOrdering(), false));
-                out << "Resolved id.order to int " << *result << "\n";
-                return result;
-            }
-        }
-        out << "Could not resolve id.order(" << _widthBits << ", "
-            << _divisionBits << ") of doc " << id << ".\n";
-        return std::unique_ptr<Value>(new InvalidValue());
-    case USER:
-        if (id.getScheme().hasNumber()) {
-            std::unique_ptr<Value> result(
-                    new IntegerValue(id.getScheme().getNumber(), false));
-            out << "Resolved user of doc type " << id.getScheme().getType()
-                << " to " << *result << ".\n";
-            return result;
-        } else {
-            out << "Could not resolve user of doc " << id << ".\n";
-            return std::unique_ptr<Value>(new InvalidValue);
-        }
-    }
-
-    return std::unique_ptr<Value>(new StringValue(value));
-}
-
-
-void
-IdValueNode::visit(Visitor &visitor) const
-{
-    visitor.visitIdValueNode(*this);
-}
-
-
-void
-IdValueNode::print(std::ostream& out, bool verbose,
-                   const std::string& indent) const
-{
-    (void) verbose; (void) indent;
-    if (hadParentheses()) out << '(';
-    out << _id;
-    if (_type != ALL) {
-        out << '.' << _typestring;
-    }
-    if (_type == ORDER) {
-        out << "(" << _widthBits << "," << _divisionBits << ")";
-    }
-    if (hadParentheses()) out << ')';
-}
-
-SearchColumnValueNode::SearchColumnValueNode(
-        const BucketIdFactory& bucketIdFactory,
-        const vespalib::stringref & name, int numColumns)
-    : _bucketIdFactory(bucketIdFactory),
-      _id(name),
-      _numColumns(numColumns),
-      _distribution(_numColumns, 16)
-{
-}
-
-int64_t
-SearchColumnValueNode::getValue(const BucketId& id) const
-{
-    return _distribution.getColumn(id);
-}
-
-
-std::unique_ptr<Value>
-SearchColumnValueNode::getValue(const Context& context) const
-{
-    if (context._doc != NULL) {
-        return getValue(context._doc->getId());
-    } else if (context._docId != NULL) {
-        return getValue(*context._docId);
-    } else {
-        return getValue(context._docUpdate->getId());
-    }
-}
-
-
-std::unique_ptr<Value>
-SearchColumnValueNode::getValue(const DocumentId& id) const
-{
-    return std::unique_ptr<Value>(new IntegerValue(
-                getValue(_bucketIdFactory.getBucketId(id)), false));
-}
+#include <ostream>
 
+namespace document::select {
 
 std::unique_ptr<Value>
-SearchColumnValueNode::traceValue(const Context& context,
-                                  std::ostream &out) const
-{
-    if (context._doc != NULL) {
-        return traceValue(context._doc->getId(), out);
-    } else if (context._docId != NULL) {
-        return traceValue(*context._docId, out);
-    } else {
-        return traceValue(context._docUpdate->getId(), out);
-    }
+ValueNode::traceValue(const Context &context, std::ostream &out) const {
+    return defaultTrace(getValue(context), out);
 }
 
-
 std::unique_ptr<Value>
-SearchColumnValueNode::traceValue(const DocumentId& id,
-                                  std::ostream& out) const
+ValueNode::defaultTrace(std::unique_ptr<Value> val, std::ostream& out) const
 {
-    std::unique_ptr<Value> result(new IntegerValue(
-                getValue(_bucketIdFactory.getBucketId(id)), false));
-    out << "Resolved search column of doc \"" << id << "\" to " << *result
-        << "\n";
-    return result;
-}
-
-
-void
-SearchColumnValueNode::visit(Visitor &visitor) const
-{
-    visitor.visitSearchColumnValueNode(*this);
-}
-
-
-void
-SearchColumnValueNode::print(std::ostream& out, bool verbose,
-                   const std::string& indent) const
-{
-    (void) verbose; (void) indent;
-    if (hadParentheses()) out << '(';
-    out << _id;
-    out << '.' << _numColumns;
-    if (hadParentheses()) out << ')';
-}
-
-namespace {
-    union HashUnion {
-        unsigned char _key[16];
-        int64_t       _hash[2];
-    };
-    int64_t hash(const void* data, uint32_t len) {
-        HashUnion hash;
-        fastc_md5sum((const unsigned char*) data, len, hash._key);
-        return hash._hash[0];
-    }
+    out << "Returning value " << *val << ".\n";
+    return std::move(val);
 }
 
-FunctionValueNode::FunctionValueNode(const vespalib::stringref & name,
-                                     std::unique_ptr<ValueNode> src)
-    : _function(),
-      _funcname(name),
-      _source(std::move(src))
-{
-    if (name == "lowercase") {
-        _function = LOWERCASE;
-    } else if (name == "hash") {
-        _function = HASH;
-    } else if (name == "abs") {
-        _function = ABS;
-    } else {
-        throw ParsingFailedException("No function '"+name+"' exist.",
-                                     VESPA_STRLOC);
-    }
-}
-
-std::unique_ptr<Value>
-FunctionValueNode::getValue(std::unique_ptr<Value> val) const
-{
-    switch (val->getType()) {
-        case Value::String:
-        {
-            StringValue& sval(static_cast<StringValue&>(*val));
-            if (_function == LOWERCASE) {
-                return std::unique_ptr<Value>(new StringValue(
-                    vespalib::LowerCase::convert(sval.getValue())));
-            } else if (_function == HASH) {
-                return std::unique_ptr<Value>(new IntegerValue(
-                    hash(sval.getValue().c_str(), sval.getValue().size()),
-                    false));
-            }
-            break;
-        }
-        case Value::Float:
-        {
-            FloatValue& fval(static_cast<FloatValue&>(*val));
-            if (_function == HASH) {
-                FloatValue::ValueType ffval = fval.getValue();
-                return std::unique_ptr<Value>(new IntegerValue(
-                    hash(&ffval, sizeof(ffval)), false));
-            } else if (_function == ABS) {
-                FloatValue::ValueType ffval = fval.getValue();
-                if (ffval < 0) ffval *= -1;
-                return std::unique_ptr<Value>(new FloatValue(ffval));
-            }
-            break;
-        }
-        case Value::Integer:
-        {
-            IntegerValue& ival(static_cast<IntegerValue&>(*val));
-            if (_function == HASH) {
-                IntegerValue::ValueType iival = ival.getValue();
-                return std::unique_ptr<Value>(new IntegerValue(
-                    hash(&iival, sizeof(iival)), false));
-            } else if (_function == ABS) {
-                IntegerValue::ValueType iival = ival.getValue();
-                if (iival < 0) iival *= -1;
-                return std::unique_ptr<Value>(new IntegerValue(iival, false));
-            }
-            break;
-        }
-        case Value::Bucket:
-        {
-            throw ParsingFailedException(
-                    "No functioncalls are allowed on value of type bucket",
-                    VESPA_STRLOC);
-            break;
-        }
-
-        case Value::Array: break;
-        case Value::Struct: break;
-        case Value::Invalid: break;
-        case Value::Null: break;
-    }
-    return std::unique_ptr<Value>(new InvalidValue);
-}
-
-std::unique_ptr<Value>
-FunctionValueNode::traceValue(std::unique_ptr<Value> val,
-                              std::ostream& out) const
-{
-    switch (val->getType()) {
-        case Value::String:
-        {
-            StringValue& sval(static_cast<StringValue&>(*val));
-            if (_function == LOWERCASE) {
-                std::unique_ptr<Value> result(new StringValue(
-                    vespalib::LowerCase::convert(sval.getValue())));
-                out << "Performed lowercase function on '" << sval
-                    << "' => '" << *result << "'.\n";
-                return result;
-            } else if (_function == HASH) {
-                std::unique_ptr<Value> result(new IntegerValue(
-                    hash(sval.getValue().c_str(), sval.getValue().size()),
-                    false));
-                out << "Performed hash on string '" << sval << "' -> "
-                    << *result << "\n";
-                return result;
-            }
-            break;
-        }
-        case Value::Float:
-        {
-            FloatValue& fval(static_cast<FloatValue&>(*val));
-            if (_function == HASH) {
-                FloatValue::ValueType ffval = fval.getValue();
-                std::unique_ptr<Value> result(new IntegerValue(
-                    hash(&ffval, sizeof(ffval)), false));
-                out << "Performed hash on float " << ffval << " -> " << *result
-                    << "\n";
-                return result;
-            } else if (_function == ABS) {
-                FloatValue::ValueType ffval = fval.getValue();
-                if (ffval < 0) ffval *= -1;
-                out << "Performed abs on float " << fval.getValue() << " -> "
-                    << ffval << "\n";
-                return std::unique_ptr<Value>(new FloatValue(ffval));
-            }
-            break;
-        }
-        case Value::Integer:
-        {
-            IntegerValue& ival(static_cast<IntegerValue&>(*val));
-            if (_function == HASH) {
-                IntegerValue::ValueType iival = ival.getValue();
-                std::unique_ptr<Value> result(new IntegerValue(
-                    hash(&iival, sizeof(iival)), false));
-                out << "Performed hash on float " << iival << " -> " << *result
-                    << "\n";
-                return result;
-            } else if (_function == ABS) {
-                IntegerValue::ValueType iival = ival.getValue();
-                if (iival < 0) iival *= -1;
-                out << "Performed abs on integer " << ival.getValue() << " -> "
-                    << iival << "\n";
-                return std::unique_ptr<Value>(new IntegerValue(iival, false));
-            }
-            break;
-        }
-        case Value::Bucket: break;
-        case Value::Array: break;
-        case Value::Struct: break;
-        case Value::Invalid: break;
-        case Value::Null: break;
-    }
-    out << "Cannot use function " << _function << " on a value of type "
-        << val->getType() << ". Resolving invalid.\n";
-    return std::unique_ptr<Value>(new InvalidValue);
-}
-
-
-void
-FunctionValueNode::visit(Visitor &visitor) const
-{
-    visitor.visitFunctionValueNode(*this);
-}
-
-
-void
-FunctionValueNode::print(std::ostream& out, bool verbose,
-                         const std::string& indent) const
-{
-    if (hadParentheses()) out << '(';
-    _source->print(out, verbose, indent);
-    out << '.' << _funcname << "()";
-    if (hadParentheses()) out << ')';
-}
-
-ArithmeticValueNode::ArithmeticValueNode(
-        std::unique_ptr<ValueNode> left, const vespalib::stringref & op,
-        std::unique_ptr<ValueNode> right)
-    : _operator(),
-      _left(std::move(left)),
-      _right(std::move(right))
-{
-    if (op.size() == 1) switch (op[0]) {
-        case '+': _operator = ADD; return;
-        case '-': _operator = SUB; return;
-        case '*': _operator = MUL; return;
-        case '/': _operator = DIV; return;
-        case '%': _operator = MOD; return;
-    }
-    throw ParsingFailedException(
-            "Arithmetic operator '"+op+"' does not exist.", VESPA_STRLOC);
-}
-
-const char*
-ArithmeticValueNode::getOperatorName() const
-{
-    switch (_operator) {
-        case ADD: return "+";
-        case SUB: return "-";
-        case MUL: return "*";
-        case DIV: return "/";
-        case MOD: return "%";
-    }
-    return "UNKNOWN";
-}
-
-
-
-std::unique_ptr<Value>
-ArithmeticValueNode::getValue(std::unique_ptr<Value> lval,
-                              std::unique_ptr<Value> rval) const
-{
-    switch (_operator) {
-        case ADD:
-        {
-            if (lval->getType() == Value::String &&
-                rval->getType() == Value::String)
-            {
-                StringValue& slval(static_cast<StringValue&>(*lval));
-                StringValue& srval(static_cast<StringValue&>(*rval));
-                return std::unique_ptr<Value>(new StringValue(
-                            slval.getValue() + srval.getValue()));
-            }
-        }
-        case SUB:
-        case MUL:
-        case DIV:
-        {
-            if (lval->getType() == Value::Integer &&
-                rval->getType() == Value::Integer)
-            {
-                IntegerValue& ilval(static_cast<IntegerValue&>(*lval));
-                IntegerValue& irval(static_cast<IntegerValue&>(*rval));
-                IntegerValue::ValueType res = 0;
-                switch (_operator) {
-                    case ADD: res = ilval.getValue() + irval.getValue(); break;
-                    case SUB: res = ilval.getValue() - irval.getValue(); break;
-                    case MUL: res = ilval.getValue() * irval.getValue(); break;
-                    case DIV:
-                        if (irval.getValue() != 0) {
-                            res = ilval.getValue() / irval.getValue();
-                        } else {
-                            throw vespalib::IllegalArgumentException("Division by zero");
-                        }
-                        break;
-                    case MOD: assert(0);
-                }
-                return std::unique_ptr<Value>(new IntegerValue(res, false));
-            }
-            NumberValue* nlval(dynamic_cast<NumberValue*>(lval.get()));
-            NumberValue* nrval(dynamic_cast<NumberValue*>(rval.get()));
-            if (nlval != 0 && nrval != 0) {
-                NumberValue::CommonValueType res = 0;
-                switch (_operator) {
-                    case ADD: res = nlval->getCommonValue()
-                                  + nrval->getCommonValue(); break;
-                    case SUB: res = nlval->getCommonValue()
-                                  - nrval->getCommonValue(); break;
-                    case MUL: res = nlval->getCommonValue()
-                                  * nrval->getCommonValue(); break;
-                    case DIV:
-                        if (nrval->getCommonValue() != 0) {
-                            res = nlval->getCommonValue()
-                                  / nrval->getCommonValue();
-                        } else {
-                            throw vespalib::IllegalArgumentException("Division by zero");
-                        }
-                        break;
-                    case MOD: assert(0);
-                }
-                return std::unique_ptr<Value>(new FloatValue(res));
-            }
-        }
-        case MOD:
-        {
-            if (lval->getType() == Value::Integer &&
-                rval->getType() == Value::Integer)
-            {
-                IntegerValue& ilval(static_cast<IntegerValue&>(*lval));
-                IntegerValue& irval(static_cast<IntegerValue&>(*rval));
-                if (irval.getValue() != 0) {
-                    return std::unique_ptr<Value>(new IntegerValue(ilval.getValue() % irval.getValue(), false));
-                } else {
-                    throw vespalib::IllegalArgumentException("Division by zero");
-                }
-            }
-        }
-    }
-    return std::unique_ptr<Value>(new InvalidValue);
-}
-
-std::unique_ptr<Value>
-ArithmeticValueNode::traceValue(std::unique_ptr<Value> lval,
-                                std::unique_ptr<Value> rval,
-                                std::ostream& out) const
-{
-    switch (_operator) {
-        case ADD:
-        {
-            if (lval->getType() == Value::String &&
-                rval->getType() == Value::String)
-            {
-                StringValue& slval(static_cast<StringValue&>(*lval));
-                StringValue& srval(static_cast<StringValue&>(*rval));
-                std::unique_ptr<Value> result(new StringValue(
-                            slval.getValue() + srval.getValue()));
-                out << "Appended strings '" << slval << "' + '" << srval
-                    << "' -> '" << *result << "'.\n";
-                return result;
-            }
-        }
-        case SUB:
-        case MUL:
-        case DIV:
-        {
-            if (lval->getType() == Value::Integer &&
-                rval->getType() == Value::Integer)
-            {
-                IntegerValue& ilval(static_cast<IntegerValue&>(*lval));
-                IntegerValue& irval(static_cast<IntegerValue&>(*rval));
-                IntegerValue::ValueType res = 0;
-                switch (_operator) {
-                    case ADD: res = ilval.getValue() + irval.getValue(); break;
-                    case SUB: res = ilval.getValue() - irval.getValue(); break;
-                    case MUL: res = ilval.getValue() * irval.getValue(); break;
-                    case DIV: res = ilval.getValue() / irval.getValue(); break;
-                    case MOD: assert(0);
-                }
-                std::unique_ptr<Value> result(new IntegerValue(res, false));
-                out << "Performed integer operation " << ilval << " "
-                    << getOperatorName() << " " << irval << " = " << *result
-                    << "\n";
-                return result;
-            }
-            NumberValue* nlval(dynamic_cast<NumberValue*>(lval.get()));
-            NumberValue* nrval(dynamic_cast<NumberValue*>(lval.get()));
-            if (nlval != 0 && nrval != 0) {
-                NumberValue::CommonValueType res = 0;
-                switch (_operator) {
-                    case ADD: res = nlval->getCommonValue()
-                                  + nrval->getCommonValue(); break;
-                    case SUB: res = nlval->getCommonValue()
-                                  - nrval->getCommonValue(); break;
-                    case MUL: res = nlval->getCommonValue()
-                                  * nrval->getCommonValue(); break;
-                    case DIV: res = nlval->getCommonValue()
-                                  / nrval->getCommonValue(); break;
-                    case MOD: assert(0);
-                }
-                std::unique_ptr<Value> result(new FloatValue(res));
-                out << "Performed float operation " << nlval << " "
-                    << getOperatorName() << " " << nrval << " = " << *result
-                    << "\n";
-                return result;
-            }
-        }
-        case MOD:
-        {
-            if (lval->getType() == Value::Integer &&
-                rval->getType() == Value::Integer)
-            {
-                IntegerValue& ilval(static_cast<IntegerValue&>(*lval));
-                IntegerValue& irval(static_cast<IntegerValue&>(*rval));
-                std::unique_ptr<Value> result(new IntegerValue(
-                            ilval.getValue() % irval.getValue(), false));
-                out << "Performed integer operation " << ilval << " "
-                    << getOperatorName() << " " << irval << " = " << *result
-                    << "\n";
-                return result;
-            }
-        }
-    }
-    out << "Failed to do operation " << getOperatorName()
-        << " on values of type " << lval->getType() << " and "
-        << rval->getType() << ". Resolving invalid.\n";
-    return std::unique_ptr<Value>(new InvalidValue);
-}
-
-
-void
-ArithmeticValueNode::visit(Visitor &visitor) const
-{
-    visitor.visitArithmeticValueNode(*this);
-}
-
-
-void
-ArithmeticValueNode::print(std::ostream& out, bool verbose,
-                           const std::string& indent) const
-{
-    if (hadParentheses()) out << '(';
-    _left->print(out, verbose, indent);
-    switch (_operator) {
-        case ADD: out << " + "; break;
-        case SUB: out << " - "; break;
-        case MUL: out << " * "; break;
-        case DIV: out << " / "; break;
-        case MOD: out << " % "; break;
-    }
-    _right->print(out, verbose, indent);
-    if (hadParentheses()) out << ')';
 }
 
-} // select
-} // document
diff --git a/document/src/vespa/document/select/valuenode.h b/document/src/vespa/document/select/valuenode.h
index e3f0b4c4a94..f36085f769d 100644
--- a/document/src/vespa/document/select/valuenode.h
+++ b/document/src/vespa/document/select/valuenode.h
@@ -13,51 +13,28 @@
 #pragma once
 
 #include "value.h"
-#include "context.h"
-#include <vespa/document/fieldvalue/document.h>
-#include <vespa/document/update/documentupdate.h>
-#include <vespa/document/bucket/bucketdistribution.h>
-#include <vespa/document/bucket/bucketidfactory.h>
 
-namespace document {
+namespace document::select {
 
-class Document;
-class Field;
-
-namespace select {
+class Context;
+class Visitor;
 
 class ValueNode : public Printable
 {
 public:
-    typedef std::unique_ptr<ValueNode> UP;
-    typedef Context::VariableMap VariableMap;
+    using UP = std::unique_ptr<ValueNode>;
 
     ValueNode() : _parentheses(false) {}
     virtual ~ValueNode() {}
 
     void setParentheses() { _parentheses = true; }
-
-    void clearParentheses()
-    {
-        _parentheses = false;
-    }
-
+    void clearParentheses() { _parentheses = false; }
     bool hadParentheses() const { return _parentheses; }
 
-    virtual std::unique_ptr<Value>
-    getValue(const Context& context) const  = 0;
-
-    virtual std::unique_ptr<Value>
-    traceValue(const Context &context,
-               std::ostream &out) const {
-        return defaultTrace(getValue(context), out);
-    }
-
-    virtual void print(std::ostream& out, bool verbose, const std::string& indent) const override = 0;
-
+    virtual std::unique_ptr<Value> getValue(const Context& context) const  = 0;
     virtual void visit(Visitor&) const = 0;
-
     virtual ValueNode::UP clone() const = 0;
+    virtual std::unique_ptr<Value> traceValue(const Context &context, std::ostream &out) const;
 private:
     bool _parentheses; // Set to true if parentheses was used around this part
                        // Set such that we can recreate original query in print.
@@ -70,349 +47,8 @@ protected:
         return ret;
     }
 
-    std::unique_ptr<Value> defaultTrace(std::unique_ptr<Value> val,
-                                      std::ostream& out) const
-    {
-        out << "Returning value " << *val << ".\n";
-        return std::move(val);
-    }
-};
-
-class InvalidValueNode : public ValueNode
-{
-    vespalib::string _name;
-public:
-    InvalidValueNode(const vespalib::stringref & name);
-
-    std::unique_ptr<Value> getValue(const Context&) const override {
-        return std::unique_ptr<Value>(new InvalidValue());
-    }
-
-    void print(std::ostream& out, bool verbose, const std::string& indent) const override;
-    void visit(Visitor& visitor) const override;
-
-    ValueNode::UP clone() const override {
-        return wrapParens(new InvalidValueNode(_name));
-    }
-};
-
-class NullValueNode : public ValueNode
-{
-    vespalib::string _name;
-public:
-    NullValueNode(const vespalib::stringref & name);
-
-    std::unique_ptr<Value> getValue(const Context&) const override {
-        return std::unique_ptr<Value>(new NullValue());
-    }
-
-    void print(std::ostream& out, bool verbose, const std::string& indent) const override;
-
-    void visit(Visitor& visitor) const override;
-
-    ValueNode::UP clone() const override {
-        return wrapParens(new NullValueNode(_name));
-    }
-};
-
-class StringValueNode : public ValueNode
-{
-    vespalib::string _value;
-public:
-    StringValueNode(const vespalib::stringref & val);
-
-    const vespalib::string& getValue() const { return _value; }
-
-    std::unique_ptr<Value> getValue(const Context&) const override {
-        return std::unique_ptr<Value>(new StringValue(_value));
-    }
-
-    void print(std::ostream& out, bool verbose, const std::string& indent) const override;
-    void visit(Visitor& visitor) const override;
-
-    ValueNode::UP clone() const override {
-        return wrapParens(new StringValueNode(_value));
-    }
-};
-
-class IntegerValueNode : public ValueNode
-{
-    int64_t _value;
-    bool _isBucketValue;
-public:
-    IntegerValueNode(int64_t val, bool isBucketValue)
-        : _value(val), _isBucketValue(isBucketValue) {}
-
-    int64_t getValue() const { return _value; }
-
-    virtual std::unique_ptr<Value> getValue(const Context&) const override {
-        return std::unique_ptr<Value>(new IntegerValue(_value, _isBucketValue));
-    }
-
-    void print(std::ostream& out, bool verbose, const std::string& indent) const override;
-    void visit(Visitor& visitor) const override;
-
-    ValueNode::UP clone() const override {
-        return wrapParens(new IntegerValueNode(_value, _isBucketValue));
-    }
-};
-
-class CurrentTimeValueNode : public ValueNode
-{
-public:
-    int64_t getValue() const;
-
-    std::unique_ptr<Value> getValue(const Context&) const override {
-        return std::unique_ptr<Value>(new IntegerValue(getValue(), false));
-    }
-
-    void print(std::ostream& out, bool verbose, const std::string& indent) const override;
-    void visit(Visitor& visitor) const override;
-
-    ValueNode::UP clone() const override {
-        return wrapParens(new CurrentTimeValueNode);
-    }
-};
-
-class VariableValueNode : public ValueNode
-{
-    vespalib::string _value;
-public:
-    VariableValueNode(const vespalib::string & variableName) : _value(variableName) {}
-
-    const vespalib::string& getVariableName() const { return _value; }
-
-    std::unique_ptr<Value> getValue(const Context& context) const override;
-    void print(std::ostream& out, bool verbose, const std::string& indent) const override;
-    void visit(Visitor& visitor) const override;
-
-    ValueNode::UP clone() const override {
-        return wrapParens(new VariableValueNode(_value));
-    }
-};
-
-class FloatValueNode : public ValueNode
-{
-    double _value;
-public:
-    FloatValueNode(double val) : _value(val) {}
-
-    double getValue() const { return _value; }
-
-    std::unique_ptr<Value> getValue(const Context&) const override {
-        return std::unique_ptr<Value>(new FloatValue(_value));
-    }
-
-    void print(std::ostream& out, bool verbose, const std::string& indent) const override;
-    void visit(Visitor& visitor) const override;
-
-    ValueNode::UP clone() const override {
-        return wrapParens(new FloatValueNode(_value));
-    }
-};
-
-class FieldValueNode : public ValueNode
-{
-    vespalib::string _doctype;
-    vespalib::string _fieldExpression;
-    vespalib::string _fieldName;
-    mutable FieldPath _fieldPath;
-
-public:
-    FieldValueNode(const vespalib::string& doctype, const vespalib::string& fieldExpression);
-    FieldValueNode(const FieldValueNode &);
-    FieldValueNode & operator = (const FieldValueNode &);
-    FieldValueNode(FieldValueNode &&) = default;
-    FieldValueNode & operator = (FieldValueNode &&) = default;
-    ~FieldValueNode();
-
-    const vespalib::string& getDocType() const { return _doctype; }
-    const vespalib::string& getRealFieldName() const { return _fieldName; }
-    const vespalib::string& getFieldName() const { return _fieldExpression; }
-
-    std::unique_ptr<Value> getValue(const Context& context) const override;
-    std::unique_ptr<Value> traceValue(const Context &context, std::ostream& out) const override;
-    void print(std::ostream& out, bool verbose, const std::string& indent) const override;
-    void visit(Visitor& visitor) const override;
-
-    ValueNode::UP clone() const override {
-        return wrapParens(new FieldValueNode(_doctype, _fieldExpression));
-    }
-
-    static vespalib::string extractFieldName(const std::string & fieldExpression);
-
-private:
-    class IteratorHandler : public FieldValue::IteratorHandler
-    {
-    public:
-        IteratorHandler();
-        ~IteratorHandler();
-        bool hasSingleValue() const;
-
-        std::unique_ptr<Value> getSingleValue();
-        const std::vector<ArrayValue::VariableValue>& getValues();
-
-    private:
-        std::unique_ptr<Value> _firstValue;
-        std::vector<ArrayValue::VariableValue> _values;
-
-        void onPrimitive(uint32_t fid, const Content & fv) override;
-        std::unique_ptr<Value> getInternalValue(const FieldValue& fval) const;
-    };
-
-    void initFieldPath(const DocumentType&) const;
-};
-
-class IdValueNode : public ValueNode
-{
-public:
-    enum Type { SCHEME, NS, TYPE, USER, GROUP, GID, SPEC, BUCKET, ORDER, ALL };
-
-    IdValueNode(const BucketIdFactory& bucketIdFactory,
-                const vespalib::stringref & name, const vespalib::stringref & type,
-                int widthBits = -1, int divisionBits = -1);
-
-    Type getType() const { return _type; }
-
-    std::unique_ptr<Value> getValue(const Context& context) const override;
-
-    std::unique_ptr<Value> getValue(const DocumentId& id) const;
-
-    std::unique_ptr<Value> traceValue(const Context& context, std::ostream &out) const override;
-
-    std::unique_ptr<Value> traceValue(const DocumentId& val, std::ostream& out) const;
-
-    void print(std::ostream& out, bool verbose, const std::string& indent) const override;
-
-    void visit(Visitor& visitor) const override;
-
-    ValueNode::UP clone() const override {
-        return wrapParens(new IdValueNode(_bucketIdFactory, _id, _typestring, _widthBits, _divisionBits));
-    }
-
-    int getWidthBits() const { return _widthBits; }
-    int getDivisionBits() const { return _divisionBits; }
-
-private:
-    const BucketIdFactory& _bucketIdFactory;
-    vespalib::string _id;
-    vespalib::string _typestring;
-    Type _type;
-    int _widthBits;
-    int _divisionBits;
+    std::unique_ptr<Value> defaultTrace(std::unique_ptr<Value> val, std::ostream& out) const;
 };
 
-class SearchColumnValueNode : public ValueNode
-{
-public:
-    SearchColumnValueNode(const BucketIdFactory& bucketIdFactory,
-                          const vespalib::stringref & name,
-                          int numColumns);
-
-    int getColumns() { return _numColumns; }
-
-    std::unique_ptr<Value> getValue(const Context& context) const override;
-    std::unique_ptr<Value> getValue(const DocumentId& id) const;
-    std::unique_ptr<Value> traceValue(const Context& context, std::ostream &out) const override;
-    std::unique_ptr<Value> traceValue(const DocumentId& val, std::ostream& out) const;
-    
-    int64_t getValue(const BucketId& bucketId) const;
-    void print(std::ostream& out, bool verbose, const std::string& indent) const override;
-    void visit(Visitor& visitor) const override;
-
-    ValueNode::UP clone() const override {
-        return wrapParens(new SearchColumnValueNode(_bucketIdFactory, _id, _numColumns));
 }
 
-private:
-    const BucketIdFactory& _bucketIdFactory;
-    vespalib::string _id;
-    int _numColumns;
-    BucketDistribution _distribution;
-};
-
-class FunctionValueNode : public ValueNode
-{
-public:
-    enum Function { LOWERCASE, HASH, ABS };
-
-    FunctionValueNode(const vespalib::stringref & name, std::unique_ptr<ValueNode> src);
-
-    Function getFunction() const { return _function; }
-    const vespalib::string &getFunctionName(void) const { return _funcname; }
-
-    std::unique_ptr<Value> getValue(const Context& context) const override {
-        return getValue(_source->getValue(context));
-    }
-
-    std::unique_ptr<Value> traceValue(const Context &context, std::ostream& out) const override {
-        return traceValue(_source->getValue(context), out);
-    }
-
-    void print(std::ostream& out, bool verbose, const std::string& indent) const override;
-    void visit(Visitor& visitor) const override;
-
-    ValueNode::UP clone() const override {
-        return wrapParens(new FunctionValueNode(_funcname, _source->clone()));
-    }
-
-    const ValueNode& getChild() const { return *_source; }
-
-private:
-    Function _function;
-    vespalib::string _funcname;
-    std::unique_ptr<ValueNode> _source;
-
-    virtual std::unique_ptr<Value> getValue(std::unique_ptr<Value> val) const;
-    virtual std::unique_ptr<Value> traceValue(std::unique_ptr<Value> val,
-                                            std::ostream& out) const;
-};
-
-class ArithmeticValueNode : public ValueNode
-{
-public:
-    enum Operator { ADD, SUB, MUL, DIV, MOD };
-
-    ArithmeticValueNode(std::unique_ptr<ValueNode> left,
-                        const vespalib::stringref & op,
-                        std::unique_ptr<ValueNode> right);
-
-    Operator getOperator() const { return _operator; }
-    const char* getOperatorName() const;
-
-    std::unique_ptr<Value>
-    getValue(const Context& context) const override {
-        return getValue(_left->getValue(context), _right->getValue(context));
-    }
-    
-    std::unique_ptr<Value>
-    traceValue(const Context &context, std::ostream& out) const override {
-        return traceValue(_left->getValue(context), _right->getValue(context), out);
-    }
-
-    void print(std::ostream& out, bool verbose, const std::string& indent) const override;
-    void visit(Visitor& visitor) const override;
-
-    ValueNode::UP clone() const override {
-        return wrapParens(new ArithmeticValueNode(_left->clone(),
-                                                  getOperatorName(),
-                                                  _right->clone()));
-    }
-
-    const ValueNode& getLeft() const { return *_left; }
-    const ValueNode& getRight() const { return *_right; }
-
-private:
-    Operator _operator;
-    std::unique_ptr<ValueNode> _left;
-    std::unique_ptr<ValueNode> _right;
-
-    virtual std::unique_ptr<Value> getValue(std::unique_ptr<Value> lval,
-                                          std::unique_ptr<Value> rval) const;
-    virtual std::unique_ptr<Value> traceValue(std::unique_ptr<Value> lval,
-                                            std::unique_ptr<Value> rval,
-                                            std::ostream&) const;
-};
-
-} // select
-} // document
diff --git a/document/src/vespa/document/select/valuenodes.cpp b/document/src/vespa/document/select/valuenodes.cpp
new file mode 100644
index 00000000000..3a4edbef093
--- /dev/null
+++ b/document/src/vespa/document/select/valuenodes.cpp
@@ -0,0 +1,1183 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include "valuenodes.h"
+#include "visitor.h"
+#include "parser.h"
+#include <vespa/document/bucket/bucketdistribution.h>
+#include <vespa/document/base/exceptions.h>
+#include <vespa/document/update/documentupdate.h>
+#include <vespa/document/fieldvalue/fieldvalues.h>
+#include <vespa/document/fieldvalue/iteratorhandler.h>
+#include <vespa/document/datatype/documenttype.h>
+#include <vespa/vespalib/util/md5.h>
+#include <vespa/document/util/stringutil.h>
+#include <vespa/vespalib/text/lowercase.h>
+#include <regex>
+#include <iomanip>
+
+
+#include <vespa/log/log.h>
+LOG_SETUP(".document.select.valuenode");
+
+namespace document::select {
+
+namespace {
+    static const std::regex FIELD_NAME_REGEX("^([_A-Za-z][_A-Za-z0-9]*).*");
+}
+
+namespace {
+    bool documentTypeEqualsName(const DocumentType& type, const vespalib::stringref& name)
+    {
+        if (type.getName() == name) return true;
+        for (std::vector<const DocumentType *>::const_iterator it
+                = type.getInheritedTypes().begin();
+             it != type.getInheritedTypes().end(); ++it)
+        {
+            if (documentTypeEqualsName(**it, name)) return true;
+        }
+        return false;
+    }
+}
+
+InvalidValueNode::InvalidValueNode(const vespalib::stringref & name)
+    : _name(name)
+{ }
+
+
+void
+InvalidValueNode::visit(Visitor &visitor) const
+{
+    visitor.visitInvalidValueNode(*this);
+}
+
+
+void
+InvalidValueNode::print(std::ostream& out, bool verbose,
+                        const std::string& indent) const
+{
+    (void) verbose; (void) indent;
+    if (hadParentheses()) out << '(';
+    out << _name;
+    if (hadParentheses()) out << ')';
+}
+
+NullValueNode::NullValueNode(const vespalib::stringref & name)
+    : _name(name)
+{ }
+
+
+void
+NullValueNode::visit(Visitor &visitor) const
+{
+    visitor.visitNullValueNode(*this);
+}
+
+
+void
+NullValueNode::print(std::ostream& out, bool verbose,
+                        const std::string& indent) const
+{
+    (void) verbose; (void) indent;
+    if (hadParentheses()) out << '(';
+    out << _name;
+    if (hadParentheses()) out << ')';
+}
+
+StringValueNode::StringValueNode(const vespalib::stringref & val)
+    : _value(val)
+{
+}
+
+
+void
+StringValueNode::visit(Visitor &visitor) const
+{
+    visitor.visitStringValueNode(*this);
+}
+
+
+void
+StringValueNode::print(std::ostream& out, bool verbose,
+                       const std::string& indent) const
+{
+    (void) verbose; (void) indent;
+    if (hadParentheses()) out << '(';
+    out << "\"" << StringUtil::escape(_value) << "\"";
+    if (hadParentheses()) out << ')';
+}
+
+
+void
+IntegerValueNode::visit(Visitor &visitor) const
+{
+    visitor.visitIntegerValueNode(*this);
+}
+
+
+void
+IntegerValueNode::print(std::ostream& out, bool verbose,
+                       const std::string& indent) const
+{
+    (void) verbose; (void) indent;
+    if (hadParentheses()) out << '(';
+    out << _value;
+    if (hadParentheses()) out << ')';
+}
+
+int64_t
+CurrentTimeValueNode::getValue() const
+{
+    struct timeval mytime;
+    gettimeofday(&mytime, 0);
+    return mytime.tv_sec;
+}
+
+
+void
+CurrentTimeValueNode::visit(Visitor &visitor) const
+{
+    visitor.visitCurrentTimeValueNode(*this);
+}
+
+
+void
+CurrentTimeValueNode::print(std::ostream& out, bool verbose,
+                            const std::string& indent) const
+{
+    (void) verbose; (void) indent;
+    out << "now()";
+}
+
+std::unique_ptr<Value>
+VariableValueNode::getValue(const Context& context) const {
+    return context.getValue(_value);
+}
+
+void
+VariableValueNode::visit(Visitor &visitor) const
+{
+    visitor.visitVariableValueNode(*this);
+}
+
+
+void
+VariableValueNode::print(std::ostream& out, bool verbose,
+                       const std::string& indent) const
+{
+    (void) verbose; (void) indent;
+    if (hadParentheses()) out << '(';
+    out << "$" << _value;
+    if (hadParentheses()) out << ')';
+}
+
+
+void
+FloatValueNode::visit(Visitor &visitor) const
+{
+    visitor.visitFloatValueNode(*this);
+}
+
+
+void
+FloatValueNode::print(std::ostream& out, bool verbose,
+                       const std::string& indent) const
+{
+    (void) verbose; (void) indent;
+    if (hadParentheses()) out << '(';
+    out << _value;
+    if (hadParentheses()) out << ')';
+}
+
+FieldValueNode::FieldValueNode(const vespalib::string& doctype,
+                               const vespalib::string& fieldExpression)
+    : _doctype(doctype),
+      _fieldExpression(fieldExpression),
+      _fieldName(extractFieldName(fieldExpression))
+{
+}
+
+FieldValueNode::FieldValueNode(const FieldValueNode &) = default;
+FieldValueNode & FieldValueNode::operator = (const FieldValueNode &) = default;
+
+FieldValueNode::~FieldValueNode() {}
+vespalib::string
+FieldValueNode::extractFieldName(const std::string & fieldExpression) {
+    std::smatch match;
+
+    if (std::regex_match(fieldExpression, match, FIELD_NAME_REGEX) && match[1].matched) {
+        return vespalib::string(match[1].first, match[1].second);
+    }
+
+    throw ParsingFailedException("Fatal: could not extract field name from field expression '" + fieldExpression + "'");
+}
+
+namespace {
+
+class IteratorHandler : public fieldvalue::IteratorHandler {
+public:
+    IteratorHandler();
+    ~IteratorHandler();
+    bool hasSingleValue() const;
+    std::unique_ptr<Value> getSingleValue();
+    const std::vector<ArrayValue::VariableValue> &getValues();
+
+private:
+    std::unique_ptr<Value> _firstValue;
+    std::vector<ArrayValue::VariableValue> _values;
+
+    void onPrimitive(uint32_t fid, const Content &fv) override;
+    std::unique_ptr<Value> getInternalValue(const FieldValue &fval) const;
+};
+
+IteratorHandler::IteratorHandler() { }
+IteratorHandler::~IteratorHandler() { }
+
+bool
+IteratorHandler::hasSingleValue() const {
+    return _firstValue.get() && (_values.size() == 0);
+}
+
+std::unique_ptr<Value>
+IteratorHandler::getSingleValue() {
+    return std::move(_firstValue);
+}
+
+const std::vector<ArrayValue::VariableValue>&
+IteratorHandler::getValues() {
+    if (_firstValue.get()) {
+        _values.insert(_values.begin(), ArrayValue::VariableValue(fieldvalue::VariableMap(), Value::SP(_firstValue.release())));
+    }
+
+    return _values;
+}
+
+void
+IteratorHandler::onPrimitive(uint32_t fid, const Content& fv) {
+    (void) fid;
+    if (!_firstValue && getVariables().empty()) {
+        _firstValue = getInternalValue(fv.getValue());
+    } else {
+        _values.emplace_back(getVariables(), Value::SP(getInternalValue(fv.getValue()).release()));
+    }
+}
+
+std::unique_ptr<Value>
+IteratorHandler::getInternalValue(const FieldValue& fval) const
+{
+    switch(fval.getClass().id()) {
+        case document::IntFieldValue::classId:
+        {
+            const IntFieldValue& val(dynamic_cast<const IntFieldValue&>(fval));
+            return std::make_unique<IntegerValue>(val.getAsInt(), false);
+        }
+        case document::ByteFieldValue::classId:
+        {
+            const ByteFieldValue& val(dynamic_cast<const ByteFieldValue&>(fval));
+            return std::make_unique<IntegerValue>(val.getAsByte(), false);
+        }
+        case LongFieldValue::classId:
+        {
+            const LongFieldValue& val(dynamic_cast<const LongFieldValue&>(fval));
+            return std::make_unique<IntegerValue>(val.getAsLong(), false);
+        }
+        case FloatFieldValue::classId:
+        {
+            const FloatFieldValue& val(dynamic_cast<const FloatFieldValue&>(fval));
+            return std::make_unique<FloatValue>(val.getAsFloat());
+        }
+        case DoubleFieldValue::classId:
+        {
+            const DoubleFieldValue& val(dynamic_cast<const DoubleFieldValue&>(fval));
+            return std::make_unique<FloatValue>(val.getAsDouble());
+        }
+        case StringFieldValue::classId:
+        {
+            const StringFieldValue& val(dynamic_cast<const StringFieldValue&>(fval));
+            return std::make_unique<StringValue>(val.getAsString());
+        }
+        case ArrayFieldValue::classId:
+        {
+            const ArrayFieldValue& val(dynamic_cast<const ArrayFieldValue&>(fval));
+            if (val.size() == 0) {
+                return std::make_unique<NullValue>();
+            } else {
+                std::vector<ArrayValue::VariableValue> values;
+                // TODO: Array comparison.
+                return std::make_unique<ArrayValue>(values);
+            }
+        }
+        case StructFieldValue::classId:
+        {
+            const StructFieldValue& val(dynamic_cast<const StructFieldValue&>(fval));
+            if (val.empty()) {
+                return std::make_unique<NullValue>();
+            } else {
+                StructValue::ValueMap values;
+                for (StructFieldValue::const_iterator it(val.begin()); it != val.end(); ++it) {
+                    FieldValue::UP fv(val.getValue(it.field()));
+                    values[it.field().getName()] = Value::SP(getInternalValue(*fv).release());
+                }
+                return std::make_unique<StructValue>(values);
+            }
+        }
+        case MapFieldValue::classId:
+        {
+            const MapFieldValue& val(static_cast<const MapFieldValue&>(fval));
+            if (val.isEmpty()) {
+                return std::make_unique<NullValue>();
+            } else {
+                std::vector<ArrayValue::VariableValue> values;
+                // TODO: Map comparison
+                return std::make_unique<ArrayValue>(values);
+            }
+        }
+    }
+    LOG(warning, "Tried to use unsupported datatype %s in field comparison",
+        fval.getDataType()->toString().c_str());
+    return std::make_unique<InvalidValue>();
+}
+
+}
+
+void
+FieldValueNode::initFieldPath(const DocumentType& type) const {
+    if (_fieldPath.size() == 0) {
+        FieldPath::UP path(type.buildFieldPath(_fieldExpression));
+        if (!path.get()) {
+            throw FieldNotFoundException(
+                    vespalib::make_string("Could not create field path for doc type: '%s' field: '%s'",
+                                          type.toString().c_str(), _fieldExpression.c_str()),
+                    VESPA_STRLOC);
+        }
+        _fieldPath = *path;
+    }
+}
+
+std::unique_ptr<Value>
+FieldValueNode::getValue(const Context& context) const
+{
+    if (context._doc == NULL) {
+        return std::unique_ptr<Value>(new InvalidValue());
+    }
+
+    const Document& doc = *context._doc;
+
+    if (!documentTypeEqualsName(doc.getType(), _doctype)) {
+        return std::unique_ptr<Value>(new InvalidValue());
+    }
+    try{
+        initFieldPath(doc.getType());
+
+        IteratorHandler handler;
+        doc.iterateNested(_fieldPath.getFullRange(), handler);
+
+        if (handler.hasSingleValue()) {
+            return handler.getSingleValue();
+        } else {
+            const std::vector<ArrayValue::VariableValue>& values = handler.getValues();
+
+            if (values.size() == 0) {
+                return std::unique_ptr<Value>(new NullValue());
+            } else {
+                return std::unique_ptr<Value>(new ArrayValue(handler.getValues()));
+            }
+        }
+    } catch (vespalib::IllegalArgumentException& e) {
+        LOG(warning, "Caught exception while fetching field from document: %s", e.what());
+        return std::unique_ptr<Value>(new InvalidValue());
+    } catch (FieldNotFoundException& e) {
+        LOG(warning, "Tried to compare to field %s, not found in document type", _fieldExpression.c_str());
+        return std::unique_ptr<Value>(new InvalidValue());
+    }
+}
+
+void
+FieldValueNode::visit(Visitor &visitor) const
+{
+    visitor.visitFieldValueNode(*this);
+}
+
+
+void
+FieldValueNode::print(std::ostream& out, bool verbose,
+                       const std::string& indent) const
+{
+    (void) verbose; (void) indent;
+    if (hadParentheses()) out << '(';
+    out << _doctype << "." << _fieldExpression;
+    if (hadParentheses()) out << ')';
+}
+
+
+std::unique_ptr<Value>
+FieldValueNode::traceValue(const Context &context, std::ostream& out) const
+{
+    if (context._doc == NULL) {
+        return defaultTrace(getValue(context), out);
+    }
+    const Document &doc(*context._doc);
+    if (!documentTypeEqualsName(doc.getType(), _doctype)) {
+        out << "Document is of type " << doc.getType() << " which isn't a "
+            << _doctype << " document, thus resolving invalid.\n";
+        return std::unique_ptr<Value>(new InvalidValue());
+    }
+    try{
+        initFieldPath(doc.getType());
+
+        IteratorHandler handler;
+        doc.iterateNested(_fieldPath.getFullRange(), handler);
+
+        if (handler.hasSingleValue()) {
+            return handler.getSingleValue();
+        } else {
+            const std::vector<ArrayValue::VariableValue>& values = handler.getValues();
+
+            if (values.size() == 0) {
+                return std::unique_ptr<Value>(new NullValue());
+            } else {
+                return std::unique_ptr<Value>(new ArrayValue(handler.getValues()));
+            }
+        }
+    } catch (FieldNotFoundException& e) {
+        LOG(warning, "Tried to compare to field %s, not found in document type",
+                     _fieldExpression.c_str());
+        out << "Field not found in document type " << doc.getType()
+            << ". Returning invalid.\n";
+        return std::unique_ptr<Value>(new InvalidValue());
+    }
+}
+
+IdValueNode::IdValueNode(const BucketIdFactory& bucketIdFactory,
+                         const vespalib::stringref & name, const vespalib::stringref & type,
+                         int widthBits, int divisionBits)
+    : _bucketIdFactory(bucketIdFactory),
+      _id(name),
+      _typestring(type),
+      _type(ALL),
+      _widthBits(widthBits),
+      _divisionBits(divisionBits)
+{
+    if (type.length() > 2) switch (type[0]) {
+    case 'b': _type = BUCKET;
+        break;
+    case 'n': _type = NS;
+        break;
+    case 'g':
+        if (type[1] == 'r') {
+            _type = GROUP;
+        } else if (type[1] == 'i') {
+            _type = GID;
+        }
+        break;
+    case 's': {
+        if (type[1] == 'c') { _type = SCHEME; } else { _type = SPEC; }
+        break;
+    }
+    case 't':
+        _type = TYPE;
+        break;
+    case 'u':
+        _type = USER;
+        break;
+    case 'o':
+        _type = ORDER;
+        break;
+    }
+}
+
+
+std::unique_ptr<Value>
+IdValueNode::getValue(const Context& context) const
+{
+    if (context._doc != NULL) {
+        return getValue(context._doc->getId());
+    } else if (context._docId != NULL) {
+        return getValue(*context._docId);
+    } else {
+        return getValue(context._docUpdate->getId());
+    }
+}
+
+
+std::unique_ptr<Value>
+IdValueNode::getValue(const DocumentId& id) const
+{
+    vespalib::string value;
+    switch (_type) {
+    case BUCKET:
+        return std::unique_ptr<Value>(
+            new IntegerValue(
+                _bucketIdFactory.getBucketId(id).getId(), true));
+    case NS:
+        value = id.getScheme().getNamespace(); break;
+    case SCHEME:
+        value = id.getScheme().getTypeName(id.getScheme().getType());
+        break;
+    case TYPE:
+        if (id.getScheme().hasDocType()) {
+            value = id.getScheme().getDocType();
+        } else {
+            return std::unique_ptr<Value>(new InvalidValue);
+        }
+        break;
+    case SPEC:
+        value = id.getScheme().getNamespaceSpecific();
+        break;
+    case ALL:
+        value = id.getScheme().toString();
+        break;
+    case GROUP:
+        if (id.getScheme().hasGroup()) {
+            value = id.getScheme().getGroup();
+        } else {
+            fprintf(stderr, "***** Returning invalid value for %s\n",
+                    id.toString().c_str());
+            return std::unique_ptr<Value>(new InvalidValue);
+        }
+        break;
+    case GID:
+        value = id.getGlobalId().toString();
+        break;
+    case ORDER:
+        if (id.getScheme().getType() == IdString::ORDERDOC) {
+            const OrderDocIdString& ods(
+                    static_cast<const OrderDocIdString&>(id.getScheme()));
+            if (ods.getWidthBits() == _widthBits
+                && ods.getDivisionBits() == _divisionBits)
+            {
+                return std::unique_ptr<Value>(new IntegerValue(
+                        static_cast<const OrderDocIdString&>(id.getScheme())
+                            .getOrdering(), false));
+            }
+        }
+        return std::unique_ptr<Value>(new InvalidValue());
+    case USER:
+        if (id.getScheme().hasNumber()) {
+            return std::unique_ptr<Value>(
+                    new IntegerValue(id.getScheme().getNumber(), false));
+        } else {
+            return std::unique_ptr<Value>(new InvalidValue);
+        }
+    }
+
+    return std::unique_ptr<Value>(new StringValue(value));
+}
+
+
+std::unique_ptr<Value>
+IdValueNode::traceValue(const Context& context,
+                        std::ostream &out) const
+{
+    if (context._doc != NULL) {
+        return traceValue(context._doc->getId(), out);
+    } else if (context._docId != NULL) {
+        return traceValue(*context._docId, out);
+    } else {
+        return traceValue(context._docUpdate->getId(), out);
+    }
+}
+
+
+std::unique_ptr<Value>
+IdValueNode::traceValue(const DocumentId& id, std::ostream& out) const
+{
+    vespalib::string value;
+    switch (_type) {
+    case BUCKET:
+        {
+            document::BucketId bucket(_bucketIdFactory.getBucketId(id));
+            std::unique_ptr<Value> result(
+                new IntegerValue(bucket.getId(), true));
+            out << "Found id.bucket specification. Resolved to "
+                << bucket.toString() << ".\n";
+            return result;
+        }
+    case NS:
+        value = id.getScheme().getNamespace();
+        out << "Resolved id.namespace to value\"" << value << "\".\n";
+        break;
+    case SCHEME:
+        value = id.getScheme().getTypeName(id.getScheme().getType());
+        out << "Resolved id.scheme to value\"" << value << "\".\n";
+        break;
+    case TYPE:
+        if (id.getScheme().hasDocType()) {
+            value = id.getScheme().getDocType();
+            out << "Resolved id.type to value\"" << value << "\".\n";
+        } else {
+            out << "Could not resolve type of doc " << id << ".\n";
+            return std::unique_ptr<Value>(new InvalidValue);
+        }
+        break;
+    case SPEC:
+        value = id.getScheme().getNamespaceSpecific();
+        out << "Resolved id.specific to value\"" << value << "\".\n";
+        break;
+    case ALL:
+        value = id.getScheme().toString();
+        out << "Resolved id to \"" << value << "\".\n";
+        break;
+    case GROUP:
+        if (id.getScheme().hasGroup()) {
+            value = id.getScheme().getGroup();
+            out << "Resolved group of doc (type " << id.getScheme().getType()
+                << ") to \"" << value << "\".\n";
+        } else {
+            out << "Can't resolve group of doc \"" << id << "\".\n";
+            return std::unique_ptr<Value>(new InvalidValue);
+        }
+        break;
+    case GID:
+        value = id.getGlobalId().toString();
+        out << "Resolved gid to \"" << value << "\".\n";
+        break;
+    case ORDER:
+        if (id.getScheme().getType() == IdString::ORDERDOC) {
+            const OrderDocIdString& ods(
+                    static_cast<const OrderDocIdString&>(id.getScheme()));
+            if (ods.getWidthBits() == _widthBits
+                && ods.getDivisionBits() == _divisionBits)
+            {
+                std::unique_ptr<Value> result(new IntegerValue(
+                        static_cast<const OrderDocIdString&>(id.getScheme())
+                            .getOrdering(), false));
+                out << "Resolved id.order to int " << *result << "\n";
+                return result;
+            }
+        }
+        out << "Could not resolve id.order(" << _widthBits << ", "
+            << _divisionBits << ") of doc " << id << ".\n";
+        return std::unique_ptr<Value>(new InvalidValue());
+    case USER:
+        if (id.getScheme().hasNumber()) {
+            std::unique_ptr<Value> result(
+                    new IntegerValue(id.getScheme().getNumber(), false));
+            out << "Resolved user of doc type " << id.getScheme().getType()
+                << " to " << *result << ".\n";
+            return result;
+        } else {
+            out << "Could not resolve user of doc " << id << ".\n";
+            return std::unique_ptr<Value>(new InvalidValue);
+        }
+    }
+
+    return std::unique_ptr<Value>(new StringValue(value));
+}
+
+
+void
+IdValueNode::visit(Visitor &visitor) const
+{
+    visitor.visitIdValueNode(*this);
+}
+
+
+void
+IdValueNode::print(std::ostream& out, bool verbose,
+                   const std::string& indent) const
+{
+    (void) verbose; (void) indent;
+    if (hadParentheses()) out << '(';
+    out << _id;
+    if (_type != ALL) {
+        out << '.' << _typestring;
+    }
+    if (_type == ORDER) {
+        out << "(" << _widthBits << "," << _divisionBits << ")";
+    }
+    if (hadParentheses()) out << ')';
+}
+
+SearchColumnValueNode::SearchColumnValueNode(
+        const BucketIdFactory& bucketIdFactory,
+        const vespalib::stringref & name, int numColumns)
+    : _bucketIdFactory(bucketIdFactory),
+      _id(name),
+      _numColumns(numColumns),
+      _distribution(std::make_unique<BucketDistribution>(_numColumns, 16))
+{
+}
+
+int64_t
+SearchColumnValueNode::getValue(const BucketId& id) const
+{
+    return _distribution->getColumn(id);
+}
+
+
+std::unique_ptr<Value>
+SearchColumnValueNode::getValue(const Context& context) const
+{
+    if (context._doc != NULL) {
+        return getValue(context._doc->getId());
+    } else if (context._docId != NULL) {
+        return getValue(*context._docId);
+    } else {
+        return getValue(context._docUpdate->getId());
+    }
+}
+
+
+std::unique_ptr<Value>
+SearchColumnValueNode::getValue(const DocumentId& id) const
+{
+    return std::unique_ptr<Value>(new IntegerValue(
+                getValue(_bucketIdFactory.getBucketId(id)), false));
+}
+
+
+std::unique_ptr<Value>
+SearchColumnValueNode::traceValue(const Context& context,
+                                  std::ostream &out) const
+{
+    if (context._doc != NULL) {
+        return traceValue(context._doc->getId(), out);
+    } else if (context._docId != NULL) {
+        return traceValue(*context._docId, out);
+    } else {
+        return traceValue(context._docUpdate->getId(), out);
+    }
+}
+
+
+std::unique_ptr<Value>
+SearchColumnValueNode::traceValue(const DocumentId& id,
+                                  std::ostream& out) const
+{
+    std::unique_ptr<Value> result(new IntegerValue(
+                getValue(_bucketIdFactory.getBucketId(id)), false));
+    out << "Resolved search column of doc \"" << id << "\" to " << *result
+        << "\n";
+    return result;
+}
+
+
+void
+SearchColumnValueNode::visit(Visitor &visitor) const
+{
+    visitor.visitSearchColumnValueNode(*this);
+}
+
+
+void
+SearchColumnValueNode::print(std::ostream& out, bool verbose,
+                   const std::string& indent) const
+{
+    (void) verbose; (void) indent;
+    if (hadParentheses()) out << '(';
+    out << _id;
+    out << '.' << _numColumns;
+    if (hadParentheses()) out << ')';
+}
+
+namespace {
+    union HashUnion {
+        unsigned char _key[16];
+        int64_t       _hash[2];
+    };
+    int64_t hash(const void* data, uint32_t len) {
+        HashUnion hash;
+        fastc_md5sum((const unsigned char*) data, len, hash._key);
+        return hash._hash[0];
+    }
+}
+
+FunctionValueNode::FunctionValueNode(const vespalib::stringref & name,
+                                     std::unique_ptr<ValueNode> src)
+    : _function(),
+      _funcname(name),
+      _source(std::move(src))
+{
+    if (name == "lowercase") {
+        _function = LOWERCASE;
+    } else if (name == "hash") {
+        _function = HASH;
+    } else if (name == "abs") {
+        _function = ABS;
+    } else {
+        throw ParsingFailedException("No function '"+name+"' exist.",
+                                     VESPA_STRLOC);
+    }
+}
+
+std::unique_ptr<Value>
+FunctionValueNode::getValue(std::unique_ptr<Value> val) const
+{
+    switch (val->getType()) {
+        case Value::String:
+        {
+            StringValue& sval(static_cast<StringValue&>(*val));
+            if (_function == LOWERCASE) {
+                return std::unique_ptr<Value>(new StringValue(
+                    vespalib::LowerCase::convert(sval.getValue())));
+            } else if (_function == HASH) {
+                return std::unique_ptr<Value>(new IntegerValue(
+                    hash(sval.getValue().c_str(), sval.getValue().size()),
+                    false));
+            }
+            break;
+        }
+        case Value::Float:
+        {
+            FloatValue& fval(static_cast<FloatValue&>(*val));
+            if (_function == HASH) {
+                FloatValue::ValueType ffval = fval.getValue();
+                return std::unique_ptr<Value>(new IntegerValue(
+                    hash(&ffval, sizeof(ffval)), false));
+            } else if (_function == ABS) {
+                FloatValue::ValueType ffval = fval.getValue();
+                if (ffval < 0) ffval *= -1;
+                return std::unique_ptr<Value>(new FloatValue(ffval));
+            }
+            break;
+        }
+        case Value::Integer:
+        {
+            IntegerValue& ival(static_cast<IntegerValue&>(*val));
+            if (_function == HASH) {
+                IntegerValue::ValueType iival = ival.getValue();
+                return std::unique_ptr<Value>(new IntegerValue(
+                    hash(&iival, sizeof(iival)), false));
+            } else if (_function == ABS) {
+                IntegerValue::ValueType iival = ival.getValue();
+                if (iival < 0) iival *= -1;
+                return std::unique_ptr<Value>(new IntegerValue(iival, false));
+            }
+            break;
+        }
+        case Value::Bucket:
+        {
+            throw ParsingFailedException(
+                    "No functioncalls are allowed on value of type bucket",
+                    VESPA_STRLOC);
+            break;
+        }
+
+        case Value::Array: break;
+        case Value::Struct: break;
+        case Value::Invalid: break;
+        case Value::Null: break;
+    }
+    return std::unique_ptr<Value>(new InvalidValue);
+}
+
+std::unique_ptr<Value>
+FunctionValueNode::traceValue(std::unique_ptr<Value> val,
+                              std::ostream& out) const
+{
+    switch (val->getType()) {
+        case Value::String:
+        {
+            StringValue& sval(static_cast<StringValue&>(*val));
+            if (_function == LOWERCASE) {
+                std::unique_ptr<Value> result(new StringValue(
+                    vespalib::LowerCase::convert(sval.getValue())));
+                out << "Performed lowercase function on '" << sval
+                    << "' => '" << *result << "'.\n";
+                return result;
+            } else if (_function == HASH) {
+                std::unique_ptr<Value> result(new IntegerValue(
+                    hash(sval.getValue().c_str(), sval.getValue().size()),
+                    false));
+                out << "Performed hash on string '" << sval << "' -> "
+                    << *result << "\n";
+                return result;
+            }
+            break;
+        }
+        case Value::Float:
+        {
+            FloatValue& fval(static_cast<FloatValue&>(*val));
+            if (_function == HASH) {
+                FloatValue::ValueType ffval = fval.getValue();
+                std::unique_ptr<Value> result(new IntegerValue(
+                    hash(&ffval, sizeof(ffval)), false));
+                out << "Performed hash on float " << ffval << " -> " << *result
+                    << "\n";
+                return result;
+            } else if (_function == ABS) {
+                FloatValue::ValueType ffval = fval.getValue();
+                if (ffval < 0) ffval *= -1;
+                out << "Performed abs on float " << fval.getValue() << " -> "
+                    << ffval << "\n";
+                return std::unique_ptr<Value>(new FloatValue(ffval));
+            }
+            break;
+        }
+        case Value::Integer:
+        {
+            IntegerValue& ival(static_cast<IntegerValue&>(*val));
+            if (_function == HASH) {
+                IntegerValue::ValueType iival = ival.getValue();
+                std::unique_ptr<Value> result(new IntegerValue(
+                    hash(&iival, sizeof(iival)), false));
+                out << "Performed hash on float " << iival << " -> " << *result
+                    << "\n";
+                return result;
+            } else if (_function == ABS) {
+                IntegerValue::ValueType iival = ival.getValue();
+                if (iival < 0) iival *= -1;
+                out << "Performed abs on integer " << ival.getValue() << " -> "
+                    << iival << "\n";
+                return std::unique_ptr<Value>(new IntegerValue(iival, false));
+            }
+            break;
+        }
+        case Value::Bucket: break;
+        case Value::Array: break;
+        case Value::Struct: break;
+        case Value::Invalid: break;
+        case Value::Null: break;
+    }
+    out << "Cannot use function " << _function << " on a value of type "
+        << val->getType() << ". Resolving invalid.\n";
+    return std::unique_ptr<Value>(new InvalidValue);
+}
+
+
+void
+FunctionValueNode::visit(Visitor &visitor) const
+{
+    visitor.visitFunctionValueNode(*this);
+}
+
+
+void
+FunctionValueNode::print(std::ostream& out, bool verbose,
+                         const std::string& indent) const
+{
+    if (hadParentheses()) out << '(';
+    _source->print(out, verbose, indent);
+    out << '.' << _funcname << "()";
+    if (hadParentheses()) out << ')';
+}
+
+ArithmeticValueNode::ArithmeticValueNode(
+        std::unique_ptr<ValueNode> left, const vespalib::stringref & op,
+        std::unique_ptr<ValueNode> right)
+    : _operator(),
+      _left(std::move(left)),
+      _right(std::move(right))
+{
+    if (op.size() == 1) switch (op[0]) {
+        case '+': _operator = ADD; return;
+        case '-': _operator = SUB; return;
+        case '*': _operator = MUL; return;
+        case '/': _operator = DIV; return;
+        case '%': _operator = MOD; return;
+    }
+    throw ParsingFailedException(
+            "Arithmetic operator '"+op+"' does not exist.", VESPA_STRLOC);
+}
+
+const char*
+ArithmeticValueNode::getOperatorName() const
+{
+    switch (_operator) {
+        case ADD: return "+";
+        case SUB: return "-";
+        case MUL: return "*";
+        case DIV: return "/";
+        case MOD: return "%";
+    }
+    return "UNKNOWN";
+}
+
+
+
+std::unique_ptr<Value>
+ArithmeticValueNode::getValue(std::unique_ptr<Value> lval,
+                              std::unique_ptr<Value> rval) const
+{
+    switch (_operator) {
+        case ADD:
+        {
+            if (lval->getType() == Value::String &&
+                rval->getType() == Value::String)
+            {
+                StringValue& slval(static_cast<StringValue&>(*lval));
+                StringValue& srval(static_cast<StringValue&>(*rval));
+                return std::unique_ptr<Value>(new StringValue(
+                            slval.getValue() + srval.getValue()));
+            }
+        }
+        case SUB:
+        case MUL:
+        case DIV:
+        {
+            if (lval->getType() == Value::Integer &&
+                rval->getType() == Value::Integer)
+            {
+                IntegerValue& ilval(static_cast<IntegerValue&>(*lval));
+                IntegerValue& irval(static_cast<IntegerValue&>(*rval));
+                IntegerValue::ValueType res = 0;
+                switch (_operator) {
+                    case ADD: res = ilval.getValue() + irval.getValue(); break;
+                    case SUB: res = ilval.getValue() - irval.getValue(); break;
+                    case MUL: res = ilval.getValue() * irval.getValue(); break;
+                    case DIV:
+                        if (irval.getValue() != 0) {
+                            res = ilval.getValue() / irval.getValue();
+                        } else {
+                            throw vespalib::IllegalArgumentException("Division by zero");
+                        }
+                        break;
+                    case MOD: assert(0);
+                }
+                return std::unique_ptr<Value>(new IntegerValue(res, false));
+            }
+            NumberValue* nlval(dynamic_cast<NumberValue*>(lval.get()));
+            NumberValue* nrval(dynamic_cast<NumberValue*>(rval.get()));
+            if (nlval != 0 && nrval != 0) {
+                NumberValue::CommonValueType res = 0;
+                switch (_operator) {
+                    case ADD: res = nlval->getCommonValue()
+                                  + nrval->getCommonValue(); break;
+                    case SUB: res = nlval->getCommonValue()
+                                  - nrval->getCommonValue(); break;
+                    case MUL: res = nlval->getCommonValue()
+                                  * nrval->getCommonValue(); break;
+                    case DIV:
+                        if (nrval->getCommonValue() != 0) {
+                            res = nlval->getCommonValue()
+                                  / nrval->getCommonValue();
+                        } else {
+                            throw vespalib::IllegalArgumentException("Division by zero");
+                        }
+                        break;
+                    case MOD: assert(0);
+                }
+                return std::unique_ptr<Value>(new FloatValue(res));
+            }
+        }
+        case MOD:
+        {
+            if (lval->getType() == Value::Integer &&
+                rval->getType() == Value::Integer)
+            {
+                IntegerValue& ilval(static_cast<IntegerValue&>(*lval));
+                IntegerValue& irval(static_cast<IntegerValue&>(*rval));
+                if (irval.getValue() != 0) {
+                    return std::unique_ptr<Value>(new IntegerValue(ilval.getValue() % irval.getValue(), false));
+                } else {
+                    throw vespalib::IllegalArgumentException("Division by zero");
+                }
+            }
+        }
+    }
+    return std::unique_ptr<Value>(new InvalidValue);
+}
+
+std::unique_ptr<Value>
+ArithmeticValueNode::traceValue(std::unique_ptr<Value> lval,
+                                std::unique_ptr<Value> rval,
+                                std::ostream& out) const
+{
+    switch (_operator) {
+        case ADD:
+        {
+            if (lval->getType() == Value::String &&
+                rval->getType() == Value::String)
+            {
+                StringValue& slval(static_cast<StringValue&>(*lval));
+                StringValue& srval(static_cast<StringValue&>(*rval));
+                std::unique_ptr<Value> result(new StringValue(
+                            slval.getValue() + srval.getValue()));
+                out << "Appended strings '" << slval << "' + '" << srval
+                    << "' -> '" << *result << "'.\n";
+                return result;
+            }
+        }
+        case SUB:
+        case MUL:
+        case DIV:
+        {
+            if (lval->getType() == Value::Integer &&
+                rval->getType() == Value::Integer)
+            {
+                IntegerValue& ilval(static_cast<IntegerValue&>(*lval));
+                IntegerValue& irval(static_cast<IntegerValue&>(*rval));
+                IntegerValue::ValueType res = 0;
+                switch (_operator) {
+                    case ADD: res = ilval.getValue() + irval.getValue(); break;
+                    case SUB: res = ilval.getValue() - irval.getValue(); break;
+                    case MUL: res = ilval.getValue() * irval.getValue(); break;
+                    case DIV: res = ilval.getValue() / irval.getValue(); break;
+                    case MOD: assert(0);
+                }
+                std::unique_ptr<Value> result(new IntegerValue(res, false));
+                out << "Performed integer operation " << ilval << " "
+                    << getOperatorName() << " " << irval << " = " << *result
+                    << "\n";
+                return result;
+            }
+            NumberValue* nlval(dynamic_cast<NumberValue*>(lval.get()));
+            NumberValue* nrval(dynamic_cast<NumberValue*>(lval.get()));
+            if (nlval != 0 && nrval != 0) {
+                NumberValue::CommonValueType res = 0;
+                switch (_operator) {
+                    case ADD: res = nlval->getCommonValue()
+                                  + nrval->getCommonValue(); break;
+                    case SUB: res = nlval->getCommonValue()
+                                  - nrval->getCommonValue(); break;
+                    case MUL: res = nlval->getCommonValue()
+                                  * nrval->getCommonValue(); break;
+                    case DIV: res = nlval->getCommonValue()
+                                  / nrval->getCommonValue(); break;
+                    case MOD: assert(0);
+                }
+                std::unique_ptr<Value> result(new FloatValue(res));
+                out << "Performed float operation " << nlval << " "
+                    << getOperatorName() << " " << nrval << " = " << *result
+                    << "\n";
+                return result;
+            }
+        }
+        case MOD:
+        {
+            if (lval->getType() == Value::Integer &&
+                rval->getType() == Value::Integer)
+            {
+                IntegerValue& ilval(static_cast<IntegerValue&>(*lval));
+                IntegerValue& irval(static_cast<IntegerValue&>(*rval));
+                std::unique_ptr<Value> result(new IntegerValue(
+                            ilval.getValue() % irval.getValue(), false));
+                out << "Performed integer operation " << ilval << " "
+                    << getOperatorName() << " " << irval << " = " << *result
+                    << "\n";
+                return result;
+            }
+        }
+    }
+    out << "Failed to do operation " << getOperatorName()
+        << " on values of type " << lval->getType() << " and "
+        << rval->getType() << ". Resolving invalid.\n";
+    return std::unique_ptr<Value>(new InvalidValue);
+}
+
+
+void
+ArithmeticValueNode::visit(Visitor &visitor) const
+{
+    visitor.visitArithmeticValueNode(*this);
+}
+
+
+void
+ArithmeticValueNode::print(std::ostream& out, bool verbose,
+                           const std::string& indent) const
+{
+    if (hadParentheses()) out << '(';
+    _left->print(out, verbose, indent);
+    switch (_operator) {
+        case ADD: out << " + "; break;
+        case SUB: out << " - "; break;
+        case MUL: out << " * "; break;
+        case DIV: out << " / "; break;
+        case MOD: out << " % "; break;
+    }
+    _right->print(out, verbose, indent);
+    if (hadParentheses()) out << ')';
+}
+
+}
+
diff --git a/document/src/vespa/document/select/valuenodes.h b/document/src/vespa/document/select/valuenodes.h
new file mode 100644
index 00000000000..06d92190943
--- /dev/null
+++ b/document/src/vespa/document/select/valuenodes.h
@@ -0,0 +1,338 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "valuenode.h"
+#include <vespa/document/base/fieldpath.h>
+
+namespace document {
+
+class BucketDistribution;
+class BucketIdFactory;
+class DocumentId;
+class BucketId;
+class DocumentType;
+
+namespace select {
+
+class InvalidValueNode : public ValueNode
+{
+    vespalib::string _name;
+public:
+    InvalidValueNode(const vespalib::stringref & name);
+
+    std::unique_ptr<Value> getValue(const Context&) const override {
+        return std::unique_ptr<Value>(new InvalidValue());
+    }
+
+    void print(std::ostream& out, bool verbose, const std::string& indent) const override;
+    void visit(Visitor& visitor) const override;
+
+    ValueNode::UP clone() const override {
+        return wrapParens(new InvalidValueNode(_name));
+    }
+};
+
+class NullValueNode : public ValueNode
+{
+    vespalib::string _name;
+public:
+    NullValueNode(const vespalib::stringref & name);
+
+    std::unique_ptr<Value> getValue(const Context&) const override {
+        return std::unique_ptr<Value>(new NullValue());
+    }
+
+    void print(std::ostream& out, bool verbose, const std::string& indent) const override;
+
+    void visit(Visitor& visitor) const override;
+
+    ValueNode::UP clone() const override {
+        return wrapParens(new NullValueNode(_name));
+    }
+};
+
+class StringValueNode : public ValueNode
+{
+    vespalib::string _value;
+public:
+    StringValueNode(const vespalib::stringref & val);
+
+    const vespalib::string& getValue() const { return _value; }
+
+    std::unique_ptr<Value> getValue(const Context&) const override {
+        return std::unique_ptr<Value>(new StringValue(_value));
+    }
+
+    void print(std::ostream& out, bool verbose, const std::string& indent) const override;
+    void visit(Visitor& visitor) const override;
+
+    ValueNode::UP clone() const override {
+        return wrapParens(new StringValueNode(_value));
+    }
+};
+
+class IntegerValueNode : public ValueNode
+{
+    int64_t _value;
+    bool _isBucketValue;
+public:
+    IntegerValueNode(int64_t val, bool isBucketValue)
+        : _value(val), _isBucketValue(isBucketValue) {}
+
+    int64_t getValue() const { return _value; }
+
+    virtual std::unique_ptr<Value> getValue(const Context&) const override {
+        return std::unique_ptr<Value>(new IntegerValue(_value, _isBucketValue));
+    }
+
+    void print(std::ostream& out, bool verbose, const std::string& indent) const override;
+    void visit(Visitor& visitor) const override;
+
+    ValueNode::UP clone() const override {
+        return wrapParens(new IntegerValueNode(_value, _isBucketValue));
+    }
+};
+
+class CurrentTimeValueNode : public ValueNode
+{
+public:
+    int64_t getValue() const;
+
+    std::unique_ptr<Value> getValue(const Context&) const override {
+        return std::unique_ptr<Value>(new IntegerValue(getValue(), false));
+    }
+
+    void print(std::ostream& out, bool verbose, const std::string& indent) const override;
+    void visit(Visitor& visitor) const override;
+
+    ValueNode::UP clone() const override {
+        return wrapParens(new CurrentTimeValueNode);
+    }
+};
+
+class VariableValueNode : public ValueNode
+{
+    vespalib::string _value;
+public:
+    VariableValueNode(const vespalib::string & variableName) : _value(variableName) {}
+
+    const vespalib::string& getVariableName() const { return _value; }
+
+    std::unique_ptr<Value> getValue(const Context& context) const override;
+    void print(std::ostream& out, bool verbose, const std::string& indent) const override;
+    void visit(Visitor& visitor) const override;
+
+    ValueNode::UP clone() const override {
+        return wrapParens(new VariableValueNode(_value));
+    }
+};
+
+class FloatValueNode : public ValueNode
+{
+    double _value;
+public:
+    FloatValueNode(double val) : _value(val) {}
+
+    double getValue() const { return _value; }
+
+    std::unique_ptr<Value> getValue(const Context&) const override {
+        return std::unique_ptr<Value>(new FloatValue(_value));
+    }
+
+    void print(std::ostream& out, bool verbose, const std::string& indent) const override;
+    void visit(Visitor& visitor) const override;
+
+    ValueNode::UP clone() const override {
+        return wrapParens(new FloatValueNode(_value));
+    }
+};
+
+class FieldValueNode : public ValueNode
+{
+    vespalib::string _doctype;
+    vespalib::string _fieldExpression;
+    vespalib::string _fieldName;
+    mutable FieldPath _fieldPath;
+
+public:
+    FieldValueNode(const vespalib::string& doctype, const vespalib::string& fieldExpression);
+    FieldValueNode(const FieldValueNode &);
+    FieldValueNode & operator = (const FieldValueNode &);
+    FieldValueNode(FieldValueNode &&) = default;
+    FieldValueNode & operator = (FieldValueNode &&) = default;
+    ~FieldValueNode();
+
+    const vespalib::string& getDocType() const { return _doctype; }
+    const vespalib::string& getRealFieldName() const { return _fieldName; }
+    const vespalib::string& getFieldName() const { return _fieldExpression; }
+
+    std::unique_ptr<Value> getValue(const Context& context) const override;
+    std::unique_ptr<Value> traceValue(const Context &context, std::ostream& out) const override;
+    void print(std::ostream& out, bool verbose, const std::string& indent) const override;
+    void visit(Visitor& visitor) const override;
+
+    ValueNode::UP clone() const override {
+        return wrapParens(new FieldValueNode(_doctype, _fieldExpression));
+    }
+
+    static vespalib::string extractFieldName(const std::string & fieldExpression);
+
+private:
+
+    void initFieldPath(const DocumentType&) const;
+};
+
+class IdValueNode : public ValueNode
+{
+public:
+    enum Type { SCHEME, NS, TYPE, USER, GROUP, GID, SPEC, BUCKET, ORDER, ALL };
+
+    IdValueNode(const BucketIdFactory& bucketIdFactory,
+                const vespalib::stringref & name, const vespalib::stringref & type,
+                int widthBits = -1, int divisionBits = -1);
+
+    Type getType() const { return _type; }
+
+    std::unique_ptr<Value> getValue(const Context& context) const override;
+
+    std::unique_ptr<Value> getValue(const DocumentId& id) const;
+
+    std::unique_ptr<Value> traceValue(const Context& context, std::ostream &out) const override;
+
+    std::unique_ptr<Value> traceValue(const DocumentId& val, std::ostream& out) const;
+
+    void print(std::ostream& out, bool verbose, const std::string& indent) const override;
+
+    void visit(Visitor& visitor) const override;
+
+    ValueNode::UP clone() const override {
+        return wrapParens(new IdValueNode(_bucketIdFactory, _id, _typestring, _widthBits, _divisionBits));
+    }
+
+    int getWidthBits() const { return _widthBits; }
+    int getDivisionBits() const { return _divisionBits; }
+
+private:
+    const BucketIdFactory& _bucketIdFactory;
+    vespalib::string _id;
+    vespalib::string _typestring;
+    Type _type;
+    int _widthBits;
+    int _divisionBits;
+};
+
+class SearchColumnValueNode : public ValueNode
+{
+public:
+    SearchColumnValueNode(const BucketIdFactory& bucketIdFactory,
+                          const vespalib::stringref & name,
+                          int numColumns);
+
+    int getColumns() { return _numColumns; }
+
+    std::unique_ptr<Value> getValue(const Context& context) const override;
+    std::unique_ptr<Value> getValue(const DocumentId& id) const;
+    std::unique_ptr<Value> traceValue(const Context& context, std::ostream &out) const override;
+    std::unique_ptr<Value> traceValue(const DocumentId& val, std::ostream& out) const;
+    
+    int64_t getValue(const BucketId& bucketId) const;
+    void print(std::ostream& out, bool verbose, const std::string& indent) const override;
+    void visit(Visitor& visitor) const override;
+
+    ValueNode::UP clone() const override {
+        return wrapParens(new SearchColumnValueNode(_bucketIdFactory, _id, _numColumns));
+}
+
+private:
+    const BucketIdFactory& _bucketIdFactory;
+    vespalib::string _id;
+    int _numColumns;
+    std::unique_ptr<BucketDistribution> _distribution;
+};
+
+class FunctionValueNode : public ValueNode
+{
+public:
+    enum Function { LOWERCASE, HASH, ABS };
+
+    FunctionValueNode(const vespalib::stringref & name, std::unique_ptr<ValueNode> src);
+
+    Function getFunction() const { return _function; }
+    const vespalib::string &getFunctionName(void) const { return _funcname; }
+
+    std::unique_ptr<Value> getValue(const Context& context) const override {
+        return getValue(_source->getValue(context));
+    }
+
+    std::unique_ptr<Value> traceValue(const Context &context, std::ostream& out) const override {
+        return traceValue(_source->getValue(context), out);
+    }
+
+    void print(std::ostream& out, bool verbose, const std::string& indent) const override;
+    void visit(Visitor& visitor) const override;
+
+    ValueNode::UP clone() const override {
+        return wrapParens(new FunctionValueNode(_funcname, _source->clone()));
+    }
+
+    const ValueNode& getChild() const { return *_source; }
+
+private:
+    Function _function;
+    vespalib::string _funcname;
+    std::unique_ptr<ValueNode> _source;
+
+    virtual std::unique_ptr<Value> getValue(std::unique_ptr<Value> val) const;
+    virtual std::unique_ptr<Value> traceValue(std::unique_ptr<Value> val,
+                                            std::ostream& out) const;
+};
+
+class ArithmeticValueNode : public ValueNode
+{
+public:
+    enum Operator { ADD, SUB, MUL, DIV, MOD };
+
+    ArithmeticValueNode(std::unique_ptr<ValueNode> left,
+                        const vespalib::stringref & op,
+                        std::unique_ptr<ValueNode> right);
+
+    Operator getOperator() const { return _operator; }
+    const char* getOperatorName() const;
+
+    std::unique_ptr<Value>
+    getValue(const Context& context) const override {
+        return getValue(_left->getValue(context), _right->getValue(context));
+    }
+    
+    std::unique_ptr<Value>
+    traceValue(const Context &context, std::ostream& out) const override {
+        return traceValue(_left->getValue(context), _right->getValue(context), out);
+    }
+
+    void print(std::ostream& out, bool verbose, const std::string& indent) const override;
+    void visit(Visitor& visitor) const override;
+
+    ValueNode::UP clone() const override {
+        return wrapParens(new ArithmeticValueNode(_left->clone(),
+                                                  getOperatorName(),
+                                                  _right->clone()));
+    }
+
+    const ValueNode& getLeft() const { return *_left; }
+    const ValueNode& getRight() const { return *_right; }
+
+private:
+    Operator _operator;
+    std::unique_ptr<ValueNode> _left;
+    std::unique_ptr<ValueNode> _right;
+
+    virtual std::unique_ptr<Value> getValue(std::unique_ptr<Value> lval,
+                                          std::unique_ptr<Value> rval) const;
+    virtual std::unique_ptr<Value> traceValue(std::unique_ptr<Value> lval,
+                                            std::unique_ptr<Value> rval,
+                                            std::ostream&) const;
+};
+
+} // select
+} // document
diff --git a/document/src/vespa/document/select/variablemap.h b/document/src/vespa/document/select/variablemap.h
new file mode 100644
index 00000000000..86a1cb85e63
--- /dev/null
+++ b/document/src/vespa/document/select/variablemap.h
@@ -0,0 +1,15 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/vespalib/stllike/hash_map.h>
+#include <vespa/vespalib/stllike/string.h>
+
+namespace document::select {
+
+using VariableMapT = vespalib::hash_map<vespalib::string, double>;
+
+class VariableMap : public VariableMapT {
+public:
+    using VariableMapT::VariableMapT;
+};
+
+}
diff --git a/document/src/vespa/document/serialization/vespadocumentdeserializer.cpp b/document/src/vespa/document/serialization/vespadocumentdeserializer.cpp
index 93856b376aa..b2a2bd098a6 100644
--- a/document/src/vespa/document/serialization/vespadocumentdeserializer.cpp
+++ b/document/src/vespa/document/serialization/vespadocumentdeserializer.cpp
@@ -27,6 +27,7 @@
 #include <vespa/document/util/serializableexceptions.h>
 #include <vespa/document/base/exceptions.h>
 #include <vespa/vespalib/objects/nbostream.h>
+#include <vespa/document/util/bytebuffer.h>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".vespadocumentdeserializer");
diff --git a/document/src/vespa/document/serialization/vespadocumentserializer.cpp b/document/src/vespa/document/serialization/vespadocumentserializer.cpp
index 11317f2ff0a..1f6a4a4f0b0 100644
--- a/document/src/vespa/document/serialization/vespadocumentserializer.cpp
+++ b/document/src/vespa/document/serialization/vespadocumentserializer.cpp
@@ -24,7 +24,9 @@
 #include <vespa/document/update/fieldpathupdates.h>
 #include <vespa/vespalib/data/slime/binary_format.h>
 #include <vespa/vespalib/objects/nbostream.h>
+#include <vespa/vespalib/data/databuffer.h>
 #include <vespa/eval/tensor/serialization/typed_binary_format.h>
+#include <vespa/document/util/compressor.h>
 
 using std::make_pair;
 using std::pair;
diff --git a/document/src/vespa/document/update/addfieldpathupdate.cpp b/document/src/vespa/document/update/addfieldpathupdate.cpp
index 03a31d882f3..0f8b730367e 100644
--- a/document/src/vespa/document/update/addfieldpathupdate.cpp
+++ b/document/src/vespa/document/update/addfieldpathupdate.cpp
@@ -1,27 +1,24 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "addfieldpathupdate.h"
-#include <vespa/document/fieldvalue/fieldvalues.h>
-#include <vespa/document/select/parser.h>
+#include <vespa/document/fieldvalue/iteratorhandler.h>
+#include <vespa/document/fieldvalue/arrayfieldvalue.h>
 #include <vespa/document/serialization/vespadocumentdeserializer.h>
 #include <vespa/vespalib/objects/nbostream.h>
 #include <vespa/vespalib/util/exceptions.h>
-#include <vespa/log/log.h>
+#include <ostream>
 
 using vespalib::nbostream;
 
-LOG_SETUP(".document.update.fieldpathupdate");
-
 namespace document {
 
+using namespace fieldvalue;
+using vespalib::make_string;
+
 IMPLEMENT_IDENTIFIABLE(AddFieldPathUpdate, FieldPathUpdate);
 
-AddFieldPathUpdate::AddFieldPathUpdate(
-        const DocumentTypeRepo& repo,
-        const DataType& type,
-        stringref fieldPath,
-        stringref whereClause,
-        const ArrayFieldValue& values)
+AddFieldPathUpdate::AddFieldPathUpdate(const DocumentTypeRepo& repo, const DataType& type,
+                                       stringref fieldPath, stringref whereClause, const ArrayFieldValue& values)
     : FieldPathUpdate(repo, type, fieldPath, whereClause),
       _values(vespalib::CloneablePtr<ArrayFieldValue>(values.clone()))
 {
@@ -39,22 +36,33 @@ AddFieldPathUpdate::clone() const {
     return new AddFieldPathUpdate(*this);
 }
 
-FieldValue::IteratorHandler::ModificationStatus
-AddFieldPathUpdate::AddIteratorHandler::doModify(FieldValue& fv)
-{
-    LOG(spam, "Adding values to %s", fv.toString().c_str());
+namespace {
+
+class AddIteratorHandler : public fieldvalue::IteratorHandler {
+public:
+    AddIteratorHandler(const ArrayFieldValue &values) : _values(values) {}
+    fieldvalue::ModificationStatus doModify(FieldValue &fv) override;
+    bool createMissingPath() const override { return true; }
+    bool onComplex(const fieldvalue::IteratorHandler::Content &) override { return false; }
+private:
+    const ArrayFieldValue &_values;
+};
+
+
+ModificationStatus
+AddIteratorHandler::doModify(FieldValue &fv) {
     if (fv.inherits(CollectionFieldValue::classId)) {
-        CollectionFieldValue& cf = static_cast<CollectionFieldValue&>(fv);
+        CollectionFieldValue &cf = static_cast<CollectionFieldValue &>(fv);
         for (std::size_t i = 0; i < _values.size(); ++i) {
             cf.add(_values[i]);
         }
     } else {
-        vespalib::string err = vespalib::make_string(
-                "Unable to add a value to a \"%s\" field value.",
-                fv.getClass().name());
-                throw vespalib::IllegalArgumentException(err, VESPA_STRLOC);
+        vespalib::string err = make_string("Unable to add a value to a \"%s\" field value.", fv.getClass().name());
+        throw vespalib::IllegalArgumentException(err, VESPA_STRLOC);
     }
-    return MODIFIED;
+    return ModificationStatus::MODIFIED;
+}
+
 }
 
 bool
@@ -68,8 +76,7 @@ AddFieldPathUpdate::operator==(const FieldPathUpdate& other) const
 }
 
 void
-AddFieldPathUpdate::print(std::ostream& out, bool verbose,
-                          const std::string& indent) const
+AddFieldPathUpdate::print(std::ostream& out, bool verbose, const std::string& indent) const
 {
     out << "AddFieldPathUpdate(\n";
     FieldPathUpdate::print(out, verbose, indent + "  ");
@@ -79,8 +86,7 @@ AddFieldPathUpdate::print(std::ostream& out, bool verbose,
 }
 
 void
-AddFieldPathUpdate::deserialize(const DocumentTypeRepo& repo,
-                                const DataType& type,
+AddFieldPathUpdate::deserialize(const DocumentTypeRepo& repo, const DataType& type,
                                 ByteBuffer& buffer, uint16_t version)
 {
     FieldPathUpdate::deserialize(repo, type, buffer, version);
@@ -95,4 +101,11 @@ AddFieldPathUpdate::deserialize(const DocumentTypeRepo& repo,
     buffer.incPos(buffer.getRemaining() - stream.size());
 }
 
+std::unique_ptr<IteratorHandler>
+AddFieldPathUpdate::getIteratorHandler(Document&) const {
+    return std::make_unique<AddIteratorHandler>(*_values);
+}
+
+
+
 } // ns document
diff --git a/document/src/vespa/document/update/addfieldpathupdate.h b/document/src/vespa/document/update/addfieldpathupdate.h
index 6d75ebc4209..94156156257 100644
--- a/document/src/vespa/document/update/addfieldpathupdate.h
+++ b/document/src/vespa/document/update/addfieldpathupdate.h
@@ -10,7 +10,6 @@ class AddFieldPathUpdate : public FieldPathUpdate
 public:
     /** For deserialization */
     AddFieldPathUpdate();
-
     AddFieldPathUpdate(const DocumentTypeRepo& repo, const DataType& type, stringref fieldPath,
                        stringref whereClause, const ArrayFieldValue& values);
     ~AddFieldPathUpdate();
@@ -18,7 +17,6 @@ public:
     FieldPathUpdate* clone() const override;
     bool operator==(const FieldPathUpdate& other) const override;
     void print(std::ostream& out, bool verbose, const std::string& indent) const override;
-
     const ArrayFieldValue & getValues() const { return *_values; }
 
     DECLARE_IDENTIFIABLE(AddFieldPathUpdate);
@@ -29,22 +27,7 @@ private:
     void deserialize(const DocumentTypeRepo& repo, const DataType& type,
                      ByteBuffer& buffer, uint16_t version) override;
 
-    class AddIteratorHandler : public FieldValue::IteratorHandler
-    {
-    public:
-        AddIteratorHandler(const ArrayFieldValue& values) : _values(values) { }
-
-        ModificationStatus doModify(FieldValue& fv) override;
-        bool createMissingPath() const override { return true; }
-        bool onComplex(const Content&) override { return false; }
-    private:
-        const ArrayFieldValue& _values;
-    };
-
-    std::unique_ptr<FieldValue::IteratorHandler> getIteratorHandler(Document&) const override {
-        return std::unique_ptr<FieldValue::IteratorHandler>(
-                new AddIteratorHandler(*_values));
-    }
+    std::unique_ptr<fieldvalue::IteratorHandler> getIteratorHandler(Document&) const override;
 
     vespalib::CloneablePtr<ArrayFieldValue> _values;
 };
diff --git a/document/src/vespa/document/update/addvalueupdate.cpp b/document/src/vespa/document/update/addvalueupdate.cpp
index 910e3d7dd50..b1142836fc2 100644
--- a/document/src/vespa/document/update/addvalueupdate.cpp
+++ b/document/src/vespa/document/update/addvalueupdate.cpp
@@ -2,17 +2,17 @@
 #include "addvalueupdate.h"
 #include <vespa/document/base/field.h>
 #include <vespa/document/datatype/arraydatatype.h>
-#include <vespa/document/datatype/weightedsetdatatype.h>
 #include <vespa/document/fieldvalue/fieldvalues.h>
-#include <vespa/document/repo/fixedtyperepo.h>
 #include <vespa/document/serialization/vespadocumentdeserializer.h>
-#include <vespa/document/util/serializable.h>
 #include <vespa/document/util/serializableexceptions.h>
 #include <vespa/vespalib/objects/nbostream.h>
+#include <vespa/vespalib/util/xmlstream.h>
+
 
 using vespalib::IllegalArgumentException;
 using vespalib::IllegalStateException;
 using vespalib::nbostream;
+using namespace vespalib::xml;
 
 namespace document {
 
diff --git a/document/src/vespa/document/update/arithmeticvalueupdate.cpp b/document/src/vespa/document/update/arithmeticvalueupdate.cpp
index 29c24e7140f..a8fe541d0fe 100644
--- a/document/src/vespa/document/update/arithmeticvalueupdate.cpp
+++ b/document/src/vespa/document/update/arithmeticvalueupdate.cpp
@@ -3,9 +3,11 @@
 #include <vespa/document/base/field.h>
 #include <vespa/document/fieldvalue/fieldvalues.h>
 #include <vespa/vespalib/util/exceptions.h>
+#include <vespa/vespalib/util/xmlstream.h>
 
 using vespalib::IllegalArgumentException;
 using vespalib::IllegalStateException;
+using namespace vespalib::xml;
 
 namespace document
 {
diff --git a/document/src/vespa/document/update/assignfieldpathupdate.cpp b/document/src/vespa/document/update/assignfieldpathupdate.cpp
index 0a676c88df2..7ab0a43dbe1 100644
--- a/document/src/vespa/document/update/assignfieldpathupdate.cpp
+++ b/document/src/vespa/document/update/assignfieldpathupdate.cpp
@@ -1,9 +1,11 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "assignfieldpathupdate.h"
 #include <vespa/document/fieldvalue/fieldvalues.h>
-#include <vespa/document/repo/fixedtyperepo.h>
+#include <vespa/document/fieldvalue/iteratorhandler.h>
 #include <vespa/document/select/parser.h>
+#include <vespa/document/select/variablemap.h>
 #include <vespa/document/serialization/vespadocumentdeserializer.h>
-#include <vespa/document/update/assignfieldpathupdate.h>
 #include <vespa/vespalib/objects/nbostream.h>
 #include <vespa/vespalib/util/exceptions.h>
 #include <boost/numeric/conversion/cast.hpp>
@@ -15,6 +17,8 @@ using vespalib::nbostream;
 
 namespace document {
 
+using namespace fieldvalue;
+
 IMPLEMENT_IDENTIFIABLE(AssignFieldPathUpdate, FieldPathUpdate);
 
 AssignFieldPathUpdate::AssignFieldPathUpdate()
@@ -68,24 +72,56 @@ FieldPathUpdate*
 AssignFieldPathUpdate::clone() const {
     return new AssignFieldPathUpdate(*this);
 }
+namespace {
 
-std::unique_ptr<FieldValue::IteratorHandler>
-AssignFieldPathUpdate::getIteratorHandler(Document& doc) const
+class AssignValueIteratorHandler : public IteratorHandler
 {
-    if (!_expression.empty()) {
-        return std::unique_ptr<FieldValue::IteratorHandler>(
-                new AssignExpressionIteratorHandler(
-                        *_repo, doc, _expression, _removeIfZero, _createMissingPath));
-    } else {
-        return std::unique_ptr<FieldValue::IteratorHandler>(
-                new AssignValueIteratorHandler(
-                        *_newValue, _removeIfZero, _createMissingPath));
-    }
-}
-
-
-FieldValue::IteratorHandler::ModificationStatus
-AssignFieldPathUpdate::AssignValueIteratorHandler::doModify(FieldValue& fv) {
+public:
+    AssignValueIteratorHandler(const FieldValue& newValue,
+                               bool removeIfZero,
+                               bool createMissingPath_)
+            : _newValue(newValue), _removeIfZero(removeIfZero),
+              _createMissingPath(createMissingPath_)
+    {}
+
+    ModificationStatus doModify(FieldValue& fv) override;
+    bool onComplex(const Content&) override { return false; }
+    bool createMissingPath() const override { return _createMissingPath; }
+
+private:
+    const FieldValue& _newValue;
+    bool _removeIfZero;
+    bool _createMissingPath;
+};
+
+class AssignExpressionIteratorHandler : public IteratorHandler
+{
+public:
+    AssignExpressionIteratorHandler(
+            const DocumentTypeRepo& repo,
+            Document& doc,
+            const vespalib::string& expression,
+            bool removeIfZero,
+            bool createMissingPath_)
+            : _calc(repo, expression),
+              _doc(doc),
+              _removeIfZero(removeIfZero),
+              _createMissingPath(createMissingPath_)
+    {}
+
+    ModificationStatus doModify(FieldValue& fv) override;
+    bool onComplex(const Content&) override { return false; }
+    bool createMissingPath() const override { return _createMissingPath; }
+
+private:
+    DocumentCalculator _calc;
+    Document& _doc;
+    bool _removeIfZero;
+    bool _createMissingPath;
+};
+
+ModificationStatus
+AssignValueIteratorHandler::doModify(FieldValue& fv) {
     LOG(spam, "fv = %s", fv.toString().c_str());
     if (!(*fv.getDataType() == *_newValue.getDataType())) {
         std::string err = vespalib::make_string(
@@ -98,17 +134,18 @@ AssignFieldPathUpdate::AssignValueIteratorHandler::doModify(FieldValue& fv) {
         && _newValue.inherits(NumericFieldValueBase::classId)
         && static_cast<const NumericFieldValueBase&>(_newValue).getAsLong() == 0)
     {
-        return REMOVED;
+        return ModificationStatus::REMOVED;
     }
     fv.assign(_newValue);
-    return MODIFIED;
+    return ModificationStatus::MODIFIED;
 }
 
-FieldValue::IteratorHandler::ModificationStatus
-AssignFieldPathUpdate::AssignExpressionIteratorHandler::doModify(FieldValue& fv) {
+ModificationStatus
+AssignExpressionIteratorHandler::doModify(FieldValue& fv) {
     LOG(spam, "fv = %s", fv.toString().c_str());
     if (fv.inherits(NumericFieldValueBase::classId)) {
-        DocumentCalculator::VariableMap vars;
+        std::unique_ptr<select::VariableMap> varHolder = std::make_unique<select::VariableMap>();
+        select::VariableMap & vars = *varHolder;
         for (VariableMap::const_iterator i(getVariables().begin()),
                  e(getVariables().end()); i != e; ++i)
         {
@@ -122,18 +159,18 @@ AssignFieldPathUpdate::AssignExpressionIteratorHandler::doModify(FieldValue& fv)
         vars["value"] = fv.getAsDouble();
 
         try {
-            double res = _calc.evaluate(_doc, std::move(vars));
+            double res = _calc.evaluate(_doc, std::move(varHolder));
             if (_removeIfZero && static_cast<uint64_t>(res) == 0) {
-                return REMOVED;
+                return ModificationStatus::REMOVED;
             } else {
                 fv.assign(DoubleFieldValue(res));
             }
         } catch (const vespalib::IllegalArgumentException&) {
             // Divide by zero does not modify the document field
-            return NOT_MODIFIED;
+            return ModificationStatus::NOT_MODIFIED;
         } catch (const boost::bad_numeric_cast&) {
             // Underflow/overflow does not modify
-            return NOT_MODIFIED;
+            return ModificationStatus::NOT_MODIFIED;
         }
     } else {
         throw vespalib::IllegalArgumentException(
@@ -141,7 +178,19 @@ AssignFieldPathUpdate::AssignExpressionIteratorHandler::doModify(FieldValue& fv)
                                       fv.toString().c_str(), fv.getDataType()->toString().c_str()),
                 VESPA_STRLOC);
     }
-    return MODIFIED;
+    return ModificationStatus::MODIFIED;
+}
+
+}
+
+std::unique_ptr<IteratorHandler>
+AssignFieldPathUpdate::getIteratorHandler(Document& doc) const
+{
+    if (!_expression.empty()) {
+        return std::make_unique<AssignExpressionIteratorHandler>(*_repo, doc, _expression, _removeIfZero, _createMissingPath);
+    } else {
+        return std::make_unique<AssignValueIteratorHandler>(*_newValue, _removeIfZero, _createMissingPath);
+    }
 }
 
 bool
diff --git a/document/src/vespa/document/update/assignfieldpathupdate.h b/document/src/vespa/document/update/assignfieldpathupdate.h
index 81853246e54..3794cb331ce 100644
--- a/document/src/vespa/document/update/assignfieldpathupdate.h
+++ b/document/src/vespa/document/update/assignfieldpathupdate.h
@@ -58,53 +58,7 @@ private:
     void deserialize(const DocumentTypeRepo& repo, const DataType& type,
                      ByteBuffer& buffer, uint16_t version) override;
 
-    class AssignValueIteratorHandler : public FieldValue::IteratorHandler
-    {
-    public:
-        AssignValueIteratorHandler(const FieldValue& newValue,
-                              bool removeIfZero,
-                              bool createMissingPath_)
-            : _newValue(newValue), _removeIfZero(removeIfZero),
-            _createMissingPath(createMissingPath_)
-        {}
-
-        ModificationStatus doModify(FieldValue& fv) override;
-        bool onComplex(const Content&) override { return false; }
-        bool createMissingPath() const override { return _createMissingPath; }
-
-    private:
-        const FieldValue& _newValue;
-        bool _removeIfZero;
-        bool _createMissingPath;
-    };
-
-    class AssignExpressionIteratorHandler : public FieldValue::IteratorHandler
-    {
-    public:
-        AssignExpressionIteratorHandler(
-                const DocumentTypeRepo& repo,
-                Document& doc,
-                const vespalib::string& expression,
-                bool removeIfZero,
-                bool createMissingPath_)
-            : _calc(repo, expression),
-            _doc(doc),
-            _removeIfZero(removeIfZero),
-            _createMissingPath(createMissingPath_)
-        {}
-
-        ModificationStatus doModify(FieldValue& fv) override;
-        bool onComplex(const Content&) override { return false; }
-        bool createMissingPath() const override { return _createMissingPath; }
-
-    private:
-        DocumentCalculator _calc;
-        Document& _doc;
-        bool _removeIfZero;
-        bool _createMissingPath;
-    };
-
-    std::unique_ptr<FieldValue::IteratorHandler> getIteratorHandler(Document& doc) const override;
+    std::unique_ptr<fieldvalue::IteratorHandler> getIteratorHandler(Document& doc) const override;
 
     const DocumentTypeRepo *_repo;
     FieldValue::CP _newValue;
diff --git a/document/src/vespa/document/update/assignvalueupdate.cpp b/document/src/vespa/document/update/assignvalueupdate.cpp
index 0538bec10e5..63609ed18c1 100644
--- a/document/src/vespa/document/update/assignvalueupdate.cpp
+++ b/document/src/vespa/document/update/assignvalueupdate.cpp
@@ -3,14 +3,16 @@
 #include "assignvalueupdate.h"
 #include <vespa/document/base/field.h>
 #include <vespa/document/fieldvalue/fieldvalues.h>
-#include <vespa/document/repo/fixedtyperepo.h>
 #include <vespa/document/serialization/vespadocumentdeserializer.h>
 #include <vespa/vespalib/objects/nbostream.h>
 #include <vespa/vespalib/util/exceptions.h>
+#include <vespa/vespalib/util/xmlstream.h>
+
 
 using vespalib::IllegalArgumentException;
 using vespalib::IllegalStateException;
 using vespalib::nbostream;
+using namespace vespalib::xml;
 
 namespace document {
 
diff --git a/document/src/vespa/document/update/clearvalueupdate.cpp b/document/src/vespa/document/update/clearvalueupdate.cpp
index 13143e004ee..f34987ccfeb 100644
--- a/document/src/vespa/document/update/clearvalueupdate.cpp
+++ b/document/src/vespa/document/update/clearvalueupdate.cpp
@@ -1,15 +1,17 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
+#include "clearvalueupdate.h"
 #include <vespa/document/base/field.h>
-#include <vespa/document/update/clearvalueupdate.h>
 #include <vespa/document/fieldvalue/document.h>
 #include <vespa/vespalib/util/exceptions.h>
+#include <vespa/vespalib/util/xmlstream.h>
+#include <ostream>
 
 using vespalib::IllegalArgumentException;
 using vespalib::IllegalStateException;
+using namespace vespalib::xml;
 
-namespace document
-{
+namespace document {
 
 IMPLEMENT_IDENTIFIABLE(ClearValueUpdate, ValueUpdate);
 
diff --git a/document/src/vespa/document/update/documentupdate.cpp b/document/src/vespa/document/update/documentupdate.cpp
index 866a215505c..43fe6c28dc8 100644
--- a/document/src/vespa/document/update/documentupdate.cpp
+++ b/document/src/vespa/document/update/documentupdate.cpp
@@ -1,4 +1,5 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
 #include "documentupdate.h"
 #include "documentupdateflags.h"
 #include <vespa/document/fieldvalue/fieldvalues.h>
@@ -7,12 +8,15 @@
 #include <vespa/vespalib/objects/nbostream.h>
 #include <vespa/document/util/bufferexceptions.h>
 #include <vespa/document/base/exceptions.h>
+#include <vespa/document/datatype/documenttype.h>
+#include <vespa/vespalib/util/xmlstream.h>
 
 using vespalib::IllegalArgumentException;
 using vespalib::IllegalStateException;
 using vespalib::nbostream;
 using vespalib::make_string;
 using vespalib::string;
+using namespace vespalib::xml;
 
 namespace document {
 
@@ -103,7 +107,28 @@ DocumentUpdate::affectsDocumentBody() const
     return false;
 }
 
-// Print the content of this document update.
+const DocumentType&
+DocumentUpdate::getType() const {
+    return static_cast<const DocumentType &> (*_type);
+}
+
+DocumentUpdate&
+DocumentUpdate::addUpdate(const FieldUpdate& update) {
+    _updates.push_back(update);
+    return *this;
+}
+
+DocumentUpdate&
+DocumentUpdate::addFieldPathUpdate(const FieldPathUpdate::CP& update) {
+    _fieldPathUpdates.push_back(update);
+    return *this;
+}
+
+DocumentUpdate*
+DocumentUpdate::clone() const {
+    return new DocumentUpdate(*this);
+}
+
 void
 DocumentUpdate::print(std::ostream& out, bool verbose,
                       const std::string& indent) const
@@ -230,8 +255,7 @@ DocumentUpdate::deserialize42(const DocumentTypeRepo& repo, ByteBuffer& buffer)
     try{
         buffer.getShortNetwork(_version);
 
-        std::pair<const DocumentType *, DocumentId> typeAndId(
-                deserializeTypeAndId(repo, buffer));
+        std::pair<const DocumentType *, DocumentId> typeAndId(deserializeTypeAndId(repo, buffer));
         _type = typeAndId.first;
         _documentId = typeAndId.second;
         // Read field updates, if any.
diff --git a/document/src/vespa/document/update/documentupdate.h b/document/src/vespa/document/update/documentupdate.h
index 5d09bd36816..a1d3e910cb5 100644
--- a/document/src/vespa/document/update/documentupdate.h
+++ b/document/src/vespa/document/update/documentupdate.h
@@ -29,7 +29,6 @@
 #include "fieldpathupdate.h"
 #include <vespa/document/base/documentid.h>
 #include <vespa/document/base/field.h>
-#include <vespa/document/datatype/documenttype.h>
 #include <vespa/document/fieldvalue/fieldvalue.h>
 #include <vespa/document/util/bytebuffer.h>
 
@@ -118,19 +117,13 @@ public:
      * Add a field update to this document update.
      * @return A reference to this.
      */
-    DocumentUpdate& addUpdate(const FieldUpdate& update) {
-        _updates.push_back(update);
-        return *this;
-    }
+    DocumentUpdate& addUpdate(const FieldUpdate& update);
 
     /**
      * Add a fieldpath update to this document update.
      * @return A reference to this.
      */
-    DocumentUpdate& addFieldPathUpdate(const FieldPathUpdate::CP& update) {
-        _fieldPathUpdates.push_back(update);
-        return *this;
-    }
+    DocumentUpdate& addFieldPathUpdate(const FieldPathUpdate::CP& update);
 
     /** @return The list of updates. */
     const FieldUpdateV & getUpdates() const { return _updates; }
@@ -141,9 +134,8 @@ public:
     bool affectsDocumentBody() const;
 
     /** @return The type of document this update is for. */
-    const DocumentType& getType() const { return static_cast<const DocumentType &> (*_type); }
+    const DocumentType& getType() const;
 	
-    // Printable implementation
     void print(std::ostream& out, bool verbose, const std::string& indent) const override;
 
     void deserialize42(const DocumentTypeRepo&, ByteBuffer&);
@@ -155,13 +147,9 @@ public:
     void serialize42(vespalib::nbostream &stream) const;
     void serializeHEAD(vespalib::nbostream &stream) const;
 
-    // XmlSerializable implementation
     void printXml(XmlOutputStream&) const override;
 
-    // Cloneable implementation
-    virtual DocumentUpdate* clone() const {
-        return new DocumentUpdate(*this);
-    }
+    virtual DocumentUpdate* clone() const;
 
     /**
      * Sets whether this update should create the document it updates if that document does not exist.
@@ -199,7 +187,6 @@ private:
     DocumentUpdate();
 
     int deserializeFlags(int sizeAndFlags);
-
 };
 
 } // document
diff --git a/document/src/vespa/document/update/fieldpathupdate.cpp b/document/src/vespa/document/update/fieldpathupdate.cpp
index a4145a9b58a..11ed5f08c93 100644
--- a/document/src/vespa/document/update/fieldpathupdate.cpp
+++ b/document/src/vespa/document/update/fieldpathupdate.cpp
@@ -1,14 +1,21 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-#include <vespa/document/fieldvalue/fieldvalues.h>
+#include "fieldpathupdates.h"
+#include <vespa/document/fieldvalue/document.h>
+#include <vespa/document/fieldvalue/iteratorhandler.h>
 #include <vespa/document/select/parser.h>
-#include <vespa/document/update/fieldpathupdates.h>
 #include <vespa/document/util/serializableexceptions.h>
-#include <vespa/log/log.h>
+#include <ostream>
 
+#include <vespa/log/log.h>
 LOG_SETUP(".document.update.fieldpathupdate");
 
+using vespalib::make_string;
+using vespalib::IllegalArgumentException;
+
 namespace document {
 
+using namespace fieldvalue;
+
 IMPLEMENT_IDENTIFIABLE_ABSTRACT(FieldPathUpdate, Identifiable);
 
 namespace {
@@ -43,9 +50,9 @@ FieldPathUpdate::FieldPathUpdate(const DocumentTypeRepo& repo,
                  : std::unique_ptr<select::Node>())
 {
     if (!_fieldPath.get()) {
-        throw vespalib::IllegalArgumentException(
-                vespalib::make_string("Could not create field path update for: path='%s', where='%s'",
-                                      fieldPath.c_str(), whereClause.c_str()), VESPA_STRLOC);
+        throw IllegalArgumentException(
+                make_string("Could not create field path update for: path='%s', where='%s'",
+                            fieldPath.c_str(), whereClause.c_str()), VESPA_STRLOC);
     }
 }
 
@@ -61,7 +68,7 @@ FieldPathUpdate::operator==(const FieldPathUpdate& other) const
 void
 FieldPathUpdate::applyTo(Document& doc) const
 {
-    std::unique_ptr<FieldValue::IteratorHandler> handler(getIteratorHandler(doc));
+    std::unique_ptr<IteratorHandler> handler(getIteratorHandler(doc));
 
     if (!_whereClause) {
         doc.iterateNested(*_fieldPath, *handler);
@@ -70,7 +77,7 @@ FieldPathUpdate::applyTo(Document& doc) const
         for (select::ResultList::const_iterator i = results.begin();
              i != results.end(); ++i)
         {
-            LOG(spam, "vars = %s", FieldValue::IteratorHandler::toString(handler->getVariables()).c_str());
+            LOG(spam, "vars = %s", handler->getVariables().toString().c_str());
             if (*i->second == select::Result::True) {
                 handler->setVariables(i->first);
                 doc.iterateNested(*_fieldPath, *handler);
@@ -88,8 +95,7 @@ FieldPathUpdate::affectsDocumentBody() const
 }
 
 void
-FieldPathUpdate::print(std::ostream& out, bool,
-                       const std::string& indent) const
+FieldPathUpdate::print(std::ostream& out, bool, const std::string& indent) const
 {
     out << indent << "fieldPath='" << _originalFieldPath << "',\n"
         << indent << "whereClause='" << _originalWhereClause << "'";
@@ -99,10 +105,10 @@ void
 FieldPathUpdate::checkCompatibility(const FieldValue& fv) const
 {
     if ( !getResultingDataType().isValueType(fv)) {
-        throw vespalib::IllegalArgumentException(
-                vespalib::make_string("Cannot update a '%s' field with a '%s' value",
-                                      getResultingDataType().toString().c_str(),
-                                      fv.getDataType()->toString().c_str()),
+        throw IllegalArgumentException(
+                make_string("Cannot update a '%s' field with a '%s' value",
+                            getResultingDataType().toString().c_str(),
+                            fv.getDataType()->toString().c_str()),
                 VESPA_STRLOC);
     }
 }
@@ -111,8 +117,7 @@ const DataType&
 FieldPathUpdate::getResultingDataType() const
 {
     if (_fieldPath->empty()) {
-        throw vespalib::IllegalStateException("Cannot get resulting data "
-                "type from an empty field path", VESPA_STRLOC);
+        throw vespalib::IllegalStateException("Cannot get resulting data type from an empty field path", VESPA_STRLOC);
     }
     return _fieldPath->rbegin()->getDataType();
 }
@@ -138,9 +143,7 @@ FieldPathUpdate::deserialize(const DocumentTypeRepo& repo,
     try {
         _fieldPath = type.buildFieldPath(_originalFieldPath).release();
         if (!_fieldPath.get()) {
-            throw DeserializeException(
-                    vespalib::make_string("Invalid field path: '%s'", _originalFieldPath.c_str()),
-                    VESPA_STRLOC);
+            throw DeserializeException(make_string("Invalid field path: '%s'", _originalFieldPath.c_str()), VESPA_STRLOC);
         }
         _whereClause = !_originalWhereClause.empty()
                        ? parseDocumentSelection(_originalWhereClause, repo)
@@ -159,8 +162,7 @@ FieldPathUpdate::createInstance(const DocumentTypeRepo& repo,
     buffer.getByte(updateType);
 
     std::unique_ptr<FieldPathUpdate> update;
-    switch (updateType)
-    {
+    switch (updateType) {
     case 0:
         update.reset(new AssignFieldPathUpdate());
         break;
@@ -171,9 +173,7 @@ FieldPathUpdate::createInstance(const DocumentTypeRepo& repo,
         update.reset(new AddFieldPathUpdate());
         break;
     default:
-        throw DeserializeException(
-                vespalib::make_string("Unknown fieldpath update type: %d", updateType),
-                VESPA_STRLOC);
+        throw DeserializeException(make_string("Unknown fieldpath update type: %d", updateType), VESPA_STRLOC);
     }
     update->deserialize(repo, type, buffer, serializationVersion);
     return update;
diff --git a/document/src/vespa/document/update/fieldpathupdate.h b/document/src/vespa/document/update/fieldpathupdate.h
index 80a63d62281..0eaff9485bf 100644
--- a/document/src/vespa/document/update/fieldpathupdate.h
+++ b/document/src/vespa/document/update/fieldpathupdate.h
@@ -104,8 +104,7 @@ protected:
     enum SerializedMagic {AssignMagic=0, RemoveMagic=1, AddMagic=2};
 private:
     // TODO: rename to createIteratorHandler?
-    virtual std::unique_ptr<FieldValue::IteratorHandler> getIteratorHandler(
-            Document& doc) const = 0;
+    virtual std::unique_ptr<fieldvalue::IteratorHandler> getIteratorHandler(Document& doc) const = 0;
 
     vespalib::string _originalFieldPath;
     vespalib::string _originalWhereClause;
diff --git a/document/src/vespa/document/update/fieldupdate.cpp b/document/src/vespa/document/update/fieldupdate.cpp
index 3141fdf3a4f..e25b95e56c1 100644
--- a/document/src/vespa/document/update/fieldupdate.cpp
+++ b/document/src/vespa/document/update/fieldupdate.cpp
@@ -3,6 +3,7 @@
 #include "fieldupdate.h"
 #include <vespa/document/base/exceptions.h>
 #include <vespa/document/fieldvalue/document.h>
+#include <vespa/document/datatype/documenttype.h>
 
 namespace document {
 
diff --git a/document/src/vespa/document/update/mapvalueupdate.cpp b/document/src/vespa/document/update/mapvalueupdate.cpp
index 0a92fa7fb15..37d775bb204 100644
--- a/document/src/vespa/document/update/mapvalueupdate.cpp
+++ b/document/src/vespa/document/update/mapvalueupdate.cpp
@@ -5,13 +5,14 @@
 #include <vespa/document/serialization/vespadocumentdeserializer.h>
 #include <vespa/vespalib/objects/nbostream.h>
 #include <vespa/document/util/serializableexceptions.h>
+#include <vespa/vespalib/util/xmlstream.h>
 
 using vespalib::IllegalArgumentException;
 using vespalib::IllegalStateException;
 using vespalib::nbostream;
+using namespace vespalib::xml;
 
-namespace document
-{
+namespace document {
 
 IMPLEMENT_IDENTIFIABLE(MapValueUpdate, ValueUpdate);
 
diff --git a/document/src/vespa/document/update/removefieldpathupdate.cpp b/document/src/vespa/document/update/removefieldpathupdate.cpp
index 6c9d60abd2f..4af867995d9 100644
--- a/document/src/vespa/document/update/removefieldpathupdate.cpp
+++ b/document/src/vespa/document/update/removefieldpathupdate.cpp
@@ -1,11 +1,13 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "removefieldpathupdate.h"
-#include <vespa/document/fieldvalue/fieldvalues.h>
-#include <vespa/document/select/parser.h>
+#include <vespa/document/fieldvalue/iteratorhandler.h>
+#include <ostream>
 
 namespace document {
 
+using namespace fieldvalue;
+
 IMPLEMENT_IDENTIFIABLE(RemoveFieldPathUpdate, FieldPathUpdate);
 
 RemoveFieldPathUpdate::RemoveFieldPathUpdate()
@@ -30,8 +32,7 @@ RemoveFieldPathUpdate::operator==(const FieldPathUpdate& other) const
 }
 
 void
-RemoveFieldPathUpdate::print(std::ostream& out, bool verbose,
-                          const std::string& indent) const
+RemoveFieldPathUpdate::print(std::ostream& out, bool verbose, const std::string& indent) const
 {
     out << "RemoveFieldPathUpdate(\n";
     FieldPathUpdate::print(out, verbose, indent + "  ");
@@ -39,11 +40,28 @@ RemoveFieldPathUpdate::print(std::ostream& out, bool verbose,
 }
 
 void
-RemoveFieldPathUpdate::deserialize(
-        const DocumentTypeRepo& repo, const DataType& type,
-        ByteBuffer& buffer, uint16_t version)
+RemoveFieldPathUpdate::deserialize(const DocumentTypeRepo& repo, const DataType& type,
+                                   ByteBuffer& buffer, uint16_t version)
 {
     FieldPathUpdate::deserialize(repo, type, buffer, version);
 }
 
+namespace {
+
+class RemoveIteratorHandler : public IteratorHandler {
+public:
+    RemoveIteratorHandler() {}
+
+    ModificationStatus doModify(FieldValue &) override {
+        return ModificationStatus::REMOVED;
+    }
+};
+
+}
+
+std::unique_ptr<IteratorHandler>
+RemoveFieldPathUpdate::getIteratorHandler(Document&) const {
+    return std::make_unique<RemoveIteratorHandler>();
+}
+
 } // ns document
diff --git a/document/src/vespa/document/update/removefieldpathupdate.h b/document/src/vespa/document/update/removefieldpathupdate.h
index 6de2a105cbf..1a3865fe761 100644
--- a/document/src/vespa/document/update/removefieldpathupdate.h
+++ b/document/src/vespa/document/update/removefieldpathupdate.h
@@ -29,19 +29,7 @@ private:
     void deserialize(const DocumentTypeRepo& repo, const DataType& type,
                      ByteBuffer& buffer, uint16_t version) override;
 
-    class RemoveIteratorHandler : public FieldValue::IteratorHandler
-    {
-    public:
-        RemoveIteratorHandler() {}
-
-        ModificationStatus doModify(FieldValue&) override {
-            return REMOVED;
-        }
-    };
-
-    std::unique_ptr<FieldValue::IteratorHandler> getIteratorHandler(Document&) const override {
-        return std::make_unique<RemoveIteratorHandler>();
-    }
+    std::unique_ptr<fieldvalue::IteratorHandler> getIteratorHandler(Document&) const override;
 };
 
 
diff --git a/document/src/vespa/document/update/removevalueupdate.cpp b/document/src/vespa/document/update/removevalueupdate.cpp
index e0575f600da..34af57edbe8 100644
--- a/document/src/vespa/document/update/removevalueupdate.cpp
+++ b/document/src/vespa/document/update/removevalueupdate.cpp
@@ -1,20 +1,20 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include "removevalueupdate.h"
 #include <vespa/document/base/field.h>
 #include <vespa/document/datatype/arraydatatype.h>
 #include <vespa/document/datatype/weightedsetdatatype.h>
 #include <vespa/document/fieldvalue/fieldvalues.h>
-#include <vespa/document/repo/fixedtyperepo.h>
 #include <vespa/document/serialization/vespadocumentdeserializer.h>
-#include <vespa/document/update/removevalueupdate.h>
 #include <vespa/vespalib/objects/nbostream.h>
 #include <vespa/document/util/serializableexceptions.h>
+#include <vespa/vespalib/util/xmlstream.h>
 
 using vespalib::IllegalArgumentException;
 using vespalib::IllegalStateException;
 using vespalib::nbostream;
+using namespace vespalib::xml;
 
-namespace document
-{
+namespace document {
 
 IMPLEMENT_IDENTIFIABLE(RemoveValueUpdate, ValueUpdate);
 
diff --git a/document/src/vespa/document/util/xmlserializable.h b/document/src/vespa/document/util/xmlserializable.h
index 21e0b858dec..f74761a04f6 100644
--- a/document/src/vespa/document/util/xmlserializable.h
+++ b/document/src/vespa/document/util/xmlserializable.h
@@ -5,14 +5,6 @@
 #include <vespa/vespalib/util/xmlserializable.h>
 
 namespace document {
-    typedef vespalib::xml::XmlOutputStream XmlOutputStream;
-    typedef vespalib::xml::XmlSerializable XmlSerializable;
-    typedef vespalib::xml::XmlTag XmlTag;
-    typedef vespalib::xml::XmlEndTag XmlEndTag;
-    typedef vespalib::xml::XmlAttribute XmlAttribute;
-    typedef vespalib::xml::XmlContent XmlContent;
-    typedef vespalib::xml::XmlEscapedContent XmlEscapedContent;
-    typedef vespalib::xml::XmlBase64Content XmlBase64Content;
-    typedef vespalib::xml::XmlContentWrapper XmlContentWrapper;
+    using XmlSerializable = vespalib::xml::XmlSerializable;
+    using XmlOutputStream = vespalib::xml::XmlOutputStream;
 }
-
diff --git a/documentapi/src/tests/messagebus/messagebus_test.cpp b/documentapi/src/tests/messagebus/messagebus_test.cpp
index 58fc96b2e78..e4b04b0afc7 100644
--- a/documentapi/src/tests/messagebus/messagebus_test.cpp
+++ b/documentapi/src/tests/messagebus/messagebus_test.cpp
@@ -1,6 +1,7 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 #include <vespa/document/base/testdocrepo.h>
 #include <vespa/document/fieldvalue/document.h>
+#include <vespa/document/datatype/documenttype.h>
 #include <vespa/documentapi/documentapi.h>
 #include <vespa/vdslib/state/clusterstate.h>
 #include <vespa/vespalib/testkit/testapp.h>
diff --git a/documentapi/src/tests/messages/messages50test.cpp b/documentapi/src/tests/messages/messages50test.cpp
index 44aea1cb169..834bea0e757 100644
--- a/documentapi/src/tests/messages/messages50test.cpp
+++ b/documentapi/src/tests/messages/messages50test.cpp
@@ -1,11 +1,12 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "messages50test.h"
-#include <vespa/document/datatype/datatype.h>
-#include <vespa/document/fieldvalue/document.h>
-#include <vespa/document/update/fieldpathupdates.h>
 #include <vespa/documentapi/documentapi.h>
 #include <vespa/vdslib/container/writabledocumentlist.h>
+#include <vespa/document/update/fieldpathupdates.h>
+#include <vespa/document/datatype/documenttype.h>
+#include <vespa/document/bucket/bucketidfactory.h>
+#include <vespa/document/select/parser.h>
 
 using document::DataType;
 using document::DocumentTypeRepo;
diff --git a/documentapi/src/tests/messages/messages52test.cpp b/documentapi/src/tests/messages/messages52test.cpp
index 9339d23ee86..37a133add91 100644
--- a/documentapi/src/tests/messages/messages52test.cpp
+++ b/documentapi/src/tests/messages/messages52test.cpp
@@ -5,6 +5,7 @@
 #include "messages52test.h"
 #include <vespa/documentapi/documentapi.h>
 #include <vespa/document/update/fieldpathupdates.h>
+#include <vespa/document/datatype/documenttype.h>
 
 using document::DocumentTypeRepo;
 
diff --git a/documentapi/src/tests/policies/policies_test.cpp b/documentapi/src/tests/policies/policies_test.cpp
index 3527e62840b..5d92b495c48 100644
--- a/documentapi/src/tests/policies/policies_test.cpp
+++ b/documentapi/src/tests/policies/policies_test.cpp
@@ -23,7 +23,9 @@
 #include <vespa/vdslib/state/clusterstate.h>
 #include <vespa/document/base/testdocrepo.h>
 #include <vespa/document/fieldvalue/longfieldvalue.h>
+#include <vespa/document/datatype/documenttype.h>
 #include <vespa/vespalib/testkit/testapp.h>
+#include <vespa/vespalib/util/stringfmt.h>
 
 #include <vespa/log/log.h>
 LOG_SETUP("policies_test");
diff --git a/documentapi/src/tests/systemstate/systemstate.cpp b/documentapi/src/tests/systemstate/systemstate.cpp
index e3163937a3f..d1ca470f670 100644
--- a/documentapi/src/tests/systemstate/systemstate.cpp
+++ b/documentapi/src/tests/systemstate/systemstate.cpp
@@ -1,6 +1,7 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include <vespa/documentapi/messagebus/systemstate/systemstate.h>
+#include <vespa/documentapi/messagebus/systemstate/nodestate.h>
 #include <vespa/documentapi/messagebus/systemstate/systemstatehandle.h>
 #include <vespa/vespalib/testkit/testapp.h>
 
diff --git a/documentapi/src/vespa/documentapi/loadtypes/loadtypeset.cpp b/documentapi/src/vespa/documentapi/loadtypes/loadtypeset.cpp
index 4f0f3508d20..062ed58ca87 100644
--- a/documentapi/src/vespa/documentapi/loadtypes/loadtypeset.cpp
+++ b/documentapi/src/vespa/documentapi/loadtypes/loadtypeset.cpp
@@ -3,6 +3,7 @@
 #include "loadtypeset.h"
 #include <vespa/config-load-type.h>
 #include <vespa/config/config.h>
+#include <vespa/vespalib/util/stringfmt.h>
 #include <vespa/vespalib/stllike/hash_map.hpp>
 #include <vespa/config/helper/configgetter.hpp>
 
diff --git a/documentapi/src/vespa/documentapi/messagebus/documentprotocol.cpp b/documentapi/src/vespa/documentapi/messagebus/documentprotocol.cpp
index 0849a93acbc..6c63e33c745 100644
--- a/documentapi/src/vespa/documentapi/messagebus/documentprotocol.cpp
+++ b/documentapi/src/vespa/documentapi/messagebus/documentprotocol.cpp
@@ -4,16 +4,15 @@
 #include "routablefactories51.h"
 #include "routablefactories52.h"
 #include "routingpolicyfactories.h"
-#include <vespa/document/repo/documenttyperepo.h>
+#include "routablerepository.h"
+#include "routingpolicyrepository.h"
+#include "replymerger.h"
 #include <vespa/document/util/stringutil.h>
 #include <vespa/documentapi/documentapi.h>
-#include <vespa/documentapi/messagebus/replymerger.h>
 #include <vespa/messagebus/emptyreply.h>
-#include <vespa/messagebus/routing/routingcontext.h>
-#include <vespa/vespalib/component/versionspecification.h>
 #include <vespa/vespalib/util/exceptions.h>
-#include "routablerepository.h"
-#include "routingpolicyrepository.h"
+#include <sstream>
+
 #include <vespa/log/log.h>
 LOG_SETUP(".documentprotocol");
 
diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/batchdocumentupdatemessage.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/batchdocumentupdatemessage.cpp
index 01e95e9f035..aa95cba1785 100644
--- a/documentapi/src/vespa/documentapi/messagebus/messages/batchdocumentupdatemessage.cpp
+++ b/documentapi/src/vespa/documentapi/messagebus/messages/batchdocumentupdatemessage.cpp
@@ -3,8 +3,7 @@
 #include "batchdocumentupdatemessage.h"
 #include "batchdocumentupdatereply.h"
 #include <vespa/documentapi/messagebus/documentprotocol.h>
-#include <vespa/document/select/parser.h>
-#include <vespa/document/bucket/bucketselector.h>
+#include <vespa/document/bucket/bucketidfactory.h>
 #include <vespa/vespalib/util/exceptions.h>
 
 namespace documentapi {
diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/documentmessage.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/documentmessage.cpp
index e2903097934..af1203926b5 100644
--- a/documentapi/src/vespa/documentapi/messagebus/messages/documentmessage.cpp
+++ b/documentapi/src/vespa/documentapi/messagebus/messages/documentmessage.cpp
@@ -2,6 +2,7 @@
 
 #include "documentmessage.h"
 #include <vespa/documentapi/messagebus/documentprotocol.h>
+#include <cassert>
 
 namespace documentapi {
 
@@ -16,7 +17,7 @@ mbus::Reply::UP
 DocumentMessage::createReply() const
 {
     mbus::Reply::UP ret(doCreateReply().release());
-    assert(ret.get() != NULL);
+    assert(ret.get() != nullptr);
     return ret;
 }
 
diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/emptybucketsmessage.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/emptybucketsmessage.cpp
index 641feb85f9d..24ac666eae9 100644
--- a/documentapi/src/vespa/documentapi/messagebus/messages/emptybucketsmessage.cpp
+++ b/documentapi/src/vespa/documentapi/messagebus/messages/emptybucketsmessage.cpp
@@ -1,7 +1,6 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "emptybucketsmessage.h"
-#include <vespa/documentapi/messagebus/documentprotocol.h>
 
 namespace documentapi {
 
diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/removelocationmessage.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/removelocationmessage.cpp
index e07566cf22f..f23ecdf3556 100644
--- a/documentapi/src/vespa/documentapi/messagebus/messages/removelocationmessage.cpp
+++ b/documentapi/src/vespa/documentapi/messagebus/messages/removelocationmessage.cpp
@@ -1,4 +1,5 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
 #include "removelocationmessage.h"
 #include <vespa/documentapi/messagebus/documentprotocol.h>
 #include <vespa/document/select/parser.h>
diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/removelocationmessage.h b/documentapi/src/vespa/documentapi/messagebus/messages/removelocationmessage.h
index c56ef8b0ede..799fbe5bde8 100644
--- a/documentapi/src/vespa/documentapi/messagebus/messages/removelocationmessage.h
+++ b/documentapi/src/vespa/documentapi/messagebus/messages/removelocationmessage.h
@@ -3,8 +3,9 @@
 
 #include "documentmessage.h"
 #include <vespa/document/bucket/bucketid.h>
-#include <vespa/document/bucket/bucketselector.h>
-#include <vespa/document/select/parser.h>
+
+namespace document::select { class Parser; }
+namespace document { class BucketIdFactory; }
 
 namespace documentapi {
 
diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/documentrouteselectorpolicy.cpp b/documentapi/src/vespa/documentapi/messagebus/policies/documentrouteselectorpolicy.cpp
index 61c6648aee6..4dae20e52d5 100644
--- a/documentapi/src/vespa/documentapi/messagebus/policies/documentrouteselectorpolicy.cpp
+++ b/documentapi/src/vespa/documentapi/messagebus/policies/documentrouteselectorpolicy.cpp
@@ -12,6 +12,7 @@
 #include <vespa/messagebus/emptyreply.h>
 #include <vespa/messagebus/routing/routingtable.h>
 #include <vespa/messagebus/messagebus.h>
+#include <vespa/vespalib/util/stringfmt.h>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".documentrouteselectorpolicy");
diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/externpolicy.cpp b/documentapi/src/vespa/documentapi/messagebus/policies/externpolicy.cpp
index 3a35e995805..d1a2e7b135b 100644
--- a/documentapi/src/vespa/documentapi/messagebus/policies/externpolicy.cpp
+++ b/documentapi/src/vespa/documentapi/messagebus/policies/externpolicy.cpp
@@ -3,9 +3,7 @@
 #include <boost/tokenizer.hpp>
 #include <vespa/documentapi/messagebus/documentprotocol.h>
 #include <vespa/messagebus/emptyreply.h>
-#include <vespa/messagebus/errorcode.h>
-#include <vespa/messagebus/routing/route.h>
-#include <vespa/messagebus/routing/routingcontext.h>
+#include <vespa/vespalib/util/stringfmt.h>
 #include <vespa/slobrok/sbmirror.h>
 #include <vespa/fnet/frt/frt.h>
 
diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/messagetypepolicy.cpp b/documentapi/src/vespa/documentapi/messagebus/policies/messagetypepolicy.cpp
index 4d260a8e81f..ba4d0cff079 100644
--- a/documentapi/src/vespa/documentapi/messagebus/policies/messagetypepolicy.cpp
+++ b/documentapi/src/vespa/documentapi/messagebus/policies/messagetypepolicy.cpp
@@ -1,12 +1,26 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
 #include "messagetypepolicy.h"
 #include <vespa/documentapi/messagebus/documentprotocol.h>
+#include <vespa/messagebus/routing/route.h>
+#include <vespa/messagebus/routing/routingcontext.h>
 #include <vespa/vespalib/stllike/hash_map.hpp>
 
 using vespa::config::content::MessagetyperouteselectorpolicyConfig;
 
 namespace documentapi {
 
+namespace policy {
+
+using MessageTypeMapT = vespalib::hash_map<int, mbus::Route>;
+
+class MessageTypeMap : public MessageTypeMapT {
+public:
+    using MessageTypeMapT::MessageTypeMapT;
+};
+
+}
+
 MessageTypePolicy::MessageTypePolicy(const config::ConfigUri & configUri) :
     mbus::IRoutingPolicy(),
     config::IFetcherCallback<MessagetyperouteselectorpolicyConfig>(),
@@ -23,7 +37,7 @@ MessageTypePolicy::~MessageTypePolicy() {}
 void
 MessageTypePolicy::configure(std::unique_ptr<MessagetyperouteselectorpolicyConfig> cfg)
 {
-    std::unique_ptr<MessageTypeMap> map(new MessageTypeMap);
+    auto map = std::make_unique<policy::MessageTypeMap>();
     for (size_t i(0), m(cfg->route.size()); i < m; i++) {
         const MessagetyperouteselectorpolicyConfig::Route & r = cfg->route[i];
         (*map)[r.messagetype] = mbus::Route::parse(r.name);
@@ -38,8 +52,8 @@ void
 MessageTypePolicy::select(mbus::RoutingContext & context)
 {
     int messageType = context.getMessage().getType();
-    std::shared_ptr<MessageTypeMap> map = _map.get();
-    MessageTypeMap::const_iterator found = map->find(messageType);
+    std::shared_ptr<policy::MessageTypeMap> map = _map.get();
+    policy::MessageTypeMap::const_iterator found = map->find(messageType);
     if (found != map->end()) {
          context.addChild(found->second);
     } else {
diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/messagetypepolicy.h b/documentapi/src/vespa/documentapi/messagebus/policies/messagetypepolicy.h
index d9a324a79c5..18ae4d1acb8 100644
--- a/documentapi/src/vespa/documentapi/messagebus/policies/messagetypepolicy.h
+++ b/documentapi/src/vespa/documentapi/messagebus/policies/messagetypepolicy.h
@@ -1,20 +1,20 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 #pragma once
 
-#include <vespa/document/select/node.h>
-#include <map>
 #include <vespa/messagebus/routing/iroutingpolicy.h>
-#include <vespa/messagebus/routing/route.h>
-#include <vespa/messagebus/routing/routingcontext.h>
-#include <vespa/vespalib/util/sync.h>
 #include <vespa/vespalib/util/ptrholder.h>
 #include <vespa/config-messagetyperouteselectorpolicy.h>
 #include <vespa/config/config.h>
 #include <vespa/config/helper/configfetcher.h>
 #include <vespa/documentapi/common.h>
 
+namespace mbus {
+    class RoutingContext;
+    class Route;
+}
 namespace documentapi {
 
+namespace policy {class MessageTypeMap; }
 /**
  * This policy is responsible for selecting among the given recipient routes
  * according to the configured document selection properties. To factilitate
@@ -26,8 +26,7 @@ class MessageTypePolicy : public mbus::IRoutingPolicy,
                           public config::IFetcherCallback<vespa::config::content::MessagetyperouteselectorpolicyConfig>
 {
 private:
-    typedef vespalib::hash_map<int, mbus::Route> MessageTypeMap;
-    typedef vespalib::PtrHolder<MessageTypeMap> MessageTypeHolder;
+    typedef vespalib::PtrHolder<policy::MessageTypeMap> MessageTypeHolder;
     typedef vespalib::PtrHolder<mbus::Route> RouteHolder;
 
     MessageTypeHolder     _map;
diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/storagepolicy.cpp b/documentapi/src/vespa/documentapi/messagebus/policies/storagepolicy.cpp
index 5aaf7919d71..f19e0c4c85f 100644
--- a/documentapi/src/vespa/documentapi/messagebus/policies/storagepolicy.cpp
+++ b/documentapi/src/vespa/documentapi/messagebus/policies/storagepolicy.cpp
@@ -7,6 +7,7 @@
 #include <vespa/documentapi/documentapi.h>
 #include <vespa/vdslib/state/clusterstate.h>
 #include <vespa/vespalib/stllike/asciistream.h>
+#include <vespa/vespalib/util/stringfmt.h>
 #include <vespa/config-stor-distribution.h>
 #include <vespa/config/helper/ifetchercallback.h>
 #include <vespa/config/helper/configfetcher.h>
diff --git a/documentapi/src/vespa/documentapi/messagebus/replymerger.cpp b/documentapi/src/vespa/documentapi/messagebus/replymerger.cpp
index 50cf7129016..ef791cc55f1 100644
--- a/documentapi/src/vespa/documentapi/messagebus/replymerger.cpp
+++ b/documentapi/src/vespa/documentapi/messagebus/replymerger.cpp
@@ -6,6 +6,7 @@
 #include <vespa/documentapi/messagebus/messages/updatedocumentreply.h>
 #include <vespa/documentapi/messagebus/messages/getdocumentreply.h>
 #include <vespa/messagebus/emptyreply.h>
+#include <cassert>
 
 namespace documentapi {
 
diff --git a/documentapi/src/vespa/documentapi/messagebus/routablefactories50.cpp b/documentapi/src/vespa/documentapi/messagebus/routablefactories50.cpp
index b620611695e..d9c8cd765a1 100644
--- a/documentapi/src/vespa/documentapi/messagebus/routablefactories50.cpp
+++ b/documentapi/src/vespa/documentapi/messagebus/routablefactories50.cpp
@@ -4,6 +4,8 @@
 #include <vespa/documentapi/documentapi.h>
 #include <vespa/documentapi/loadtypes/loadtypeset.h>
 #include <vespa/vespalib/objects/nbostream.h>
+#include <vespa/document/bucket/bucketidfactory.h>
+#include <vespa/document/select/parser.h>
 
 using vespalib::nbostream;
 using std::make_unique;
diff --git a/documentapi/src/vespa/documentapi/messagebus/routablerepository.cpp b/documentapi/src/vespa/documentapi/messagebus/routablerepository.cpp
index 4572b2bbcc6..32956ff23bd 100644
--- a/documentapi/src/vespa/documentapi/messagebus/routablerepository.cpp
+++ b/documentapi/src/vespa/documentapi/messagebus/routablerepository.cpp
@@ -4,6 +4,8 @@
 #include <vespa/documentapi/loadtypes/loadtypeset.h>
 #include <vespa/document/util/stringutil.h>
 #include <vespa/vespalib/util/exceptions.h>
+#include <sstream>
+
 #include <vespa/log/log.h>
 LOG_SETUP(".routablerepository");
 
@@ -11,9 +13,7 @@ namespace documentapi {
 
 RoutableRepository::VersionMap::VersionMap() :
     _factoryVersions()
-{
-    // empty
-}
+{ }
 
 bool
 RoutableRepository::VersionMap::putFactory(const vespalib::VersionSpecification &version,
diff --git a/documentapi/src/vespa/documentapi/messagebus/systemstate/systemstate.cpp b/documentapi/src/vespa/documentapi/messagebus/systemstate/systemstate.cpp
index 627a760b324..64e05fec793 100644
--- a/documentapi/src/vespa/documentapi/messagebus/systemstate/systemstate.cpp
+++ b/documentapi/src/vespa/documentapi/messagebus/systemstate/systemstate.cpp
@@ -1,6 +1,8 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "systemstate.h"
+#include "nodestate.h"
+#include <vespa/vespalib/util/sync.h>
 #include <vespa/vespalib/util/stringfmt.h>
 #include <boost/spirit/include/classic_core.hpp>
 #include <boost/spirit/include/classic_parse_tree.hpp>
@@ -256,7 +258,9 @@ parseSystemState(SystemStateGrammar &grammar, boost::spirit::classic::tree_node<
     return ret;
 }
 
-vespalib::Lock SystemState::_parseLock;
+namespace {
+    vespalib::Lock _G_parseLock;
+}
 
 SystemState::UP
 SystemState::newInstance(const string &state)
@@ -265,7 +269,7 @@ SystemState::newInstance(const string &state)
         return SystemState::UP(new SystemState(NodeState::UP(new NodeState())));
     }
     try {
-        vespalib::LockGuard guard(_parseLock);
+        vespalib::LockGuard guard(_G_parseLock);
         SystemStateGrammar grammar;
         boost::spirit::classic::tree_parse_info<> info =
             boost::spirit::classic::pt_parse(static_cast<const char *>(&*state.begin()),
@@ -289,15 +293,14 @@ SystemState::newInstance(const string &state)
         }
     }
     catch(std::exception& e) {
-        std::cerr << "SystemState::parse() internal error: "
-                  << e.what() << std::endl;
+        LOG(fatal, "SystemState::parse() internal error: %s", e.what());
     }
     return SystemState::UP();
 }
 
 SystemState::SystemState(NodeState::UP root) :
     _root(std::move(root)),
-    _lock() {
-    // empty
-}
+    _lock(std::make_unique<vespalib::Lock>())
+{}
 
+SystemState::~SystemState() {}
+\ No newline at end of file
diff --git a/documentapi/src/vespa/documentapi/messagebus/systemstate/systemstate.h b/documentapi/src/vespa/documentapi/messagebus/systemstate/systemstate.h
index 1d958c42b2d..169e1fbb9b6 100644
--- a/documentapi/src/vespa/documentapi/messagebus/systemstate/systemstate.h
+++ b/documentapi/src/vespa/documentapi/messagebus/systemstate/systemstate.h
@@ -1,11 +1,13 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 #pragma once
 
-#include "nodestate.h"
-#include <vespa/vespalib/util/sync.h>
+#include <vespa/documentapi/common.h>
 
+namespace vespalib { class Lock; }
 namespace documentapi {
 
+class NodeState;
+
 /**
  * This class is a factory to create a tree of {@link NodeState} objects from a parseable node state
  * string. The naming of this class is intended to capture the fact that this annotated service tree actually
@@ -13,10 +15,8 @@ namespace documentapi {
  */
 class SystemState {
 private:
-    static vespalib::Lock _parseLock;
-
-    NodeState::UP  _root;
-    vespalib::Lock _lock;
+    std::unique_ptr<NodeState>      _root;
+    std::unique_ptr<vespalib::Lock> _lock;
 
     friend class SystemStateHandle;
 
@@ -26,9 +26,10 @@ private:
      *
      * @param root The root node state.
      */
-    SystemState(NodeState::UP root);
+    SystemState(std::unique_ptr<NodeState> root);
 
 public:
+    ~SystemState();
     SystemState(const SystemState &) = delete;
     SystemState & operator = (const SystemState &) = delete;
     /**
@@ -47,4 +48,3 @@ public:
 };
 
 }
-
diff --git a/documentapi/src/vespa/documentapi/messagebus/systemstate/systemstatehandle.cpp b/documentapi/src/vespa/documentapi/messagebus/systemstate/systemstatehandle.cpp
index 6d3d9a19568..ea43665facb 100644
--- a/documentapi/src/vespa/documentapi/messagebus/systemstate/systemstatehandle.cpp
+++ b/documentapi/src/vespa/documentapi/messagebus/systemstate/systemstatehandle.cpp
@@ -11,14 +11,14 @@ SystemStateHandover::SystemStateHandover(SystemState *state, vespalib::LockGuard
 
 SystemStateHandle::SystemStateHandle(SystemState &state) :
     _state(&state),
-    _guard(state._lock)
+    _guard(*state._lock)
 {}
 
 SystemStateHandle::SystemStateHandle(SystemStateHandle &rhs) :
     _state(rhs._state),
     _guard(rhs._guard)
 {
-    rhs._state = NULL;
+    rhs._state = nullptr;
 }
 
 SystemStateHandle::SystemStateHandle(const SystemStateHandover &rhs) :
@@ -31,7 +31,7 @@ SystemStateHandle::~SystemStateHandle() {}
 SystemStateHandle::operator
 SystemStateHandover() {
     SystemStateHandover ret(_state, _guard);
-    _state = NULL;
+    _state = nullptr;
     return ret;
 }
 
diff --git a/eval/src/tests/eval/gbdt/gbdt_test.cpp b/eval/src/tests/eval/gbdt/gbdt_test.cpp
index c96779edeb0..0535c7280bf 100644
--- a/eval/src/tests/eval/gbdt/gbdt_test.cpp
+++ b/eval/src/tests/eval/gbdt/gbdt_test.cpp
@@ -323,4 +323,34 @@ TEST("require that forests evaluate to approximately the same for all evaluation
 
 //-----------------------------------------------------------------------------
 
+TEST("require that GDBT expressions can be detected") {
+    Function function = Function::parse("if((a<1),1.0,if((b in [1,2,3]),if((c in 1),2.0,3.0),4.0))+"
+                                        "if((d in 1),10.0,if((e<1),20.0,30.0))+"
+                                        "if((d in 1),10.0,if((e<1),20.0,30.0))");
+    EXPECT_TRUE(contains_gbdt(function.root(), 9));
+    EXPECT_TRUE(!contains_gbdt(function.root(), 10));
+}
+
+TEST("require that wrapped GDBT expressions can be detected") {
+    Function function = Function::parse("10*(if((a<1),1.0,if((b in [1,2,3]),if((c in 1),2.0,3.0),4.0))+"
+                                        "if((d in 1),10.0,if((e<1),20.0,30.0))+"
+                                        "if((d in 1),10.0,if((e<1),20.0,30.0)))");
+    EXPECT_TRUE(contains_gbdt(function.root(), 9));
+    EXPECT_TRUE(!contains_gbdt(function.root(), 10));
+}
+
+TEST("require that lazy parameters are not suggested for GBDT models") {
+    Function function = Function::parse(Model().make_forest(10, 8));
+    EXPECT_TRUE(!CompiledFunction::should_use_lazy_params(function));
+}
+
+TEST("require that lazy parameters can be suggested for small GBDT models") {
+    Function function = Function::parse("if((a<1),1.0,if((b in [1,2,3]),if((c in 1),2.0,3.0),4.0))+"
+                                        "if((d in 1),10.0,if((e<1),20.0,30.0))+"
+                                        "if((d in 1),10.0,if((e<1),20.0,30.0))");
+    EXPECT_TRUE(CompiledFunction::should_use_lazy_params(function));
+}
+
+//-----------------------------------------------------------------------------
+
 TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/eval/src/tests/eval/param_usage/param_usage_test.cpp b/eval/src/tests/eval/param_usage/param_usage_test.cpp
index ff0c6667279..24478046289 100644
--- a/eval/src/tests/eval/param_usage/param_usage_test.cpp
+++ b/eval/src/tests/eval/param_usage/param_usage_test.cpp
@@ -2,6 +2,7 @@
 #include <vespa/vespalib/testkit/test_kit.h>
 #include <vespa/eval/eval/function.h>
 #include <vespa/eval/eval/param_usage.h>
+#include <vespa/eval/eval/llvm/compiled_function.h>
 #include <vespa/vespalib/test/insertion_operators.h>
 
 using vespalib::approx_equal;
@@ -62,4 +63,14 @@ TEST("require that multi-level if statements are combined correctly") {
     EXPECT_EQUAL(List(check_param_usage(function)), List({1.0, 0.5, 1.0, 1.0}));
 }
 
+TEST("require that lazy parameters are suggested for functions with parameters that might not be used") {
+    Function function = Function::parse("if(z,x,y)+if(w,y,x)");
+    EXPECT_TRUE(CompiledFunction::should_use_lazy_params(function));
+}
+
+TEST("require that lazy parameters are not suggested for functions where all parameters are always used") {
+    Function function = Function::parse("a*b*c");
+    EXPECT_TRUE(!CompiledFunction::should_use_lazy_params(function));
+}
+
 TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/eval/src/vespa/eval/eval/gbdt.cpp b/eval/src/vespa/eval/eval/gbdt.cpp
index 4b0fc8bf066..b787a9a04f0 100644
--- a/eval/src/vespa/eval/eval/gbdt.cpp
+++ b/eval/src/vespa/eval/eval/gbdt.cpp
@@ -2,6 +2,7 @@
 
 #include "gbdt.h"
 #include "vm_forest.h"
+#include "node_traverser.h"
 #include <vespa/eval/eval/basic_nodes.h>
 #include <vespa/eval/eval/call_nodes.h>
 #include <vespa/eval/eval/operator_nodes.h>
@@ -117,6 +118,25 @@ ForestStats::ForestStats(const std::vector<const nodes::Node *> &trees)
 
 //-----------------------------------------------------------------------------
 
+bool contains_gbdt(const nodes::Node &node, size_t limit) {
+    struct FindGBDT : NodeTraverser {
+        size_t seen;
+        size_t limit;
+        explicit FindGBDT(size_t limit_in) : seen(0), limit(limit_in) {}
+        bool found() const { return (seen >= limit); }
+        bool open(const nodes::Node &) override { return !found(); }
+        void close(const nodes::Node &node) override {
+            if (node.is_tree() || node.is_forest()) {
+                ++seen;
+            }
+        }
+    } findGBDT(limit);
+    node.traverse(findGBDT);
+    return findGBDT.found(); 
+}
+
+//-----------------------------------------------------------------------------
+
 Optimize::Result
 Optimize::select_best(const ForestStats &stats,
                       const std::vector<const nodes::Node *> &trees)
diff --git a/eval/src/vespa/eval/eval/gbdt.h b/eval/src/vespa/eval/eval/gbdt.h
index 6cf48528dff..eda0d16229f 100644
--- a/eval/src/vespa/eval/eval/gbdt.h
+++ b/eval/src/vespa/eval/eval/gbdt.h
@@ -60,6 +60,15 @@ struct ForestStats {
 //-----------------------------------------------------------------------------
 
 /**
+ * Check if the given sub-expression contains GBDT. This function
+ * returns true if the number of tree/forest nodes exceeds the given
+ * limit.
+ **/
+bool contains_gbdt(const nodes::Node &node, size_t limit);
+
+//-----------------------------------------------------------------------------
+
+/**
  * A Forest object represents deletable custom prepared state that may
  * be used to evaluate a GBDT forest from within LLVM generated
  * machine code. It is very important that the evaluation function
diff --git a/eval/src/vespa/eval/eval/llvm/compiled_function.cpp b/eval/src/vespa/eval/eval/llvm/compiled_function.cpp
index f56272231cb..30d21a987e0 100644
--- a/eval/src/vespa/eval/eval/llvm/compiled_function.cpp
+++ b/eval/src/vespa/eval/eval/llvm/compiled_function.cpp
@@ -1,11 +1,14 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "compiled_function.h"
+#include <vespa/eval/eval/param_usage.h>
+#include <vespa/eval/eval/gbdt.h>
 #include <vespa/eval/eval/node_traverser.h>
 #include <vespa/eval/eval/check_type.h>
 #include <vespa/eval/eval/tensor_nodes.h>
 #include <vespa/vespalib/util/classname.h>
 #include <vespa/vespalib/util/benchmark_timer.h>
+#include <vespa/vespalib/util/approx.h>
 
 namespace vespalib {
 namespace eval {
@@ -136,5 +139,20 @@ CompiledFunction::detect_issues(const Function &function)
     return Function::Issues(std::move(checker.issues));
 }
 
+bool
+CompiledFunction::should_use_lazy_params(const Function &function)
+{
+    if (gbdt::contains_gbdt(function.root(), 16)) {
+        return false; // contains gbdt
+    }
+    auto usage = vespalib::eval::check_param_usage(function);
+    for (double p_use: usage) {
+        if (!approx_equal(p_use, 1.0)) {
+            return true; // param not always used
+        }
+    }
+    return false; // all params always used
+}
+
 } // namespace vespalib::eval
 } // namespace vespalib
diff --git a/eval/src/vespa/eval/eval/llvm/compiled_function.h b/eval/src/vespa/eval/eval/llvm/compiled_function.h
index 912453f290a..13ef322e03b 100644
--- a/eval/src/vespa/eval/eval/llvm/compiled_function.h
+++ b/eval/src/vespa/eval/eval/llvm/compiled_function.h
@@ -63,6 +63,7 @@ public:
     }
     double estimate_cost_us(const std::vector<double> &params, double budget = 5.0) const;
     static Function::Issues detect_issues(const Function &function);
+    static bool should_use_lazy_params(const Function &function);
 };
 
 } // namespace vespalib::eval
diff --git a/eval/src/vespa/eval/eval/llvm/llvm_wrapper.cpp b/eval/src/vespa/eval/eval/llvm/llvm_wrapper.cpp
index fa3cbfdbb58..4a0aef82f6b 100644
--- a/eval/src/vespa/eval/eval/llvm/llvm_wrapper.cpp
+++ b/eval/src/vespa/eval/eval/llvm/llvm_wrapper.cpp
@@ -629,7 +629,12 @@ struct InitializeNativeTarget {
     }
     ~InitializeNativeTarget() {
         llvm::llvm_shutdown();
+#ifdef HAS_LLVM_DESTROY_STATIC_MUTEX
         llvm::llvm_destroy_static_mutex();
+#endif
+#ifdef HAS_LLVM_DESTROY_OPENED_HANDLES
+        llvm::llvm_destroy_opened_handles();
+#endif
     }
 } initialize_native_target;
 
diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor_builder.cpp b/eval/src/vespa/eval/tensor/dense/dense_tensor_builder.cpp
index a00537ffa50..3c6e182a143 100644
--- a/eval/src/vespa/eval/tensor/dense/dense_tensor_builder.cpp
+++ b/eval/src/vespa/eval/tensor/dense/dense_tensor_builder.cpp
@@ -2,6 +2,7 @@
 
 #include "dense_tensor_builder.h"
 #include <vespa/vespalib/util/exceptions.h>
+#include <vespa/vespalib/util/stringfmt.h>
 #include <cassert>
 
 using vespalib::IllegalArgumentException;
diff --git a/fastos/src/tests/tests.h b/fastos/src/tests/tests.h
index cc54e66292f..6807741f86f 100644
--- a/fastos/src/tests/tests.h
+++ b/fastos/src/tests/tests.h
@@ -2,6 +2,7 @@
 
 #include <vespa/fastos/app.h>
 #include <vespa/fastos/socket.h>
+#include <vespa/fastos/thread.h>
 
 class BaseTest : public FastOS_Application
 {
diff --git a/fastos/src/vespa/fastos/app.h b/fastos/src/vespa/fastos/app.h
index 75fb5c30d43..faa6ff55c7d 100644
--- a/fastos/src/vespa/fastos/app.h
+++ b/fastos/src/vespa/fastos/app.h
@@ -13,8 +13,8 @@
 #include <vespa/fastos/types.h>
 
 class FastOS_ProcessInterface;
+class FastOS_ThreadPool;
 
-#include <vespa/fastos/thread.h>
 #include <vespa/fastos/mutex.h>
 
 /**
diff --git a/fastos/src/vespa/fastos/time.h b/fastos/src/vespa/fastos/time.h
index 97ebe21a3b5..1e37813a072 100644
--- a/fastos/src/vespa/fastos/time.h
+++ b/fastos/src/vespa/fastos/time.h
@@ -1,8 +1,6 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 #pragma once
 
-#include <vespa/fastos/types.h>
-
 /**
  * Interface to OS time functions.
  */
@@ -12,7 +10,7 @@ protected:
     /**
      * Destructor.  No cleanup needed for base class.
      */
-    virtual ~FastOS_TimeInterface(void) { }
+    virtual ~FastOS_TimeInterface() { }
 
 public:
     /**
@@ -49,30 +47,26 @@ public:
      * Note: Only millisecond accuracy is guaranteed.
      * @param  microsecs    Number of microseconds to add.
      */
-    void AddMicroSecs(double microsecs)
-    { SetMicroSecs(MicroSecs() + microsecs); }
+    void AddMicroSecs(double microsecs) { SetMicroSecs(MicroSecs() + microsecs); }
 
     /**
      * Add a specified number of milliseconds to the time.
      * @param  millisecs    Number of milliseconds to add.
      */
-    void AddMilliSecs(double millisecs)
-    { SetMilliSecs(MilliSecs() + millisecs); }
+    void AddMilliSecs(double millisecs) { SetMilliSecs(MilliSecs() + millisecs); }
 
     /**
      * Subtract a specified number of microseconds from the time.
      * Note: Only millisecond accuracy is guaranteed.
      * @param  microsecs    Number of microseconds to subtract.
      */
-    void SubtractMicroSecs(double microsecs)
-    { SetMicroSecs(MicroSecs() - microsecs); }
+    void SubtractMicroSecs(double microsecs) { SetMicroSecs(MicroSecs() - microsecs); }
 
     /**
      * Subtract a specified number of milliseconds from the time.
      * @param  millisecs    Number of milliseconds to subtract.
      */
-    void SubtractMilliSecs(double millisecs)
-    { SetMilliSecs(MilliSecs() - millisecs); }
+    void SubtractMilliSecs(double millisecs) { SetMilliSecs(MilliSecs() - millisecs); }
 
     /**
      * Return the time in microseconds.
@@ -133,7 +127,6 @@ public:
      virtual long int GetMicroSeconds() const = 0;
 };
 
-
 #include <vespa/fastos/unix_time.h>
-typedef FastOS_UNIX_Time FASTOS_PREFIX(Time);
+using FastOS_Time = FastOS_UNIX_Time;
 
diff --git a/fastos/src/vespa/fastos/unix_process.h b/fastos/src/vespa/fastos/unix_process.h
index d3b3f0f3d96..7438b96d434 100644
--- a/fastos/src/vespa/fastos/unix_process.h
+++ b/fastos/src/vespa/fastos/unix_process.h
@@ -13,7 +13,7 @@
 #include <string>
 #include <memory>
 
-class FastOS_ThreadPool;
+class FastOS_BoolCond;
 class FastOS_UNIX_RealProcess;
 
 #include <vespa/fastos/ringbuffer.h>
diff --git a/fastos/src/vespa/fastos/unix_time.cpp b/fastos/src/vespa/fastos/unix_time.cpp
index 202ffe6d82a..6b6a8e58ffb 100644
--- a/fastos/src/vespa/fastos/unix_time.cpp
+++ b/fastos/src/vespa/fastos/unix_time.cpp
@@ -1,6 +1,7 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
-#include <vespa/fastos/time.h>
+#include "time.h"
+#include "types.h"
 
 double
 FastOS_UNIX_Time::MicroSecs() const
@@ -96,3 +97,8 @@ FastOS_UNIX_Time::SetSecs(double secs)
         _time.tv_usec = - static_cast<int>((- secs - (-_time.tv_sec)) * 1000000);
     }
 }
+
+void FastOS_UNIX_Time::SetNow() {
+    gettimeofday(&_time, NULL);
+}
+
diff --git a/fastos/src/vespa/fastos/unix_time.h b/fastos/src/vespa/fastos/unix_time.h
index dd6122bc463..5ae4916418a 100644
--- a/fastos/src/vespa/fastos/unix_time.h
+++ b/fastos/src/vespa/fastos/unix_time.h
@@ -89,9 +89,8 @@ public:
     void SetMilliSecs(double millisecs) override;
     void SetSecs(double secs) override;
 
-    void SetNow() override { gettimeofday(&_time, NULL); }
+    void SetNow() override;
 
     long int GetSeconds() const override { return _time.tv_sec; }
     long int GetMicroSeconds() const override { return _time.tv_usec; }
 };
-
diff --git a/filedistribution/src/vespa/filedistribution/common/componentsdeleter.cpp b/filedistribution/src/vespa/filedistribution/common/componentsdeleter.cpp
index 3cb5f4d008f..ddef7a2d175 100644
--- a/filedistribution/src/vespa/filedistribution/common/componentsdeleter.cpp
+++ b/filedistribution/src/vespa/filedistribution/common/componentsdeleter.cpp
@@ -1,5 +1,6 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 #include "componentsdeleter.h"
+#include <cassert>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".componentsdeleter");
diff --git a/filedistribution/src/vespa/filedistribution/distributor/filedownloader.cpp b/filedistribution/src/vespa/filedistribution/distributor/filedownloader.cpp
index cbd93eb2e57..14cb8844d8b 100644
--- a/filedistribution/src/vespa/filedistribution/distributor/filedownloader.cpp
+++ b/filedistribution/src/vespa/filedistribution/distributor/filedownloader.cpp
@@ -2,6 +2,7 @@
 #include "filedownloader.h"
 #include "hostname.h"
 #include <vespa/filedistribution/model/zkfacade.h>
+#include <vespa/vespalib/util/stringfmt.h>
 
 #include <boost/filesystem.hpp>
 #include <boost/filesystem/fstream.hpp>
diff --git a/filedistribution/src/vespa/filedistribution/distributor/hostname.cpp b/filedistribution/src/vespa/filedistribution/distributor/hostname.cpp
index 7b33632df81..f307ae1fca3 100644
--- a/filedistribution/src/vespa/filedistribution/distributor/hostname.cpp
+++ b/filedistribution/src/vespa/filedistribution/distributor/hostname.cpp
@@ -1,6 +1,7 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-#include "hostname.h"
 
+#include "hostname.h"
+#include <vespa/vespalib/util/stringfmt.h>
 #include <vespa/vespalib/net/socket_address.h>
 #include <vespa/log/log.h>
 LOG_SETUP(".hostname");
diff --git a/filedistribution/src/vespa/filedistribution/model/filedistributionmodelimpl.cpp b/filedistribution/src/vespa/filedistribution/model/filedistributionmodelimpl.cpp
index 3b4d043a30b..15ff198b8d5 100644
--- a/filedistribution/src/vespa/filedistribution/model/filedistributionmodelimpl.cpp
+++ b/filedistribution/src/vespa/filedistribution/model/filedistributionmodelimpl.cpp
@@ -4,6 +4,7 @@
 #include "zkfiledbmodel.h"
 #include "deployedfilestodownload.h"
 #include "filedistributionmodelimpl.h"
+#include <vespa/vespalib/util/stringfmt.h>
 #include <boost/filesystem.hpp>
 #include <zookeeper/zookeeper.h>
 
diff --git a/filedistribution/src/vespa/filedistribution/model/zkfacade.cpp b/filedistribution/src/vespa/filedistribution/model/zkfacade.cpp
index 2400eae9eb7..28e0e82f79a 100644
--- a/filedistribution/src/vespa/filedistribution/model/zkfacade.cpp
+++ b/filedistribution/src/vespa/filedistribution/model/zkfacade.cpp
@@ -2,15 +2,15 @@
 
 #include "zkfacade.h"
 #include <vespa/vespalib/net/socket_address.h>
-#include <sstream>
-#include <thread>
-#include <boost/function_output_iterator.hpp>
-
-#include <zookeeper/zookeeper.h>
 #include <vespa/filedistribution/common/logfwd.h>
 #include <vespa/defaults.h>
 #include <vespa/vespalib/util/sync.h>
 #include <vespa/vespalib/text/stringtokenizer.h>
+#include <vespa/vespalib/util/stringfmt.h>
+#include <zookeeper/zookeeper.h>
+#include <sstream>
+#include <thread>
+#include <boost/function_output_iterator.hpp>
 
 typedef std::unique_lock<std::mutex> UniqueLock;
 
diff --git a/filedistribution/src/vespa/filedistribution/model/zkfiledbmodel.cpp b/filedistribution/src/vespa/filedistribution/model/zkfiledbmodel.cpp
index 5198b6bbeb7..4030b777afa 100644
--- a/filedistribution/src/vespa/filedistribution/model/zkfiledbmodel.cpp
+++ b/filedistribution/src/vespa/filedistribution/model/zkfiledbmodel.cpp
@@ -5,6 +5,7 @@
 #include "zkfiledbmodel.h"
 #include "deployedfilestodownload.h"
 #include <vespa/filedistribution/common/logfwd.h>
+#include <vespa/vespalib/util/stringfmt.h>
 #include <sys/file.h>
 #include <ostream>
 #include <algorithm>
diff --git a/fnet/src/examples/frt/rpc/.gitignore b/fnet/src/examples/frt/rpc/.gitignore
index 91218a0868b..907b6c8a249 100644
--- a/fnet/src/examples/frt/rpc/.gitignore
+++ b/fnet/src/examples/frt/rpc/.gitignore
@@ -5,12 +5,12 @@ echo_server
 rpc_callback_client
 rpc_callback_server
 rpc_client
-rpc_info
-rpc_invoke
-rpc_proxy
 rpc_server
 fnet_echo_client_app
 fnet_rpc_callback_client_app
 fnet_rpc_callback_server_app
 fnet_rpc_client_app
 fnet_rpc_server_app
+vespa-rpc-info
+vespa-rpc-invoke
+vespa-rpc-proxy
diff --git a/fnet/src/examples/frt/rpc/CMakeLists.txt b/fnet/src/examples/frt/rpc/CMakeLists.txt
index aae76bdcd4f..0d71ba4f91e 100644
--- a/fnet/src/examples/frt/rpc/CMakeLists.txt
+++ b/fnet/src/examples/frt/rpc/CMakeLists.txt
@@ -2,28 +2,25 @@
 vespa_add_executable(fnet_rpc_server_app
     SOURCES
     rpc_server.cpp
-    INSTALL bin
     DEPENDS
     fnet
 )
 vespa_add_executable(fnet_rpc_client_app
     SOURCES
     rpc_client.cpp
-    INSTALL bin
     DEPENDS
     fnet
 )
 vespa_add_executable(fnet_echo_client_app
     SOURCES
     echo_client.cpp
-    INSTALL bin
     DEPENDS
     fnet
 )
 vespa_add_executable(fnet_rpc_info_app
     SOURCES
     rpc_info.cpp
-    OUTPUT_NAME rpc_info
+    OUTPUT_NAME vespa-rpc-info
     INSTALL bin
     DEPENDS
     fnet
@@ -31,7 +28,7 @@ vespa_add_executable(fnet_rpc_info_app
 vespa_add_executable(fnet_rpc_proxy_app
     SOURCES
     rpc_proxy.cpp
-    OUTPUT_NAME rpc_proxy
+    OUTPUT_NAME vespa-rpc-proxy
     INSTALL bin
     DEPENDS
     fnet
@@ -39,21 +36,19 @@ vespa_add_executable(fnet_rpc_proxy_app
 vespa_add_executable(fnet_rpc_callback_server_app
     SOURCES
     rpc_callback_server.cpp
-    INSTALL bin
     DEPENDS
     fnet
 )
 vespa_add_executable(fnet_rpc_callback_client_app
     SOURCES
     rpc_callback_client.cpp
-    INSTALL bin
     DEPENDS
     fnet
 )
 vespa_add_executable(fnet_rpc_invoke_app
     SOURCES
     rpc_invoke.cpp
-    OUTPUT_NAME rpc_invoke
+    OUTPUT_NAME vespa-rpc-invoke
     INSTALL bin
     DEPENDS
     fnet
diff --git a/fnet/src/examples/ping/CMakeLists.txt b/fnet/src/examples/ping/CMakeLists.txt
index 7354ec9cf48..ff635799848 100644
--- a/fnet/src/examples/ping/CMakeLists.txt
+++ b/fnet/src/examples/ping/CMakeLists.txt
@@ -3,7 +3,6 @@ vespa_add_executable(fnet_pingserver_app
     SOURCES
     packets.cpp
     pingserver.cpp
-    INSTALL bin
     DEPENDS
     fnet
 )
@@ -11,7 +10,6 @@ vespa_add_executable(fnet_pingclient_app
     SOURCES
     packets.cpp
     pingclient.cpp
-    INSTALL bin
     DEPENDS
     fnet
 )
diff --git a/fnet/src/examples/proxy/CMakeLists.txt b/fnet/src/examples/proxy/CMakeLists.txt
index bc5dce755a7..45710fc0779 100644
--- a/fnet/src/examples/proxy/CMakeLists.txt
+++ b/fnet/src/examples/proxy/CMakeLists.txt
@@ -2,7 +2,6 @@
 vespa_add_executable(fnet_proxy_app
     SOURCES
     proxy.cpp
-    INSTALL bin
     DEPENDS
     fnet
 )
diff --git a/fnet/src/examples/timeout/CMakeLists.txt b/fnet/src/examples/timeout/CMakeLists.txt
index 5b9514ab0c8..2f4a710c1f2 100644
--- a/fnet/src/examples/timeout/CMakeLists.txt
+++ b/fnet/src/examples/timeout/CMakeLists.txt
@@ -2,7 +2,6 @@
 vespa_add_executable(fnet_timeout_app
     SOURCES
     timeout.cpp
-    INSTALL bin
     DEPENDS
     fnet
 )
diff --git a/fnet/src/tests/sync_execute/sync_execute.cpp b/fnet/src/tests/sync_execute/sync_execute.cpp
index ca77ab6f73b..8b0edb4476f 100644
--- a/fnet/src/tests/sync_execute/sync_execute.cpp
+++ b/fnet/src/tests/sync_execute/sync_execute.cpp
@@ -2,6 +2,7 @@
 #include <vespa/vespalib/testkit/test_kit.h>
 #include <vespa/fnet/transport.h>
 #include <vespa/fnet/iexecutable.h>
+#include <vespa/fastos/thread.h>
 
 struct DoIt : public FNET_IExecutable {
     vespalib::Gate gate;
diff --git a/fnet/src/vespa/fnet/scheduler.cpp b/fnet/src/vespa/fnet/scheduler.cpp
index 71f02754a92..dd7a3246090 100644
--- a/fnet/src/vespa/fnet/scheduler.cpp
+++ b/fnet/src/vespa/fnet/scheduler.cpp
@@ -2,6 +2,7 @@
 
 #include "scheduler.h"
 #include "task.h"
+#include <sstream>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".fnet.scheduler");
diff --git a/fsa/src/apps/fsadump/.gitignore b/fsa/src/apps/fsadump/.gitignore
index 36c86d6022c..9c0cf4cb39b 100644
--- a/fsa/src/apps/fsadump/.gitignore
+++ b/fsa/src/apps/fsadump/.gitignore
@@ -1 +1 @@
-fsadump
+vespa-fsadump
diff --git a/fsa/src/apps/fsadump/CMakeLists.txt b/fsa/src/apps/fsadump/CMakeLists.txt
index 069bdfb379b..cd9113be3dd 100644
--- a/fsa/src/apps/fsadump/CMakeLists.txt
+++ b/fsa/src/apps/fsadump/CMakeLists.txt
@@ -2,7 +2,7 @@
 vespa_add_executable(fsa_fsadump_app
     SOURCES
     fsadump.cpp
-    OUTPUT_NAME fsadump
+    OUTPUT_NAME vespa-fsadump
     INSTALL bin
     DEPENDS
     fsa
diff --git a/fsa/src/apps/fsainfo/.gitignore b/fsa/src/apps/fsainfo/.gitignore
index fc50ebfe566..c515ae6d54c 100644
--- a/fsa/src/apps/fsainfo/.gitignore
+++ b/fsa/src/apps/fsainfo/.gitignore
@@ -1 +1 @@
-fsainfo
+vespa-fsainfo
diff --git a/fsa/src/apps/fsainfo/CMakeLists.txt b/fsa/src/apps/fsainfo/CMakeLists.txt
index c16332ed20b..30d9c173d7f 100644
--- a/fsa/src/apps/fsainfo/CMakeLists.txt
+++ b/fsa/src/apps/fsainfo/CMakeLists.txt
@@ -2,7 +2,7 @@
 vespa_add_executable(fsa_fsainfo_app
     SOURCES
     fsainfo.cpp
-    OUTPUT_NAME fsainfo
+    OUTPUT_NAME vespa-fsainfo
     INSTALL bin
     DEPENDS
     fsa
diff --git a/fsa/src/apps/makefsa/.gitignore b/fsa/src/apps/makefsa/.gitignore
index 1ea7393bec3..2ae23b715f0 100644
--- a/fsa/src/apps/makefsa/.gitignore
+++ b/fsa/src/apps/makefsa/.gitignore
@@ -1 +1 @@
-makefsa
+vespa-makefsa
diff --git a/fsa/src/apps/makefsa/CMakeLists.txt b/fsa/src/apps/makefsa/CMakeLists.txt
index 80002338479..60231e01591 100644
--- a/fsa/src/apps/makefsa/CMakeLists.txt
+++ b/fsa/src/apps/makefsa/CMakeLists.txt
@@ -2,7 +2,7 @@
 vespa_add_executable(fsa_makefsa_app
     SOURCES
     makefsa.cpp
-    OUTPUT_NAME makefsa
+    OUTPUT_NAME vespa-makefsa
     INSTALL bin
     DEPENDS
     fsa
diff --git a/install_java.cmake b/install_java.cmake
index 361a07dad5d..408582b387e 100644
--- a/install_java.cmake
+++ b/install_java.cmake
@@ -46,6 +46,7 @@ install_fat_java_artifact(simplemetrics)
 install_fat_java_artifact(standalone-container)
 install_fat_java_artifact(vespa-http-client)
 install_fat_java_artifact(vespaclient-container-plugin)
+install_fat_java_artifact(vespaclient-java)
 install_fat_java_artifact(zkfacade)
 
 vespa_install_script(application-preprocessor/src/main/sh/vespa-preprocess-application bin)
@@ -57,6 +58,15 @@ vespa_install_script(config-model/src/main/perl/expand-config.pl bin)
 vespa_install_script(config-model/src/main/perl/vespa-replicate-log-stream bin)
 vespa_install_script(config-model/src/main/sh/validate-application bin)
 vespa_install_script(container-disc/src/main/sh/vespa-start-container-daemon.sh vespa-start-container-daemon bin)
+vespa_install_script(vespaclient-java/src/main/sh/vds-document-statistics.sh vds-document-statistics bin)
+vespa_install_script(vespaclient-java/src/main/sh/vdsstat.sh vdsstat bin)
+vespa_install_script(vespaclient-java/src/main/sh/vespa-query-profile-dump-tool.sh vespa-query-profile-dump-tool bin)
+vespa_install_script(vespaclient-java/src/main/sh/vespa-summary-benchmark.sh vespa-summary-benchmark bin)
+vespa_install_script(vespaclient-java/src/main/sh/vespadestination.sh vespadestination bin)
+vespa_install_script(vespaclient-java/src/main/sh/vespafeeder.sh vespafeeder bin)
+vespa_install_script(vespaclient-java/src/main/sh/vespaget.sh vespaget bin)
+vespa_install_script(vespaclient-java/src/main/sh/vespavisit.sh vespavisit bin)
+vespa_install_script(vespaclient-java/src/main/sh/vespavisittarget.sh vespavisittarget bin)
 
 vespa_install_script(logserver/bin/logserver-start.sh logserver-start bin)
 
diff --git a/juniper/src/vespa/juniper/Matcher.cpp b/juniper/src/vespa/juniper/Matcher.cpp
index 2eeac3417af..76460323f9b 100644
--- a/juniper/src/vespa/juniper/Matcher.cpp
+++ b/juniper/src/vespa/juniper/Matcher.cpp
@@ -6,14 +6,10 @@
 #include "juniperdebug.h"
 #include "sumdesc.h"
 #include "Matcher.h"
-#include "foreach_utils.h"
-#include "SummaryConfig.h"
-#include "querynode.h"
-#include "mcand.h"
-#include "matchobject.h"
 #include "result.h"
 #include "juniperparams.h"
 #include "config.h"
+#include <sstream>
 #include <vespa/log/log.h>
 LOG_SETUP(".juniper.matcher");
 
diff --git a/juniper/src/vespa/juniper/querynode.cpp b/juniper/src/vespa/juniper/querynode.cpp
index 8718a76bf61..47bd5146432 100644
--- a/juniper/src/vespa/juniper/querynode.cpp
+++ b/juniper/src/vespa/juniper/querynode.cpp
@@ -4,6 +4,7 @@
 #include "queryvisitor.h"
 #include "juniperdebug.h"
 #include <vespa/vespalib/util/stringfmt.h>
+#include <cassert>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".juniper.querynode");
@@ -145,12 +146,10 @@ void QueryNode::ComputeThreshold()
     else if (!(_options & X_AND))
         no_threshold = true;
 
-    for (int i = 0; i < _nchild; i++)
-    {
+    for (int i = 0; i < _nchild; i++) {
         QueryExpr* qe = _children[i];
         qe->ComputeThreshold();
-        if (!no_threshold)
-        {
+        if (!no_threshold) {
             int w = qe->_weight;
             if (_options | X_AND) th += w;
             else
@@ -232,8 +231,7 @@ bool QueryTerm::Complex()
 
 bool QueryNode::Complex()
 {
-    for (int i = 0; i < _nchild; i++)
-    {
+    for (int i = 0; i < _nchild; i++) {
         if (_children[i]->_arity > 1) return true;
     }
     return false;
@@ -243,8 +241,7 @@ bool QueryNode::Complex()
 int QueryNode::MaxArity()
 {
     int max_arity = _arity;
-    for (int i = 0; i < _nchild; i++)
-    {
+    for (int i = 0; i < _nchild; i++) {
         int ma = _children[i]->MaxArity();
         if (ma > max_arity) max_arity = ma;
     }
@@ -273,8 +270,7 @@ void SimplifyStack(QueryExpr*& orig_stack)
 
     int compact = 0;
     int i;
-    if (!node->Complete())
-    {
+    if (!node->Complete()) {
         LOG(warning, "juniper: query stack incomplete, got arity %d, expected %d",
             node->_nchild, node->_arity);
         delete node;
@@ -282,10 +278,8 @@ void SimplifyStack(QueryExpr*& orig_stack)
         return;
     }
 
-    for (i = 0; i < node->_arity; i++)
-    {
-        if (i > 0 && (node->_options & X_ONLY_1))
-        {
+    for (i = 0; i < node->_arity; i++) {
+        if (i > 0 && (node->_options & X_ONLY_1)) {
             // Get rid of children # >2 for RANK/ANDNOT
             delete node->_children[i];
             node->_children[i] = NULL;
@@ -296,15 +290,11 @@ void SimplifyStack(QueryExpr*& orig_stack)
         if (node->_children[i] == NULL)
             compact++;
     }
-    if (compact > 0)
-    {
+    if (compact > 0) {
         node->_nchild = 0;
-        for (i = 0; i < node->_arity; i++)
-        {
-            if (node->_children[i])
-            {
-                if (i > node->_nchild)
-                {
+        for (i = 0; i < node->_arity; i++) {
+            if (node->_children[i]) {
+                if (i > node->_nchild) {
                     // shift remaining nodes down - remember to update _childno for each node..
                     node->_children[node->_nchild] = node->_children[i];
                     node->_children[i]->_childno = node->_nchild;
@@ -316,11 +306,9 @@ void SimplifyStack(QueryExpr*& orig_stack)
         node->_arity = node->_nchild;
     }
 
-    if (node->_arity <= 1)
-    {
+    if (node->_arity <= 1) {
         QueryExpr* ret = NULL;
-        if (node->_arity == 1)
-        {
+        if (node->_arity == 1) {
             ret = node->_children[0];
             node->_children[0] = NULL;
             ret->_parent = node->_parent;
diff --git a/juniper/src/vespa/juniper/queryparser.cpp b/juniper/src/vespa/juniper/queryparser.cpp
index 2355e41d005..4acc70dc8ab 100644
--- a/juniper/src/vespa/juniper/queryparser.cpp
+++ b/juniper/src/vespa/juniper/queryparser.cpp
@@ -5,6 +5,7 @@
 #include "queryparser.h"
 #include "juniperdebug.h"
 #include <vector>
+#include <cassert>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".juniper.queryparser");
diff --git a/juniper/src/vespa/juniper/rpinterface.cpp b/juniper/src/vespa/juniper/rpinterface.cpp
index 9f10972a448..890481b4b94 100644
--- a/juniper/src/vespa/juniper/rpinterface.cpp
+++ b/juniper/src/vespa/juniper/rpinterface.cpp
@@ -1,20 +1,15 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "rpinterface.h"
-#include <string>
-#include <vector>
 #include "juniperparams.h"
 #include "foreach_utils.h"
-#include "juniperdebug.h"
-#include "SummaryConfig.h"
 #include "queryvisitor.h"
-#include "querynode.h"
 #include "queryhandle.h"
 #include "propreader.h"
 #include "result.h"
 #include "config.h"
-#include "querymodifier.h"
-#include <vespa/fastlib/text/normwordfolder.h>
+#include <vector>
+#include <cassert>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".juniper.rpinterface");
diff --git a/logd/src/logd/forward.cpp b/logd/src/logd/forward.cpp
index 6e980034c9b..560ccaf1063 100644
--- a/logd/src/logd/forward.cpp
+++ b/logd/src/logd/forward.cpp
@@ -3,10 +3,10 @@
 #include "forward.h"
 #include "errhandle.h"
 #include <vespa/vespalib/component/vtag.h>
+#include <unistd.h>
 
 #include <vespa/log/log.h>
 LOG_SETUP("");
-LOG_RCSID("$Id$");
 
 namespace logdemon {
 
diff --git a/logd/src/logd/service.h b/logd/src/logd/service.h
index 700ab85fe27..3ff2e7fab92 100644
--- a/logd/src/logd/service.h
+++ b/logd/src/logd/service.h
@@ -1,9 +1,10 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 #pragma once
-#include <assert.h>
+
 #include <logd/config-logd.h>
 #include <vespa/vespalib/util/hashmap.h>
 #include <vespa/log/log.h>
+#include <cassert>
 
 namespace logdemon {
 
diff --git a/memfilepersistence/src/tests/device/devicemanagertest.cpp b/memfilepersistence/src/tests/device/devicemanagertest.cpp
index f5e16032361..50a4e8ef56c 100644
--- a/memfilepersistence/src/tests/device/devicemanagertest.cpp
+++ b/memfilepersistence/src/tests/device/devicemanagertest.cpp
@@ -3,6 +3,7 @@
 #include <vespa/memfilepersistence/device/devicemanager.h>
 #include <vespa/vdstestlib/cppunit/macros.h>
 #include <vespa/vespalib/util/exception.h>
+#include <vespa/vespalib/util/stringfmt.h>
 #include <vespa/storageframework/defaultimplementation/clock/fakeclock.h>
 
 namespace storage {
diff --git a/memfilepersistence/src/vespa/memfilepersistence/common/options.cpp b/memfilepersistence/src/vespa/memfilepersistence/common/options.cpp
index 651543219ec..63e0f35c95c 100644
--- a/memfilepersistence/src/vespa/memfilepersistence/common/options.cpp
+++ b/memfilepersistence/src/vespa/memfilepersistence/common/options.cpp
@@ -1,15 +1,13 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "options.h"
-#include <vespa/config-stor-memfilepersistence.h>
 #include <vespa/vespalib/util/exceptions.h>
+#include <sstream>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".persistence.slotfile.options");
 
-namespace storage {
-
-namespace memfile {
+namespace storage::memfile {
 
 Options::Options(const vespa::config::storage::StorMemfilepersistenceConfig& newConfig,
                  const vespa::config::content::PersistenceConfig& newPersistenceConfig)
@@ -179,5 +177,3 @@ void Options::print(std::ostream& out, bool verbose,
 }
 
 }
-
-}
diff --git a/memfilepersistence/src/vespa/memfilepersistence/common/types.cpp b/memfilepersistence/src/vespa/memfilepersistence/common/types.cpp
index f02423cb194..2bc85fbc422 100644
--- a/memfilepersistence/src/vespa/memfilepersistence/common/types.cpp
+++ b/memfilepersistence/src/vespa/memfilepersistence/common/types.cpp
@@ -1,11 +1,11 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "types.h"
-#include <sstream>
 #include <vespa/vespalib/util/exceptions.h>
+#include <cassert>
+#include <sstream>
 
-namespace storage {
-namespace memfile {
+namespace storage::memfile {
 
 const framework::MicroSecTime Types::MAX_TIMESTAMP(framework::MicroSecTime::max());
 const framework::MicroSecTime Types::UNSET_TIMESTAMP(0);
@@ -34,5 +34,17 @@ operator<<(std::ostream& os, const DataLocation& loc)
     return os;
 }
 
-} // memfile
-} // storage
+const char*
+Types::getMemFileFlagName(MemFileFlag flag) {
+    switch (flag) {
+        case FILE_EXIST: return "FILE_EXIST";
+        case HEADER_BLOCK_READ: return "HEADER_BLOCK_READ";
+        case BODY_BLOCK_READ: return "BODY_BLOCK_READ";
+        case BUCKET_INFO_OUTDATED: return "BUCKET_INFO_OUTDATED";
+        case SLOTS_ALTERED: return "SLOTS_ALTERED";
+        case LEGAL_MEMFILE_FLAGS: assert(false); // Not a single flag
+        default: return "INVALID";
+    }
+}
+
+}
diff --git a/memfilepersistence/src/vespa/memfilepersistence/common/types.h b/memfilepersistence/src/vespa/memfilepersistence/common/types.h
index 7758e6091ac..c1f29571037 100644
--- a/memfilepersistence/src/vespa/memfilepersistence/common/types.h
+++ b/memfilepersistence/src/vespa/memfilepersistence/common/types.h
@@ -173,21 +173,9 @@ struct Types {
         }
     }
 
-    static const char* getMemFileFlagName(MemFileFlag flag) {
-        switch (flag) {
-            case FILE_EXIST: return "FILE_EXIST";
-            case HEADER_BLOCK_READ: return "HEADER_BLOCK_READ";
-            case BODY_BLOCK_READ: return "BODY_BLOCK_READ";
-            case BUCKET_INFO_OUTDATED: return "BUCKET_INFO_OUTDATED";
-            case SLOTS_ALTERED: return "SLOTS_ALTERED";
-            case LEGAL_MEMFILE_FLAGS: assert(false); // Not a single flag
-            default: return "INVALID";
-        }
-    }
-
-    static void verifyLegalFlags(uint32_t flags, uint32_t legal,
-                                 const char* operation);
+    static const char* getMemFileFlagName(MemFileFlag flag);
 
+    static void verifyLegalFlags(uint32_t flags, uint32_t legal, const char* operation);
 protected:
     ~Types() {} // Noone should refer to objects as Types objects
 };
diff --git a/memfilepersistence/src/vespa/memfilepersistence/device/devicemanager.cpp b/memfilepersistence/src/vespa/memfilepersistence/device/devicemanager.cpp
index 2ae27f06671..ba249df2b73 100644
--- a/memfilepersistence/src/vespa/memfilepersistence/device/devicemanager.cpp
+++ b/memfilepersistence/src/vespa/memfilepersistence/device/devicemanager.cpp
@@ -2,6 +2,7 @@
 
 #include "devicemanager.h"
 #include <vespa/vespalib/util/exceptions.h>
+#include <vespa/vespalib/util/xmlstream.h>
 
 namespace storage::memfile {
 
diff --git a/memfilepersistence/src/vespa/memfilepersistence/device/devicemapper.cpp b/memfilepersistence/src/vespa/memfilepersistence/device/devicemapper.cpp
index 8ddf215d616..fdecb38ccf0 100644
--- a/memfilepersistence/src/vespa/memfilepersistence/device/devicemapper.cpp
+++ b/memfilepersistence/src/vespa/memfilepersistence/device/devicemapper.cpp
@@ -5,13 +5,12 @@
 #include <vespa/vespalib/util/exceptions.h>
 #include <fstream>
 #include <sstream>
+#include <sys/stat.h>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".persistence.devicemapper");
 
-namespace storage {
-
-namespace memfile {
+namespace storage::memfile {
 
 namespace {
     uint64_t getDevice(const std::string& path) {
@@ -95,5 +94,3 @@ AdvancedDeviceMapper::getPartitionId(const std::string& fileOnFS) const
 }
 
 }
-
-} // storage
diff --git a/memfilepersistence/src/vespa/memfilepersistence/device/mountpointlist.cpp b/memfilepersistence/src/vespa/memfilepersistence/device/mountpointlist.cpp
index 3ff816c55eb..614d399ca22 100644
--- a/memfilepersistence/src/vespa/memfilepersistence/device/mountpointlist.cpp
+++ b/memfilepersistence/src/vespa/memfilepersistence/device/mountpointlist.cpp
@@ -9,6 +9,7 @@
 #include <vespa/vespalib/util/guard.h>
 #include <vespa/vespalib/text/stringtokenizer.h>
 #include <fstream>
+#include <sstream>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".persistence.mountpointlist");
diff --git a/memfilepersistence/src/vespa/memfilepersistence/device/partition.cpp b/memfilepersistence/src/vespa/memfilepersistence/device/partition.cpp
index 3bd855553c7..9ff402a89b5 100644
--- a/memfilepersistence/src/vespa/memfilepersistence/device/partition.cpp
+++ b/memfilepersistence/src/vespa/memfilepersistence/device/partition.cpp
@@ -3,13 +3,12 @@
 #include "partition.h"
 #include "devicemanager.h"
 #include <vespa/vespalib/util/exceptions.h>
+#include <sstream>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".persistence.device.partition");
 
-namespace storage {
-
-namespace memfile {
+namespace storage::memfile {
 
 Partition::Partition(DeviceManager& manager,
                      uint64_t id,
@@ -59,6 +58,4 @@ Partition::print(std::ostream& out, bool verbose,
     Device::print(out, verbose, indent);
 }
 
-} // memfile
-
-} // storage
+}
diff --git a/memfilepersistence/src/vespa/memfilepersistence/device/partitionmonitor.cpp b/memfilepersistence/src/vespa/memfilepersistence/device/partitionmonitor.cpp
index b5b94d29336..ebc33d40f42 100644
--- a/memfilepersistence/src/vespa/memfilepersistence/device/partitionmonitor.cpp
+++ b/memfilepersistence/src/vespa/memfilepersistence/device/partitionmonitor.cpp
@@ -3,13 +3,13 @@
 #include "partitionmonitor.h"
 #include <vespa/vespalib/util/exceptions.h>
 #include <vespa/vespalib/stllike/asciistream.h>
+#include <vespa/vespalib/util/xmlstream.h>
+#include <ostream>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".persistence.device.partition.monitor");
 
-namespace storage {
-
-namespace memfile {
+namespace storage::memfile {
 
 namespace {
 
@@ -387,5 +387,3 @@ PartitionMonitor::overrideRealStat(uint32_t blockSize, uint32_t totalBlocks,
 }
 
 }
-
-} // storage
diff --git a/memfilepersistence/src/vespa/memfilepersistence/init/filescanner.cpp b/memfilepersistence/src/vespa/memfilepersistence/init/filescanner.cpp
index 3a83cee392a..e5466634283 100644
--- a/memfilepersistence/src/vespa/memfilepersistence/init/filescanner.cpp
+++ b/memfilepersistence/src/vespa/memfilepersistence/init/filescanner.cpp
@@ -5,11 +5,10 @@
 #include <vespa/vespalib/util/exceptions.h>
 #include <iomanip>
 
-#include <vespa/log/log.h>
+#include <vespa/log/bufferedlogger.h>
 LOG_SETUP(".persistence.memfile.filescanner");
 
-namespace storage {
-namespace memfile {
+namespace storage::memfile {
 
 FileScanner::Metrics::Metrics(framework::Clock& clock)
     : metrics::MetricSet("dbinit.filescan", "",
@@ -238,5 +237,4 @@ FileScanner::handleBadLocation(const document::BucketId& bucket,
     return true;
 }
 
-} // memfile
-} // storage
+}
diff --git a/memfilepersistence/src/vespa/memfilepersistence/mapper/bufferedfilewriter.cpp b/memfilepersistence/src/vespa/memfilepersistence/mapper/bufferedfilewriter.cpp
index 65b92e212f5..175c156dc9d 100644
--- a/memfilepersistence/src/vespa/memfilepersistence/mapper/bufferedfilewriter.cpp
+++ b/memfilepersistence/src/vespa/memfilepersistence/mapper/bufferedfilewriter.cpp
@@ -4,8 +4,9 @@
 #include <vespa/vespalib/util/guard.h>
 #include <vespa/vespalib/io/fileutil.h>
 #include <vespa/vespalib/util/exceptions.h>
+#include <sstream>
 
-#include <vespa/log/log.h>
+#include <vespa/log/bufferedlogger.h>
 LOG_SETUP(".persistence.memfile.bufferedfilewriter");
 
 namespace storage {
diff --git a/memfilepersistence/src/vespa/memfilepersistence/mapper/memfile_v1_serializer.cpp b/memfilepersistence/src/vespa/memfilepersistence/mapper/memfile_v1_serializer.cpp
index 2de4e062b5e..950acf09b28 100644
--- a/memfilepersistence/src/vespa/memfilepersistence/mapper/memfile_v1_serializer.cpp
+++ b/memfilepersistence/src/vespa/memfilepersistence/mapper/memfile_v1_serializer.cpp
@@ -9,6 +9,7 @@
 #include <vespa/memfilepersistence/spi/memfilepersistenceprovidermetrics.h>
 #include <vespa/vespalib/stllike/asciistream.h>
 #include <vespa/storageframework/generic/clock/timer.h>
+#include <sstream>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".persistence.memfilev1");
diff --git a/memfilepersistence/src/vespa/memfilepersistence/mapper/memfile_v1_verifier.cpp b/memfilepersistence/src/vespa/memfilepersistence/mapper/memfile_v1_verifier.cpp
index 236afdf9a77..66f6cdff083 100644
--- a/memfilepersistence/src/vespa/memfilepersistence/mapper/memfile_v1_verifier.cpp
+++ b/memfilepersistence/src/vespa/memfilepersistence/mapper/memfile_v1_verifier.cpp
@@ -6,8 +6,9 @@
 #include <vespa/storageframework/generic/clock/timer.h>
 #include <vespa/vespalib/stllike/asciistream.h>
 #include <vespa/vespalib/stllike/hash_set.hpp>
-#include <vespa/log/log.h>
+#include <sstream>
 
+#include <vespa/log/log.h>
 LOG_SETUP(".persistence.memfilev1.verifier");
 
 namespace storage::memfile {
diff --git a/memfilepersistence/src/vespa/memfilepersistence/mapper/memfilemapper.cpp b/memfilepersistence/src/vespa/memfilepersistence/mapper/memfilemapper.cpp
index eaa20c66ca9..2d91e46c7b2 100644
--- a/memfilepersistence/src/vespa/memfilepersistence/mapper/memfilemapper.cpp
+++ b/memfilepersistence/src/vespa/memfilepersistence/mapper/memfilemapper.cpp
@@ -7,8 +7,9 @@
 #include <vespa/memfilepersistence/common/exceptions.h>
 #include <vespa/vdslib/distribution/distribution.h>
 #include <vespa/storageframework/generic/clock/timer.h>
+#include <sstream>
 
-#include <vespa/log/log.h>
+#include <vespa/log/bufferedlogger.h>
 LOG_SETUP(".persistence.memfile.mapper");
 
 namespace storage::memfile {
diff --git a/memfilepersistence/src/vespa/memfilepersistence/mapper/simplememfileiobuffer.cpp b/memfilepersistence/src/vespa/memfilepersistence/mapper/simplememfileiobuffer.cpp
index 1f5344d466e..ef5c865eb5a 100644
--- a/memfilepersistence/src/vespa/memfilepersistence/mapper/simplememfileiobuffer.cpp
+++ b/memfilepersistence/src/vespa/memfilepersistence/mapper/simplememfileiobuffer.cpp
@@ -2,10 +2,13 @@
 
 #include "simplememfileiobuffer.h"
 #include <vespa/document/repo/documenttyperepo.h>
+#include <vespa/document/datatype/documenttype.h>
 #include <vespa/memfilepersistence/common/environment.h>
 #include <vespa/vespalib/util/exceptions.h>
-#include <vespa/log/log.h>
+#include <vespa/document/util/bytebuffer.h>
+#include <sstream>
 
+#include <vespa/log/log.h>
 LOG_SETUP(".memfile.simpleiobuffer");
 
 namespace storage::memfile {
diff --git a/memfilepersistence/src/vespa/memfilepersistence/memfile/memfile.cpp b/memfilepersistence/src/vespa/memfilepersistence/memfile/memfile.cpp
index 6b3f72d9187..73defc043af 100644
--- a/memfilepersistence/src/vespa/memfilepersistence/memfile/memfile.cpp
+++ b/memfilepersistence/src/vespa/memfilepersistence/memfile/memfile.cpp
@@ -12,7 +12,7 @@
 #include <ext/algorithm>
 #include <iomanip>
 
-#include <vespa/log/log.h>
+#include <vespa/log/bufferedlogger.h>
 LOG_SETUP(".persistence.memfile.memfile");
 
 namespace {
diff --git a/memfilepersistence/src/vespa/memfilepersistence/memfile/shared_data_location_tracker.h b/memfilepersistence/src/vespa/memfilepersistence/memfile/shared_data_location_tracker.h
index e0b1a7b9a2a..42d3fb0b238 100644
--- a/memfilepersistence/src/vespa/memfilepersistence/memfile/shared_data_location_tracker.h
+++ b/memfilepersistence/src/vespa/memfilepersistence/memfile/shared_data_location_tracker.h
@@ -2,6 +2,7 @@
 #pragma once
 
 #include <vespa/memfilepersistence/common/types.h>
+#include <map>
 
 namespace storage {
 namespace memfile {
diff --git a/memfilepersistence/src/vespa/memfilepersistence/spi/iteratorhandler.cpp b/memfilepersistence/src/vespa/memfilepersistence/spi/iteratorhandler.cpp
index a6b5b5bd782..6cb1c523325 100644
--- a/memfilepersistence/src/vespa/memfilepersistence/spi/iteratorhandler.cpp
+++ b/memfilepersistence/src/vespa/memfilepersistence/spi/iteratorhandler.cpp
@@ -4,12 +4,12 @@
 #include "visitorslotmatcher.h"
 #include "cacheevictionguard.h"
 #include <vespa/document/select/bodyfielddetector.h>
+#include <sstream>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".persistence.memfile.handler.iterator");
 
-namespace storage {
-namespace memfile {
+namespace storage::memfile {
 
 CachePrefetchRequirements
 CachePrefetchRequirements::createFromSelection(const document::DocumentTypeRepo& repo,
@@ -435,4 +435,3 @@ IteratorHandler::iterate(spi::IteratorId id, uint64_t maxByteSize)
 }
 
 }
-}
diff --git a/memfilepersistence/src/vespa/memfilepersistence/spi/joinoperationhandler.cpp b/memfilepersistence/src/vespa/memfilepersistence/spi/joinoperationhandler.cpp
index 1f82ffb2172..0fc2aa2ece6 100644
--- a/memfilepersistence/src/vespa/memfilepersistence/spi/joinoperationhandler.cpp
+++ b/memfilepersistence/src/vespa/memfilepersistence/spi/joinoperationhandler.cpp
@@ -3,6 +3,7 @@
 #include "joinoperationhandler.h"
 #include "cacheevictionguard.h"
 #include <vespa/memfilepersistence/mapper/memfilemapper.h>
+#include <vespa/vespalib/util/stringfmt.h>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".persistence.memfile.handler.join");
diff --git a/memfilepersistence/src/vespa/memfilepersistence/spi/memfilepersistenceprovider.cpp b/memfilepersistence/src/vespa/memfilepersistence/spi/memfilepersistenceprovider.cpp
index d17dfdc77a4..9e580d4e234 100644
--- a/memfilepersistence/src/vespa/memfilepersistence/spi/memfilepersistenceprovider.cpp
+++ b/memfilepersistence/src/vespa/memfilepersistence/spi/memfilepersistenceprovider.cpp
@@ -6,6 +6,7 @@
 #include <vespa/document/update/documentupdate.h>
 #include <vespa/config/helper/configgetter.hpp>
 #include <vespa/storageframework/generic/status/htmlstatusreporter.h>
+#include <sstream>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".memfilepersistenceprovider");
diff --git a/memfilepersistence/src/vespa/memfilepersistence/spi/operationhandler.cpp b/memfilepersistence/src/vespa/memfilepersistence/spi/operationhandler.cpp
index da085ca9137..c9972dbd02b 100644
--- a/memfilepersistence/src/vespa/memfilepersistence/spi/operationhandler.cpp
+++ b/memfilepersistence/src/vespa/memfilepersistence/spi/operationhandler.cpp
@@ -3,12 +3,12 @@
 #include "operationhandler.h"
 #include <vespa/memfilepersistence/common/exceptions.h>
 #include <vespa/document/select/parser.h>
+#include <sstream>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".persistence.memfile.handler.operation");
 
-namespace storage {
-namespace memfile {
+namespace storage::memfile {
 
 OperationHandler::OperationHandler(Environment& env)
     : _env(env)
@@ -283,5 +283,4 @@ OperationHandler::parseDocumentSelection(
     return ret;
 }
 
-} // memfile
-} // storage
+}
diff --git a/memfilepersistence/src/vespa/memfilepersistence/tools/.gitignore b/memfilepersistence/src/vespa/memfilepersistence/tools/.gitignore
index c7687cb62d2..5b3727b1f5e 100644
--- a/memfilepersistence/src/vespa/memfilepersistence/tools/.gitignore
+++ b/memfilepersistence/src/vespa/memfilepersistence/tools/.gitignore
@@ -1,5 +1,4 @@
 /.depend
 /Makefile
-/dumpslotfile
-/vdsdisktool
-vdsdisktool-bin
+vespa-dump-slotfile
+vespa-vds-disktool-bin
diff --git a/memfilepersistence/src/vespa/memfilepersistence/tools/CMakeLists.txt b/memfilepersistence/src/vespa/memfilepersistence/tools/CMakeLists.txt
index cc8ad0a2320..dabec5b6a8e 100644
--- a/memfilepersistence/src/vespa/memfilepersistence/tools/CMakeLists.txt
+++ b/memfilepersistence/src/vespa/memfilepersistence/tools/CMakeLists.txt
@@ -8,7 +8,7 @@ vespa_add_library(memfilepersistence_tools OBJECT
 vespa_add_executable(memfilepersistence_dumpslotfile_app
     SOURCES
     dumpslotfileapp.cpp
-    OUTPUT_NAME dumpslotfile
+    OUTPUT_NAME vespa-dump-slotfile
     INSTALL bin
     DEPENDS
     memfilepersistence
@@ -16,7 +16,7 @@ vespa_add_executable(memfilepersistence_dumpslotfile_app
 vespa_add_executable(memfilepersistence_vdsdisktool_app
     SOURCES
     vdsdiskapp.cpp
-    OUTPUT_NAME vdsdisktool-bin
+    OUTPUT_NAME vespa-vds-disktool-bin
     INSTALL bin
     DEPENDS
     memfilepersistence
diff --git a/messagebus/src/vespa/messagebus/callstack.cpp b/messagebus/src/vespa/messagebus/callstack.cpp
index 5311a75c3c5..44ae4190698 100644
--- a/messagebus/src/vespa/messagebus/callstack.cpp
+++ b/messagebus/src/vespa/messagebus/callstack.cpp
@@ -4,6 +4,7 @@
 #include "message.h"
 #include "reply.h"
 #include "idiscardhandler.h"
+#include <cassert>
 
 namespace mbus {
 
diff --git a/messagebus/src/vespa/messagebus/testlib/slobrok.cpp b/messagebus/src/vespa/messagebus/testlib/slobrok.cpp
index 10c501db854..12f53e31764 100644
--- a/messagebus/src/vespa/messagebus/testlib/slobrok.cpp
+++ b/messagebus/src/vespa/messagebus/testlib/slobrok.cpp
@@ -4,6 +4,7 @@
 #include <vespa/slobrok/server/sbenv.h>
 #include <vespa/fnet/frt/supervisor.h>
 #include <vespa/fnet/transport.h>
+#include <vespa/vespalib/util/stringfmt.h>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".slobrok");
diff --git a/metrics/src/tests/metricmanagertest.cpp b/metrics/src/tests/metricmanagertest.cpp
index 0ba6bdfbd2f..e4a2761cd73 100644
--- a/metrics/src/tests/metricmanagertest.cpp
+++ b/metrics/src/tests/metricmanagertest.cpp
@@ -9,6 +9,7 @@
 #include <vespa/vdstestlib/cppunit/macros.h>
 #include <vespa/vespalib/stllike/asciistream.h>
 #include <vespa/vespalib/data/slime/slime.h>
+#include <vespa/vespalib/util/xmlstream.h>
 #include <vespa/log/log.h>
 
 LOG_SETUP(".test.metricmanager");
diff --git a/metrics/src/vespa/metrics/metric.cpp b/metrics/src/vespa/metrics/metric.cpp
index 57ce8e5ffc1..0c99409a8bb 100644
--- a/metrics/src/vespa/metrics/metric.cpp
+++ b/metrics/src/vespa/metrics/metric.cpp
@@ -8,6 +8,7 @@
 #include <vespa/vespalib/text/stringtokenizer.h>
 #include <vespa/vespalib/util/exceptions.h>
 #include <vespa/vespalib/stllike/asciistream.h>
+#include <vespa/vespalib/util/stringfmt.h>
 #include <iterator>
 
 namespace metrics {
diff --git a/metrics/src/vespa/metrics/metricmanager.cpp b/metrics/src/vespa/metrics/metricmanager.cpp
index 572779be855..c352f0ee5ab 100644
--- a/metrics/src/vespa/metrics/metricmanager.cpp
+++ b/metrics/src/vespa/metrics/metricmanager.cpp
@@ -10,11 +10,12 @@
 #include "xmlwriter.h"
 #include <vespa/config/print/ostreamconfigwriter.h>
 #include <vespa/vespalib/text/stringtokenizer.h>
-
+#include <vespa/vespalib/util/stringfmt.h>
 #include <vespa/vespalib/util/exceptions.h>
 #include <vespa/vespalib/stllike/asciistream.h>
-#include <vespa/log/log.h>
+#include <sstream>
 
+#include <vespa/log/bufferedlogger.h>
 LOG_SETUP(".metrics.manager");
 
 namespace metrics {
diff --git a/metrics/src/vespa/metrics/metricset.cpp b/metrics/src/vespa/metrics/metricset.cpp
index 15ee22ba1f2..c9acbaafce2 100644
--- a/metrics/src/vespa/metrics/metricset.cpp
+++ b/metrics/src/vespa/metrics/metricset.cpp
@@ -4,10 +4,10 @@
 #include "memoryconsumption.h"
 #include <vespa/vespalib/stllike/hash_map.hpp>
 #include <vespa/vespalib/util/exceptions.h>
+#include <vespa/vespalib/util/stringfmt.h>
 #include <list>
 
 #include <vespa/log/log.h>
-
 LOG_SETUP(".metrics.metricsset");
 
 namespace metrics {
diff --git a/metrics/src/vespa/metrics/printutils.cpp b/metrics/src/vespa/metrics/printutils.cpp
index 2373d2391ff..2209ebd5da7 100644
--- a/metrics/src/vespa/metrics/printutils.cpp
+++ b/metrics/src/vespa/metrics/printutils.cpp
@@ -1,6 +1,7 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "printutils.h"
+#include <vespa/vespalib/util/stringfmt.h>
 
 namespace metrics {
 namespace printutils {
diff --git a/metrics/src/vespa/metrics/summetric.hpp b/metrics/src/vespa/metrics/summetric.hpp
index 1a54067b01f..3e85aa0e215 100644
--- a/metrics/src/vespa/metrics/summetric.hpp
+++ b/metrics/src/vespa/metrics/summetric.hpp
@@ -5,6 +5,7 @@
 #include "metricset.h"
 #include "memoryconsumption.h"
 #include <vespa/vespalib/util/exceptions.h>
+#include <vespa/vespalib/util/stringfmt.h>
 #include <ostream>
 
 namespace metrics {
diff --git a/metrics/src/vespa/metrics/valuemetric.hpp b/metrics/src/vespa/metrics/valuemetric.hpp
index 65a0c2bb5b2..b09f35e3272 100644
--- a/metrics/src/vespa/metrics/valuemetric.hpp
+++ b/metrics/src/vespa/metrics/valuemetric.hpp
@@ -4,6 +4,7 @@
 #include "valuemetric.h"
 #include "memoryconsumption.h"
 #include <vespa/vespalib/util/exceptions.h>
+#include <vespa/vespalib/util/stringfmt.h>
 #include <sstream>
 
 namespace metrics {
diff --git a/metrics/src/vespa/metrics/valuemetricvalues.hpp b/metrics/src/vespa/metrics/valuemetricvalues.hpp
index d8becca3e72..e7f3e9e3239 100644
--- a/metrics/src/vespa/metrics/valuemetricvalues.hpp
+++ b/metrics/src/vespa/metrics/valuemetricvalues.hpp
@@ -3,6 +3,7 @@
 
 #include "valuemetricvalues.h"
 #include <vespa/vespalib/util/exceptions.h>
+#include <vespa/vespalib/util/stringfmt.h>
 #include <ostream>
 
 namespace metrics {
diff --git a/metrics/src/vespa/metrics/xmlwriter.cpp b/metrics/src/vespa/metrics/xmlwriter.cpp
index 66f83cdd4bf..4a998b79294 100644
--- a/metrics/src/vespa/metrics/xmlwriter.cpp
+++ b/metrics/src/vespa/metrics/xmlwriter.cpp
@@ -1,11 +1,11 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
-#include <vespa/metrics/xmlwriter.h>
-
-#include <vespa/metrics/countmetric.h>
-#include <vespa/metrics/metricset.h>
-#include <vespa/metrics/metricsnapshot.h>
-#include <vespa/metrics/valuemetric.h>
+#include "xmlwriter.h"
+#include "countmetric.h"
+#include "metricset.h"
+#include "metricsnapshot.h"
+#include "valuemetric.h"
+#include <vespa/vespalib/util/xmlstream.h>
 #include <sstream>
 
 namespace metrics {
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java
index b160b4ac364..8e67cac5b2f 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java
@@ -199,25 +199,14 @@ public class NodeAgentImpl implements NodeAgent {
         logger.info("Resume command successfully executed, new containerState is " + containerState);
     }
 
-    private void updateNodeRepoAndMarkNodeAsReady(ContainerNodeSpec nodeSpec) {
-        publishStateToNodeRepoIfChanged(
-                // Clear current Docker image and vespa version, as nothing is running on this node
-                new NodeAttributes()
-                        .withRestartGeneration(nodeSpec.wantedRestartGeneration.orElse(null))
-                        .withRebootGeneration(nodeSpec.wantedRebootGeneration.orElse(0L))
-                        .withDockerImage(new DockerImage(""))
-                        .withVespaVersion(""));
-        nodeRepository.markNodeAvailableForNewAllocation(nodeSpec.hostname);
-    }
-
     private void updateNodeRepoWithCurrentAttributes(final ContainerNodeSpec nodeSpec) {
         final NodeAttributes nodeAttributes = new NodeAttributes()
                 .withRestartGeneration(nodeSpec.wantedRestartGeneration.orElse(null))
                 // update reboot gen with wanted gen if set, we ignore reboot for Docker nodes but
                 // want the two to be equal in node repo
                 .withRebootGeneration(nodeSpec.wantedRebootGeneration.orElse(0L))
-                .withDockerImage(nodeSpec.wantedDockerImage.orElse(new DockerImage("")))
-                .withVespaVersion(nodeSpec.wantedVespaVersion.orElse(""));
+                .withDockerImage(nodeSpec.wantedDockerImage.filter(node -> containerState != ABSENT).orElse(new DockerImage("")))
+                .withVespaVersion(nodeSpec.wantedVespaVersion.filter(node -> containerState != ABSENT).orElse(""));
 
         publishStateToNodeRepoIfChanged(nodeAttributes);
     }
@@ -467,7 +456,8 @@ public class NodeAgentImpl implements NodeAgent {
                 removeContainerIfNeededUpdateContainerState(nodeSpec, container);
                 logger.info("State is " + nodeSpec.nodeState + ", will delete application storage and mark node as ready");
                 storageMaintainer.ifPresent(maintainer -> maintainer.archiveNodeData(containerName));
-                updateNodeRepoAndMarkNodeAsReady(nodeSpec);
+                updateNodeRepoWithCurrentAttributes(nodeSpec);
+                nodeRepository.markNodeAvailableForNewAllocation(hostname);
                 break;
             default:
                 throw new RuntimeException("UNKNOWN STATE " + nodeSpec.nodeState.name());
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MultiDockerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MultiDockerTest.java
index ba3fe71637d..c06febe3f17 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MultiDockerTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MultiDockerTest.java
@@ -54,10 +54,10 @@ public class MultiDockerTest {
                                                              "DeleteContainerStorage with ContainerName { name=host2 }");
 
             callOrderVerifier.assertInOrder(
-                    "updateNodeAttributes with HostName: host1.test.yahoo.com, NodeAttributes{restartGeneration=1, rebootGeneration=0, dockerImage=image1, vespaVersion=''}",
-                    "updateNodeAttributes with HostName: host2.test.yahoo.com, NodeAttributes{restartGeneration=1, rebootGeneration=0, dockerImage=image2, vespaVersion=''}",
+                    "updateNodeAttributes with HostName: host1.test.yahoo.com, NodeAttributes{restartGeneration=1, rebootGeneration=0, dockerImage=image1, vespaVersion='1.2.3'}",
+                    "updateNodeAttributes with HostName: host2.test.yahoo.com, NodeAttributes{restartGeneration=1, rebootGeneration=0, dockerImage=image2, vespaVersion='1.2.3'}",
                     "markNodeAvailableForNewAllocation with HostName: host2.test.yahoo.com",
-                    "updateNodeAttributes with HostName: host3.test.yahoo.com, NodeAttributes{restartGeneration=1, rebootGeneration=0, dockerImage=image1, vespaVersion=''}");
+                    "updateNodeAttributes with HostName: host3.test.yahoo.com, NodeAttributes{restartGeneration=1, rebootGeneration=0, dockerImage=image1, vespaVersion='1.2.3'}");
         }
     }
 
@@ -65,6 +65,7 @@ public class MultiDockerTest {
         ContainerNodeSpec containerNodeSpec = new ContainerNodeSpec.Builder()
                 .hostname(hostName)
                 .wantedDockerImage(dockerImage)
+                .wantedVespaVersion("1.2.3")
                 .nodeState(Node.State.active)
                 .nodeType("tenant")
                 .nodeFlavor("docker")
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RestartTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RestartTest.java
index d7fd753fabe..a0024fae3dc 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RestartTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RestartTest.java
@@ -32,7 +32,7 @@ public class RestartTest {
             CallOrderVerifier callOrderVerifier = dockerTester.getCallOrderVerifier();
             // Check that the container is started and NodeRepo has received the PATCH update
             callOrderVerifier.assertInOrder("createContainerCommand with DockerImage { imageId=image:1.2.3 }, HostName: host1.test.yahoo.com, ContainerName { name=host1 }",
-                                            "updateNodeAttributes with HostName: host1.test.yahoo.com, NodeAttributes{restartGeneration=1, rebootGeneration=0, dockerImage=image:1.2.3, vespaVersion=''}");
+                                            "updateNodeAttributes with HostName: host1.test.yahoo.com, NodeAttributes{restartGeneration=1, rebootGeneration=0, dockerImage=image:1.2.3, vespaVersion='1.2.3'}");
 
             wantedRestartGeneration = 2;
             currentRestartGeneration = 1;
@@ -46,8 +46,9 @@ public class RestartTest {
     private ContainerNodeSpec createContainerNodeSpec(long wantedRestartGeneration, long currentRestartGeneration) {
         return new ContainerNodeSpec.Builder()
                 .hostname("host1.test.yahoo.com")
-                .wantedDockerImage(new DockerImage("image:1.2.3"))
                 .nodeState(Node.State.active)
+                .wantedDockerImage(new DockerImage("image:1.2.3"))
+                .wantedVespaVersion("1.2.3")
                 .nodeType("tenant")
                 .nodeFlavor("docker")
                 .wantedRestartGeneration(wantedRestartGeneration)
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java
index ef49b6878b6..83a00241919 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java
@@ -316,6 +316,34 @@ public class NodeAgentImplTest {
                         .withVespaVersion(vespaVersion));
     }
 
+    @Test
+    public void reservedNodeDoesNotUpdateNodeRepoWithVersion() throws Exception {
+        final long restartGeneration = 1;
+        final long rebootGeneration = 0;
+
+        final ContainerNodeSpec nodeSpec = nodeSpecBuilder
+                .wantedDockerImage(dockerImage)
+                .nodeState(Node.State.reserved)
+                .wantedVespaVersion(vespaVersion)
+                .wantedRestartGeneration(restartGeneration)
+                .currentRestartGeneration(restartGeneration)
+                .wantedRebootGeneration(rebootGeneration)
+                .build();
+
+        NodeAgentImpl nodeAgent = makeNodeAgent(null, false);
+
+        when(nodeRepository.getContainerNodeSpec(hostName)).thenReturn(Optional.of(nodeSpec));
+
+        nodeAgent.converge();
+
+        verify(nodeRepository).updateNodeAttributes(
+                hostName, new NodeAttributes()
+                        .withRestartGeneration(restartGeneration)
+                        .withRebootGeneration(rebootGeneration)
+                        .withDockerImage(new DockerImage(""))
+                        .withVespaVersion(""));
+    }
+
     private void nodeRunningContainerIsTakenDownAndCleanedAndRecycled(Node.State nodeState, Optional<Long> wantedRestartGeneration) {
         wantedRestartGeneration.ifPresent(restartGeneration -> nodeSpecBuilder
                 .wantedRestartGeneration(restartGeneration)
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Maintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Maintainer.java
index 715aaf8680e..3df8187cda8 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Maintainer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Maintainer.java
@@ -18,7 +18,7 @@ import java.util.logging.Logger;
  */
 public abstract class Maintainer extends AbstractComponent implements Runnable {
 
-    protected static final Logger log = Logger.getLogger(Maintainer.class.getName());
+    protected final Logger log = Logger.getLogger(this.getClass().getName());
 
     private final NodeRepository nodeRepository;
     private final Duration interval;
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRetirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRetirer.java
index 880c5da457a..49184dab0af 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRetirer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRetirer.java
@@ -104,9 +104,12 @@ public class NodeRetirer extends Maintainer {
                     Flavor flavorWithMinSpareNodes = getMinAmongstKeys(numSpareNodesByFlavor, possibleReplacementFlavors);
                     long spareNodesForMinFlavor = numSpareNodesByFlavor.getOrDefault(flavorWithMinSpareNodes, 0L);
                     if (spareNodesForMinFlavor > 0) {
-                        log.info("Setting node " + retireableNode + " to wantToRetire. Policy: " +
+                        log.info("Setting node " + retireableNode + " to wantToRetire and wantToDeprovision. Policy: " +
                                 retirementPolicy.getClass().getSimpleName());
-                        Node updatedNode = retireableNode.with(retireableNode.status().withWantToRetire(true));
+                        Node updatedNode = retireableNode
+                                .with(retireableNode.status()
+                                        .withWantToRetire(true)
+                                        .withWantToDeprovision(true));
                         nodeRepository().write(updatedNode);
                         numSpareNodesByFlavor.put(flavorWithMinSpareNodes, spareNodesForMinFlavor - 1);
                         numNodesAllowedToRetire--;
@@ -156,6 +159,8 @@ public class NodeRetirer extends Maintainer {
     }
 
     /**
+     * Parks and sets wantToDeprovision for a subset of size 'limit' of nodes
+     *
      * @param nodesToPark Nodes that we want to park
      * @param limit Maximum number of nodes we want to park
      * @return True iff we were able to park all the nodes
@@ -163,7 +168,10 @@ public class NodeRetirer extends Maintainer {
     boolean limitedPark(Set<Node> nodesToPark, long limit) {
         nodesToPark.stream()
                 .limit(limit)
-                .forEach(node -> nodeRepository().park(node.hostname(), Agent.NodeRetirer, "Policy: " + retirementPolicy.getClass().getSimpleName()));
+                .forEach(node -> {
+                    nodeRepository().write(node.with(node.status().withWantToDeprovision(true)));
+                    nodeRepository().park(node.hostname(), Agent.NodeRetirer, "Policy: " + retirementPolicy.getClass().getSimpleName());
+                });
 
         return limit >= nodesToPark.size();
     }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredEarlyExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredEarlyExpirer.java
index 609755656b6..78eb0150fb5 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredEarlyExpirer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredEarlyExpirer.java
@@ -60,6 +60,10 @@ public class RetiredEarlyExpirer extends Maintainer {
                         nodesToRemove.add(node);
                     }
                 }
+                
+                if (nodesToRemove.isEmpty()) {
+                    continue;
+                }
 
                 nodeRepository().setRemovable(application, nodesToRemove);
 
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Status.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Status.java
index 7946b093b10..e8a24b85714 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Status.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Status.java
@@ -22,6 +22,7 @@ public class Status {
     private final int failCount;
     private final Optional<HardwareFailureType> hardwareFailure;
     private final boolean wantToRetire;
+    private final boolean wantToDeprovision;
     
     public enum HardwareFailureType {
         
@@ -40,7 +41,8 @@ public class Status {
                   Optional<String> stateVersion,
                   int failCount,
                   Optional<HardwareFailureType> hardwareFailure,
-                  boolean wantToRetire) {
+                  boolean wantToRetire,
+                  boolean wantToDeprovision) {
         this.reboot = generation;
         this.vespaVersion = vespaVersion;
         this.hostedVersion = hostedVersion;
@@ -48,28 +50,29 @@ public class Status {
         this.failCount = failCount;
         this.hardwareFailure = hardwareFailure;
         this.wantToRetire = wantToRetire;
+        this.wantToDeprovision = wantToDeprovision;
     }
 
     /** Returns a copy of this with the reboot generation changed */
-    public Status withReboot(Generation reboot) { return new Status(reboot, vespaVersion, hostedVersion, stateVersion, failCount, hardwareFailure, wantToRetire); }
+    public Status withReboot(Generation reboot) { return new Status(reboot, vespaVersion, hostedVersion, stateVersion, failCount, hardwareFailure, wantToRetire, wantToDeprovision); }
 
     /** Returns the reboot generation of this node */
     public Generation reboot() { return reboot; }
 
     /** Returns a copy of this with the vespa version changed */
-    public Status withVespaVersion(Version version) { return new Status(reboot, Optional.of(version), hostedVersion, stateVersion, failCount, hardwareFailure, wantToRetire); }
+    public Status withVespaVersion(Version version) { return new Status(reboot, Optional.of(version), hostedVersion, stateVersion, failCount, hardwareFailure, wantToRetire, wantToDeprovision); }
 
     /** Returns the Vespa version installed on the node, if known */
     public Optional<Version> vespaVersion() { return vespaVersion; }
 
     /** Returns a copy of this with the hosted version changed */
-    public Status withHostedVersion(Version version) { return new Status(reboot, vespaVersion, Optional.of(version), stateVersion, failCount, hardwareFailure, wantToRetire); }
+    public Status withHostedVersion(Version version) { return new Status(reboot, vespaVersion, Optional.of(version), stateVersion, failCount, hardwareFailure, wantToRetire, wantToDeprovision); }
 
     /** Returns the hosted version installed on the node, if known */
     public Optional<Version> hostedVersion() { return hostedVersion; }
 
     /** Returns a copy of this with the state version changed */
-    public Status withStateVersion(String version) { return new Status(reboot, vespaVersion, hostedVersion, Optional.of(version), failCount, hardwareFailure, wantToRetire); }
+    public Status withStateVersion(String version) { return new Status(reboot, vespaVersion, hostedVersion, Optional.of(version), failCount, hardwareFailure, wantToRetire, wantToDeprovision); }
 
     /**
      * Returns the state version the node last successfully converged with.
@@ -85,29 +88,29 @@ public class Status {
                 .filter(image -> !image.isEmpty())
                 .map(DockerImage::new)
                 .map(DockerImage::tagAsVersion);
-        return new Status(reboot, vespaVersion, hostedVersion, stateVersion, failCount, hardwareFailure, wantToRetire);
+        return new Status(reboot, vespaVersion, hostedVersion, stateVersion, failCount, hardwareFailure, wantToRetire, wantToDeprovision);
     }
 
     /** Returns the current docker image the node is running, if known. */
     public Optional<String> dockerImage() { return vespaVersion.map(DockerImage.defaultImage::withTag).map(DockerImage::toString); }
 
-    public Status withIncreasedFailCount() { return new Status(reboot, vespaVersion, hostedVersion, stateVersion, failCount + 1, hardwareFailure, wantToRetire); }
+    public Status withIncreasedFailCount() { return new Status(reboot, vespaVersion, hostedVersion, stateVersion, failCount + 1, hardwareFailure, wantToRetire, wantToDeprovision); }
 
-    public Status withDecreasedFailCount() { return new Status(reboot, vespaVersion, hostedVersion, stateVersion, failCount - 1, hardwareFailure, wantToRetire); }
+    public Status withDecreasedFailCount() { return new Status(reboot, vespaVersion, hostedVersion, stateVersion, failCount - 1, hardwareFailure, wantToRetire, wantToDeprovision); }
 
-    public Status setFailCount(Integer value) { return new Status(reboot, vespaVersion, hostedVersion, stateVersion, value, hardwareFailure, wantToRetire); }
+    public Status setFailCount(Integer value) { return new Status(reboot, vespaVersion, hostedVersion, stateVersion, value, hardwareFailure, wantToRetire, wantToDeprovision); }
 
     /** Returns how many times this node has been moved to the failed state. */
     public int failCount() { return failCount; }
 
-    public Status withHardwareFailure(Optional<HardwareFailureType> hardwareFailure) { return new Status(reboot, vespaVersion, hostedVersion, stateVersion, failCount, hardwareFailure, wantToRetire); }
+    public Status withHardwareFailure(Optional<HardwareFailureType> hardwareFailure) { return new Status(reboot, vespaVersion, hostedVersion, stateVersion, failCount, hardwareFailure, wantToRetire, wantToDeprovision); }
 
     /** Returns the type of the last hardware failure detected on this node, or empty if none */
     public Optional<HardwareFailureType> hardwareFailure() { return hardwareFailure; }
 
     /** Returns a copy of this with the want to retire flag changed */
     public Status withWantToRetire(boolean wantToRetire) {
-        return new Status(reboot, vespaVersion, hostedVersion, stateVersion, failCount, hardwareFailure, wantToRetire);
+        return new Status(reboot, vespaVersion, hostedVersion, stateVersion, failCount, hardwareFailure, wantToRetire, wantToDeprovision);
     }
 
     /**
@@ -118,7 +121,19 @@ public class Status {
         return wantToRetire;
     }
 
+    /** Returns a copy of this with the want to deprovision flag changed */
+    public Status withWantToDeprovision(boolean wantToDeprovision) {
+        return new Status(reboot, vespaVersion, hostedVersion, stateVersion, failCount, hardwareFailure, wantToRetire, wantToDeprovision);
+    }
+
+    /**
+     * Returns whether this node should be deprovisioned when possible.
+     */
+    public boolean wantToDeprovision() {
+        return wantToDeprovision;
+    }
+
     /** Returns the initial status of a newly provisioned node */
-    public static Status initial() { return new Status(Generation.inital(), Optional.empty(), Optional.empty(), Optional.empty(), 0, Optional.empty(), false); }
+    public static Status initial() { return new Status(Generation.inital(), Optional.empty(), Optional.empty(), Optional.empty(), 0, Optional.empty(), false, false); }
 
 }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java
index 8612dbd607e..582f97361b2 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java
@@ -60,6 +60,7 @@ public class NodeSerializer {
     private static final String hardwareFailureKey = "hardwareFailure";
     private static final String nodeTypeKey = "type";
     private static final String wantToRetireKey = "wantToRetire";
+    private static final String wantToDeprovisionKey = "wantToDeprovision";
 
     // Configuration fields
     private static final String flavorKey = "flavor";
@@ -112,6 +113,7 @@ public class NodeSerializer {
         object.setLong(failCountKey, node.status().failCount());
         node.status().hardwareFailure().ifPresent(failure -> object.setString(hardwareFailureKey, toString(failure)));
         object.setBool(wantToRetireKey, node.status().wantToRetire());
+        object.setBool(wantToDeprovisionKey, node.status().wantToDeprovision());
         node.allocation().ifPresent(allocation -> toSlime(allocation, object.setObject(instanceKey)));
         toSlime(node.history(), object.setArray(historyKey));
         object.setString(nodeTypeKey, toString(node.type()));
@@ -164,13 +166,16 @@ public class NodeSerializer {
     }
 
     private Status statusFromSlime(Inspector object) {
+        // TODO: Simplify after June 2017
+        boolean wantToDeprovision = object.field(wantToDeprovisionKey).valid() && object.field(wantToDeprovisionKey).asBool();
         return new Status(generationFromSlime(object, rebootGenerationKey, currentRebootGenerationKey),
                           versionFromSlime(object.field(vespaVersionKey)),
                           versionFromSlime(object.field(hostedVersionKey)),
                           optionalString(object.field(stateVersionKey)),
                           (int)object.field(failCountKey).asLong(),
                           hardwareFailureFromSlime(object.field(hardwareFailureKey)),
-                          object.field(wantToRetireKey).asBool());
+                          object.field(wantToRetireKey).asBool(),
+                          wantToDeprovision);
     }
 
     private Flavor flavorFromSlime(Inspector object) {
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java
index fd2bed7374b..b410b0c8ead 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java
@@ -87,6 +87,8 @@ public class NodePatcher {
                 return node.withAdditionalIpAddresses(asStringSet(value));
             case "wantToRetire" :
                 return node.with(node.status().withWantToRetire(asBoolean(value)));
+            case "wantToDeprovision" :
+                return node.with(node.status().withWantToDeprovision(asBoolean(value)));
             default :
                 throw new IllegalArgumentException("Could not apply field '" + name + "' on a node: No such modifiable field");
         }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java
index f88f7a663f0..00bec80b758 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java
@@ -171,6 +171,7 @@ class NodesResponse extends HttpResponse {
         object.setBool("hardwareFailure", node.status().hardwareFailure().isPresent());
         node.status().hardwareFailure().ifPresent(failure -> object.setString("hardwareFailureType", toString(failure)));
         object.setBool("wantToRetire", node.status().wantToRetire());
+        object.setBool("wantToDeprovision", node.status().wantToDeprovision());
         toSlime(node.history(), object.setArray("history"));
         ipAddressesToSlime(node.ipAddresses(), object.setArray("ipAddresses"));
         ipAddressesToSlime(node.additionalIpAddresses(), object.setArray("additionalIpAddresses"));
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRetirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRetirerTest.java
index c466cfca200..16f0acc9d55 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRetirerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRetirerTest.java
@@ -73,6 +73,7 @@ public class NodeRetirerTest {
                     .forEach(node -> {
                         Agent parkingAgent = node.history().event(History.Event.Type.parked).orElseThrow(RuntimeException::new).agent();
                         assertEquals(Agent.NodeRetirer, parkingAgent);
+                        assertTrue("Nodes parked by NodeRetirer should also have wantToDeprovision flag set", node.status().wantToDeprovision());
                         tester.nodeRepository.write(node.withIpAddresses(Collections.singleton("::2")));
                         tester.nodeRepository.setDirty(node.hostname());
                         tester.nodeRepository.setReady(node.hostname());
@@ -143,6 +144,10 @@ public class NodeRetirerTest {
             // min flavor count for both flavor clusters is now 0, so no further change is expected
             retireThenAssertSpareAndParkedCounts(new long[]{2, 40, 25, 0, 0, 1}, new long[]{6, 3, 5, 2, 1});
             retireThenAssertSpareAndParkedCounts(new long[]{2, 40, 25, 0, 0, 1}, new long[]{6, 3, 5, 2, 1});
+
+            tester.nodeRepository.getNodes(Node.State.parked)
+                    .forEach(node -> assertTrue("Nodes parked by NodeRetirer should also have wantToDeprovision flag set",
+                            node.status().wantToDeprovision()));
         }
 
         @Test
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java
index 2e64b0745c9..d2c3e60b420 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java
@@ -325,6 +325,20 @@ public class SerializationTest {
     }
 
     @Test
+    public void want_to_deprovision_defaults_to_false() {
+        String nodeData =
+                "{\n" +
+                        "   \"type\" : \"tenant\",\n" +
+                        "   \"flavor\" : \"large\",\n" +
+                        "   \"openStackId\" : \"myId\",\n" +
+                        "   \"hostname\" : \"myHostname\",\n" +
+                        "   \"ipAddresses\" : [\"127.0.0.1\"]\n" +
+                        "}";
+        Node node = nodeSerializer.fromJson(State.provisioned, Utf8.toBytes(nodeData));
+        assertFalse(node.status().wantToDeprovision());
+    }
+
+    @Test
     public void vespa_version_serialization() throws Exception {
         String nodeWithWantedVespaVersion =
                 "{\n" +
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java
index 0285138c9c2..9e46f11b6ab 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java
@@ -196,6 +196,9 @@ public class RestApiTest {
                                    Utf8.toBytes("{\"wantToRetire\": true}"), Request.Method.PATCH),
                        "{\"message\":\"Updated host4.yahoo.com\"}");
         assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com",
+                        Utf8.toBytes("{\"wantToDeprovision\": true}"), Request.Method.PATCH),
+                "{\"message\":\"Updated host4.yahoo.com\"}");
+        assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com",
                                    Utf8.toBytes("{\"currentDockerImage\": \"ignored-image-name:4443/vespa/ci:6.43.0\"}"), Request.Method.PATCH),
                        "{\"message\":\"Updated host4.yahoo.com\"}");
 
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node1.json
index e812b057953..3fbe1d900c5 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node1.json
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node1.json
@@ -33,6 +33,7 @@
   "failCount": 0,
   "hardwareFailure" : false,
   "wantToRetire" : false,
+  "wantToDeprovision" : false,
   "history":[{"event":"readied","at":123,"agent":"system"},{"event":"reserved","at":123,"agent":"application"},{"event":"activated","at":123,"agent":"application"}],
   "ipAddresses":["::1", "127.0.0.1"],
   "additionalIpAddresses":[]
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node10.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node10.json
index a510739a793..f6368c58196 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node10.json
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node10.json
@@ -38,6 +38,7 @@
   "failCount": 0,
   "hardwareFailure" : false,
   "wantToRetire" : false,
+  "wantToDeprovision" : false,
   "history":[{"event":"readied","at":123,"agent":"system"},{"event":"reserved","at":123,"agent":"application"}],
   "ipAddresses":["::1", "127.0.0.1"],
   "additionalIpAddresses":[]
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node11.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node11.json
index 58bbd6f0128..5ff31ef24ca 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node11.json
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node11.json
@@ -18,6 +18,7 @@
   "failCount":0,
   "hardwareFailure":false,
   "wantToRetire":false,
+  "wantToDeprovision" : false,
   "history":[],
   "ipAddresses":["::1", "127.0.0.1"],
   "additionalIpAddresses":[]
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node2.json
index 387464324dd..faaff6337db 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node2.json
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node2.json
@@ -33,6 +33,7 @@
   "failCount": 0,
   "hardwareFailure" : false,
   "wantToRetire" : false,
+  "wantToDeprovision" : false,
   "history":[{"event":"readied","at":123,"agent":"system"},{"event":"reserved","at":123,"agent":"application"},{"event":"activated","at":123,"agent":"application"}],
   "ipAddresses":["::1", "127.0.0.1"],
   "additionalIpAddresses":[]
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node3.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node3.json
index 65531808404..dd9213b9110 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node3.json
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node3.json
@@ -31,6 +31,7 @@
   "failCount": 0,
   "hardwareFailure" : false,
   "wantToRetire" : false,
+  "wantToDeprovision" : false,
   "history":[{"event":"readied","at":123,"agent":"system"},{"event":"reserved","at":123,"agent":"application"},{"event":"activated","at":123,"agent":"application"}],
   "ipAddresses":["::1", "127.0.0.1"],
   "additionalIpAddresses":[]
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json
index ff6ae96ab50..cc907a2280c 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json
@@ -39,6 +39,7 @@
   "hardwareFailure": true,
   "hardwareFailureType": "memory_mcelog",
   "wantToRetire" : true,
+  "wantToDeprovision" : true,
   "history": [
     {
       "event": "readied",
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4.json
index 65f9ae2866b..e416634bbe5 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4.json
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4.json
@@ -36,6 +36,7 @@
   "failCount": 0,
   "hardwareFailure" : false,
   "wantToRetire" : false,
+  "wantToDeprovision" : false,
   "history":[{"event":"readied","at":123,"agent":"system"},{"event":"reserved","at":123,"agent":"application"}],
   "ipAddresses":["::1", "127.0.0.1"],
   "additionalIpAddresses":[]
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5-after-changes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5-after-changes.json
index 1cbb6be6213..21f1e28dff0 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5-after-changes.json
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5-after-changes.json
@@ -18,6 +18,7 @@
   "failCount": 1,
   "hardwareFailure": false,
   "wantToRetire": false,
+  "wantToDeprovision" : false,
   "history":[{"event":"readied","at":123,"agent":"system"},{"event":"failed","at":123,"agent":"system"}],
   "ipAddresses":["::1", "127.0.0.1"],
   "additionalIpAddresses":[]
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5.json
index 495e79787fd..f8e18a08ae6 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5.json
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5.json
@@ -20,6 +20,7 @@
   "failCount": 1,
   "hardwareFailure" : false,
   "wantToRetire" : false,
+  "wantToDeprovision" : false,
   "history":[{"event":"readied","at":123,"agent":"system"},{"event":"failed","at":123,"agent":"system"}],
   "ipAddresses":["::1", "127.0.0.1"],
   "additionalIpAddresses":[]
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node55.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node55.json
index eb8355626f7..864677137b5 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node55.json
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node55.json
@@ -17,6 +17,7 @@
   "failCount": 0,
   "hardwareFailure" : false,
   "wantToRetire" : false,
+  "wantToDeprovision" : false,
   "history":[{"event":"deallocated","at":123,"agent":"system"}],
   "ipAddresses":["::1", "127.0.0.1"],
   "additionalIpAddresses":[]
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6.json
index ba9802da05f..6910c132206 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6.json
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6.json
@@ -33,6 +33,7 @@
   "failCount": 0,
   "hardwareFailure" : false,
   "wantToRetire" : false,
+  "wantToDeprovision" : false,
   "history":[{"event":"readied","at":123,"agent":"system"},{"event":"reserved","at":123,"agent":"application"},{"event":"activated","at":123,"agent":"application"}],
   "ipAddresses":["::1", "127.0.0.1"],
   "additionalIpAddresses":[]
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node7.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node7.json
index 726e5627839..161ae522f64 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node7.json
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node7.json
@@ -17,6 +17,7 @@
   "failCount": 0,
   "hardwareFailure" : false,
   "wantToRetire" : false,
+  "wantToDeprovision" : false,
   "history":[],
   "ipAddresses":["::1", "127.0.0.1"],
   "additionalIpAddresses":[]
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node8.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node8.json
index ea040be7152..4c00c984881 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node8.json
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node8.json
@@ -17,6 +17,7 @@
   "failCount": 0,
   "hardwareFailure" : false,
   "wantToRetire" : false,
+  "wantToDeprovision" : false,
   "history":[],
   "ipAddresses":["127.0.0.1"],
   "additionalIpAddresses":[]
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node9.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node9.json
index 0c894a5802d..70c20065194 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node9.json
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node9.json
@@ -17,6 +17,7 @@
   "failCount": 0,
   "hardwareFailure" : false,
   "wantToRetire" : false,
+  "wantToDeprovision" : false,
   "history":[],
   "ipAddresses":["::1", "127.0.0.1"],
   "additionalIpAddresses":[]
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent1.json
index d96afb70967..41b807116db 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent1.json
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent1.json
@@ -17,6 +17,7 @@
   "failCount": 0,
   "hardwareFailure": false,
   "wantToRetire" : false,
+  "wantToDeprovision" : false,
   "history": [
     {
       "event": "readied",
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent2.json
index 844364d34c5..92e33ef1f33 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent2.json
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent2.json
@@ -17,6 +17,7 @@
   "failCount": 0,
   "hardwareFailure": false,
   "wantToRetire" : false,
+  "wantToDeprovision" : false,
   "history": [],
   "ipAddresses":["::1", "127.0.0.1"],
   "additionalIpAddresses":[]
diff --git a/persistence/src/tests/proxy/providerproxy_test.cpp b/persistence/src/tests/proxy/providerproxy_test.cpp
index 514d9516c65..28ce33a2b5b 100644
--- a/persistence/src/tests/proxy/providerproxy_test.cpp
+++ b/persistence/src/tests/proxy/providerproxy_test.cpp
@@ -8,6 +8,7 @@
 #include <vespa/document/repo/documenttyperepo.h>
 #include <vespa/document/fieldvalue/document.h>
 #include <vespa/document/update/documentupdate.h>
+#include <vespa/document/datatype/documenttype.h>
 #include <vespa/persistence/proxy/providerproxy.h>
 #include <vespa/persistence/proxy/providerstub.h>
 #include <vespa/persistence/spi/abstractpersistenceprovider.h>
diff --git a/persistence/src/tests/proxy/providerstub_test.cpp b/persistence/src/tests/proxy/providerstub_test.cpp
index 1028e06b8bc..c750a3c94af 100644
--- a/persistence/src/tests/proxy/providerstub_test.cpp
+++ b/persistence/src/tests/proxy/providerstub_test.cpp
@@ -7,6 +7,7 @@
 #include <vespa/document/util/bytebuffer.h>
 #include <vespa/document/fieldvalue/document.h>
 #include <vespa/document/update/documentupdate.h>
+#include <vespa/document/datatype/documenttype.h>
 #include <vespa/persistence/proxy/buildid.h>
 #include <vespa/persistence/proxy/providerstub.h>
 #include <vespa/persistence/spi/abstractpersistenceprovider.h>
diff --git a/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp b/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp
index d2a3ab3be65..05e21096fef 100644
--- a/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp
+++ b/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp
@@ -9,6 +9,7 @@
 #include <vespa/document/fieldset/fieldsetrepo.h>
 #include <vespa/vespalib/stllike/asciistream.h>
 #include <vespa/vespalib/util/exceptions.h>
+#include <vespa/vespalib/util/atomic.h>
 #include <vespa/vespalib/stllike/hash_map.hpp>
 #include <vespa/log/log.h>
 
diff --git a/persistence/src/vespa/persistence/proxy/providerproxy.cpp b/persistence/src/vespa/persistence/proxy/providerproxy.cpp
index 1f6d0b19be6..29b09af9953 100644
--- a/persistence/src/vespa/persistence/proxy/providerproxy.cpp
+++ b/persistence/src/vespa/persistence/proxy/providerproxy.cpp
@@ -8,9 +8,8 @@
 #include <vespa/document/fieldset/fieldsetrepo.h>
 #include <vespa/document/serialization/vespadocumentdeserializer.h>
 #include <vespa/document/serialization/vespadocumentserializer.h>
-#include <vespa/document/util/bytebuffer.h>
+#include <vespa/vespalib/util/stringfmt.h>
 #include <vespa/vespalib/objects/nbostream.h>
-#include <vespa/vespalib/util/noncopyable.hpp>
 #include <vespa/fnet/frt/frt.h>
 #include <vespa/log/log.h>
 LOG_SETUP(".providerproxy");
diff --git a/persistence/src/vespa/persistence/spi/clusterstate.cpp b/persistence/src/vespa/persistence/spi/clusterstate.cpp
index 70bc8724c96..841fef4620d 100644
--- a/persistence/src/vespa/persistence/spi/clusterstate.cpp
+++ b/persistence/src/vespa/persistence/spi/clusterstate.cpp
@@ -5,9 +5,9 @@
 #include <vespa/vdslib/distribution/distribution.h>
 #include <vespa/vespalib/objects/nbostream.h>
 #include <vespa/vespalib/stllike/asciistream.h>
+#include <cassert>
 
-namespace storage {
-namespace spi {
+namespace storage::spi {
 
 ClusterState::ClusterState(const lib::ClusterState& state,
                            uint16_t nodeIndex,
@@ -99,5 +99,4 @@ void ClusterState::serialize(vespalib::nbostream& o) const {
     o << _distribution->serialize();
 }
 
-} // spi
-} // storage
+}
diff --git a/persistence/src/vespa/persistence/spi/docentry.cpp b/persistence/src/vespa/persistence/spi/docentry.cpp
index c9ceda982e0..1061a06fd28 100644
--- a/persistence/src/vespa/persistence/spi/docentry.cpp
+++ b/persistence/src/vespa/persistence/spi/docentry.cpp
@@ -3,6 +3,7 @@
 #include "docentry.h"
 #include <vespa/document/fieldvalue/document.h>
 #include <sstream>
+#include <cassert>
 
 namespace storage {
 namespace spi {
diff --git a/persistence/src/vespa/persistence/spi/result.h b/persistence/src/vespa/persistence/spi/result.h
index 359f6af5164..93db642611f 100644
--- a/persistence/src/vespa/persistence/spi/result.h
+++ b/persistence/src/vespa/persistence/spi/result.h
@@ -5,6 +5,7 @@
 #include "bucketinfo.h"
 #include "bucket.h"
 #include "docentry.h"
+#include <vespa/document/bucket/bucketidlist.h>
 
 namespace storage::spi {
 
@@ -196,7 +197,7 @@ private:
 
 class BucketIdListResult : public Result {
 public:
-    typedef document::BucketId::List List;
+    using List = document::bucket::BucketIdList;
 
     /**
      * Constructor used when there was an error listing the buckets.
diff --git a/pom.xml b/pom.xml
index b45a4f31c4b..42665292a23 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1066,7 +1066,7 @@
         <curator.version>2.9.1</curator.version>
         <jackson2.version>2.8.3</jackson2.version>
         <jersey2.version>2.23.2</jersey2.version>
-        <jetty.version>9.4.5.v20170502</jetty.version>
+        <jetty.version>9.4.6.v20170531</jetty.version>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
         <test.hide>true</test.hide>
@@ -1168,6 +1168,7 @@
         <module>vdslib</module>
         <module>vespaclient-core</module>
         <module>vespaclient-container-plugin</module>
+        <module>vespaclient-java</module>
         <module>vespa-application-maven-plugin</module>
         <module>vespa-documentgen-plugin</module>
         <module>vespa_feed_perf</module>
diff --git a/sample-apps/blog-recommendation/README.md b/sample-apps/blog-recommendation/README.md
index 0b6a00c9743..705c4e6879c 100644
--- a/sample-apps/blog-recommendation/README.md
+++ b/sample-apps/blog-recommendation/README.md
@@ -1,55 +1 @@
-Basic Search Application
-==================
-Start by [deploying a sample application](http://vespa.corp.yahoo.com/6/documentation/developing-with-vespa.html).
-
-### Find the endpoint
-
-When you have successfully deployed your own compiled version of the application above, you need to find the name of the "endpoint".
-The endpoint is used for feeding and searching for data.
-**Please allow a few minutes for the endpoint to appear after deployment**
-You can find this endpoint by doing:
- ```sh
-
- mvn vespa:endpoints | grep Endpoints
- ```
-
-You can also find it by looking at the [Hosted Vespa Dashboard](http://dashboard.vespa.corp.yahoo.com).
-
-
-### Feed and search
- 1. **Feed** the data that is to be searched
- ```sh
-
- # Feeding two documents
- curl -X POST --data-binary  @music-data-1.json <endpoint url>/document/v1/music/music/docid/1 | python -m json.tool
- curl -X POST --data-binary  @music-data-2.json <endpoint url>/document/v1/music/music/docid/2 | python -m json.tool
-
-  ```
-
-For feeding many documents fast and reliable, checkout [feeding example](https://git.corp.yahoo.com/vespa-samples/basic-feeding-client)
-
- 2. **Visit documents
-
- Since we do not have many documents we can list them all
- ```sh
-
- # All documents
- curl <endpoint url>/document/v1/music/music/docid | python -m json.tool
-
- # Document with id 1
- curl <endpoint url>/document/v1/music/music/docid/1 | python -m json.tool
-
-  ```
-
- 3. **Search**
- We can also search for documents:
-    ```sh
-
-    curl '<endpoint url>/search/?query=bad' | python -m json.tool
-
-
-    ```
-
-### Next step: from development to production
-See [continuous deployments](http://vespa.corp.yahoo.com/6/documentation/continuous-deployment.html) for how to implement continuous deployments for production.
-See [RESTified Document Operation API](http://vespa.corp.yahoo.com/6/documentation/document_api_v1.html) for documentation about the REST API for document operations.
+Blog Recommendation
diff --git a/sample-apps/blog-recommendation/src/pig/feed_content_and_tensor_vespa.pig b/sample-apps/blog-recommendation/src/pig/feed_content_and_tensor_vespa.pig
new file mode 100644
index 00000000000..9a536f38779
--- /dev/null
+++ b/sample-apps/blog-recommendation/src/pig/feed_content_and_tensor_vespa.pig
@@ -0,0 +1,100 @@
+REGISTER vespa-hadoop.jar
+
+-- Create valid Vespa put operations
+DEFINE VespaPutOperationDoc
+       com.yahoo.vespa.hadoop.pig.VespaDocumentOperation(
+            'operation=put',
+            'docid=id:blog-recommendation:blog_post::<post_id>',
+            'create-tensor-fields=user_item_cf',
+            'simple-array-fields=tags,categories'
+       );
+
+DEFINE VespaPutOperationUser
+       com.yahoo.vespa.hadoop.pig.VespaDocumentOperation(
+            'operation=put',
+            'docid=id:blog-recommendation:user::<user_id>',
+            'create-tensor-fields=user_item_cf',
+            'simple-array-fields=has_read_items'
+       );
+
+-- Transform tabular data to a Vespa document operation JSON format
+DEFINE VespaStorage
+       com.yahoo.vespa.hadoop.pig.VespaStorage();
+
+-- DEFINE VespaStorage
+--        com.yahoo.vespa.hadoop.pig.VespaStorage(
+--             'create-document-operation=true',
+--             'operation=put',
+--             'docid=id:blog-recommendation:blog_post::<post_id>'
+--        );
+
+-- Load data from any source - here we load using PigStorage
+data = LOAD 'blog-recommendation/trainPostsFinal' USING JsonLoader('date_gmt:chararray, language:chararray, author:chararray, url:chararray, title:chararray, blog:chararray, post_id:chararray, tags:{T:(tag_name:chararray)}, blogname:chararray, date:chararray, content:chararray, categories:{T:(category_name:chararray)}, likes:{T:(dt:chararray, uid:chararray)}');
+
+data_for_feed = FOREACH data GENERATE 
+  date_gmt, 
+  language, 
+  author, 
+  url, 
+  title, 
+  blog, 
+  post_id, 
+  tags, 
+  blogname,
+  content, 
+  categories;
+
+data_doc = LOAD 'blog-recommendation/user_item_cf/product_features' USING JsonLoader('post_id:chararray, user_item_cf:[double]');
+
+data_content_and_doc_tensor = JOIN data_for_feed BY post_id LEFT, data_doc BY post_id;
+data_content_and_doc_tensor = FOREACH data_content_and_doc_tensor GENERATE
+  date_gmt AS date_gmt, 
+  language AS language, 
+  author AS author, 
+  url AS url, 
+  title AS title, 
+  blog AS blog, 
+  data_for_feed::post_id as post_id, 
+  tags AS tags, 
+  blogname AS blogname,
+  content AS content, 
+  categories AS categories,
+  user_item_cf AS user_item_cf,
+  (user_item_cf IS NOT NULL ? 1 : 0) AS has_user_item_cf;
+
+data_content_and_doc_tensor_feed = FOREACH data_content_and_doc_tensor GENERATE VespaPutOperationDoc(*);
+
+-- use cf latent factor
+data_user = LOAD 'blog-recommendation/user_item_cf/user_features' USING JsonLoader('user_id:chararray, user_item_cf:[double]');
+data_user = FOREACH data_user GENERATE  
+  user_id AS user_id,
+  user_item_cf AS user_item_cf;
+  
+
+-- Articles already liked
+data_likes = FOREACH data GENERATE post_id, FLATTEN(likes) AS (dt, uid); 
+
+post_liked_per_user = GROUP data_likes BY uid;
+post_liked_per_user = FOREACH post_liked_per_user GENERATE group AS user_id, data_likes.post_id AS has_read_items;
+
+-- Join user data
+data_user = JOIN post_liked_per_user BY user_id FULL, data_user BY user_id;
+
+data_user = FOREACH data_user GENERATE (post_liked_per_user::user_id IS NOT NULL ? post_liked_per_user::user_id : data_user::user_id) AS user_id,
+                                       user_item_cf AS user_item_cf,
+                                       (user_item_cf IS NOT NULL ? 1 : 0) AS has_user_item_cf,
+                                       has_read_items AS has_read_items;
+data_user = FILTER data_user BY user_id IS NOT NULL;
+
+data_user_for_feed = FOREACH data_user GENERATE VespaPutOperationUser(*);
+
+joint_content_tensors = UNION data_content_and_doc_tensor_feed, data_user_for_feed;
+
+-- STORE data_for_feed into 'vespa_put_operation';
+
+-- Store into Vespa
+STORE joint_content_tensors INTO '$ENDPOINT' USING VespaStorage();
+
+
+
+
diff --git a/sample-apps/blog-recommendation/src/pig/feed_content_vespa.pig b/sample-apps/blog-recommendation/src/pig/feed_content_vespa.pig
new file mode 100644
index 00000000000..59b173e16f4
--- /dev/null
+++ b/sample-apps/blog-recommendation/src/pig/feed_content_vespa.pig
@@ -0,0 +1,71 @@
+REGISTER vespa-hadoop.jar
+
+-- UDF to create valid Vespa document operation in JSON format
+DEFINE VespaPutOperationDoc
+       com.yahoo.vespa.hadoop.pig.VespaDocumentOperation(
+            'operation=put',
+            'docid=id:blog-search:blog_post::<post_id>',
+            'simple-array-fields=tags,categories'
+            );
+
+-- UDF to send data to a Vespa endpoint
+DEFINE VespaStorage
+       com.yahoo.vespa.hadoop.pig.VespaStorage();
+
+-- Load data from any source - here we load using JsonLoader
+data_first_release = LOAD 'blog-recommendation/first_release/trainPosts.json' USING 
+  JsonLoader('date_gmt:chararray, 
+              language:chararray, 
+              author:chararray, 
+              url:chararray, 
+              title:chararray, 
+              blog:chararray, 
+              post_id:chararray, 
+              tags:{T:(tag_name:chararray)}, 
+              blogname:chararray, 
+              date:chararray, 
+              content:chararray, 
+              categories:{T:(category_name:chararray)}, 
+              likes:{T:(dt:chararray, uid:chararray)}');
+
+data_second_release = LOAD 'blog-recommendation/second_release/trainPosts.json' USING 
+  JsonLoader('date_gmt:chararray, 
+              language:chararray, 
+              author:chararray, 
+              url:chararray, 
+              title:chararray, 
+              blog:chararray, 
+              post_id:chararray, 
+              tags:{T:(tag_name:chararray)}, 
+              blogname:chararray, 
+              date:chararray, 
+              content:chararray, 
+              categories:{T:(category_name:chararray)}, 
+              likes:{T:(dt:chararray, uid:chararray)}');
+
+data = UNION data_first_release, data_second_release;
+
+-- Select fields that will be sent to Vespa.
+-- This should follow blog_post.sd
+data_for_feed = FOREACH data GENERATE 
+  date_gmt,  
+  language, 
+  author, 
+  url, 
+  title, 
+  blog, 
+  post_id, 
+  tags, 
+  blogname,
+  content, 
+  categories;
+
+-- Create valid Vespa put operations in JSON format
+data_for_feed_json = FOREACH data_for_feed GENERATE VespaPutOperationDoc(*);
+
+-- Sample Vespa operations
+-- data_for_feed_json_sample = SAMPLE data_for_feed_json 0.0005;
+-- STORE data_for_feed_json_sample INTO 'blog-sample';
+
+-- Store into Vespa
+STORE data_for_feed_json INTO '$ENDPOINT' USING VespaStorage();
+\ No newline at end of file
diff --git a/sample-apps/blog-recommendation/src/pig/feed_user_item_cf_vespa.pig b/sample-apps/blog-recommendation/src/pig/feed_user_item_cf_vespa.pig
new file mode 100644
index 00000000000..fd06394c3af
--- /dev/null
+++ b/sample-apps/blog-recommendation/src/pig/feed_user_item_cf_vespa.pig
@@ -0,0 +1,37 @@
+REGISTER vespa-hadoop.jar
+
+-- Create valid Vespa put operations
+DEFINE VespaPutOperationDoc
+       com.yahoo.vespa.hadoop.pig.VespaDocumentOperation(
+            'operation=put',
+            'docid=id:blog-recommendation:blog_post::<post_id>',
+            'create-tensor-fields=user_item_cf'
+       );
+
+DEFINE VespaPutOperationUser
+       com.yahoo.vespa.hadoop.pig.VespaDocumentOperation(
+            'operation=put',
+            'docid=id:blog-recommendation:user::<user_id>',
+            'create-tensor-fields=user_item_cf'
+       );
+
+-- Transform tabular data to a Vespa document operation JSON format
+DEFINE VespaStorage
+       com.yahoo.vespa.hadoop.pig.VespaStorage();
+
+
+data_doc = LOAD 'blog-recommendation/user_item_cf/product_features' USING JsonLoader('post_id:chararray, user_item_cf:[double]');
+data_doc_for_feed = FOREACH data_doc GENERATE VespaPutOperationDoc(*);
+
+
+data_user = LOAD 'blog-recommendation/user_item_cf/user_features' USING JsonLoader('user_id:chararray, user_item_cf:[double]');
+data_user_for_feed = FOREACH data_user GENERATE VespaPutOperationUser(*);
+
+
+-- Store into Vespa
+STORE data_doc_for_feed INTO '$ENDPOINT' USING VespaStorage();
+STORE data_user_for_feed INTO '$ENDPOINT' USING VespaStorage();
+
+
+
+
diff --git a/sample-apps/blog-recommendation/src/pig/generate_user_item_cf_dataset.pig b/sample-apps/blog-recommendation/src/pig/generate_user_item_cf_dataset.pig
new file mode 100644
index 00000000000..2e71dcbe9e3
--- /dev/null
+++ b/sample-apps/blog-recommendation/src/pig/generate_user_item_cf_dataset.pig
@@ -0,0 +1,15 @@
+-- Load data from any source - here we load using PigStorage
+data = LOAD 'blog-recommendation/trainPostsFinal' USING JsonLoader('date_gmt:chararray, language:chararray, author:chararray, url:chararray, title:chararray, blog:chararray, post_id:chararray, tags:{T:(tag_name:chararray)}, blogname:chararray, date:chararray, content:chararray, categories:{T:(category_name:chararray)}, likes:{T:(dt:chararray, uid:chararray)}');
+
+data_likes = FOREACH data GENERATE post_id, FLATTEN(likes) AS (dt, uid);
+
+data_cf = FOREACH data_likes GENERATE uid, post_id, 1 as rate;
+
+data_cf = FILTER data_cf BY (uid IS NOT NULL) AND (uid != '') AND (post_id IS NOT NULL) AND (post_id != ''); 
+
+-- data_cf_sample = SAMPLE data_cf 0.001;
+
+-- data_cf = LIMIT data_cf 10;
+
+STORE data_cf INTO 'blog-recommendation/trainPostsFinal_user_item_cf';
+
diff --git a/sample-apps/blog-recommendation/src/pig/get_recommendations.pig b/sample-apps/blog-recommendation/src/pig/get_recommendations.pig
new file mode 100644
index 00000000000..00b03b0f49a
--- /dev/null
+++ b/sample-apps/blog-recommendation/src/pig/get_recommendations.pig
@@ -0,0 +1,29 @@
+-- REGISTER $VESPA_HADOOP_JAR
+REGISTER vespa-hadoop.jar
+-- REGISTER parquet-pig-bundle-1.8.1.jar
+
+-- Define Vespa query for retrieving blog posts
+DEFINE  BlogPostRecommendations
+        com.yahoo.vespa.hadoop.pig.VespaQuery(
+            'query=http://ENDPOINT:8080/search/?user_id=<userid>&hits=100',
+            'schema=rank:int,id:chararray,relevance:double,fields/post_id:chararray'
+        );
+
+-- Load test_set data from a local file
+test_set = LOAD 'data/cv/test_set_exploded' AS (post_id:chararray, userid:chararray);
+users = FOREACH test_set GENERATE userid;
+users = FILTER users BY userid IS NOT null;
+users = DISTINCT users;
+
+users_limit = LIMIT users 10;
+
+-- Run a set of queries against Vespa
+recommendations = FOREACH users_limit GENERATE userid, 
+                                               FLATTEN(BlogPostRecommendations(*)) AS (rank, id, relevance, post_id);
+recommendations = FOREACH recommendations GENERATE userid, rank, post_id;
+
+recommendations = FILTER recommendations BY rank IS NOT NULL AND post_id IS NOT NULL;                                              
+
+-- Output recommendations
+STORE recommendations INTO 'data/recommendations' USING PigStorage('\t', '-schema');
+-- STORE recommendations INTO 'data/recommendations' USING org.apache.parquet.pig.ParquetStorer();
diff --git a/sample-apps/blog-recommendation/src/pig/tutorial_blog_popularity.pig b/sample-apps/blog-recommendation/src/pig/tutorial_blog_popularity.pig
new file mode 100644
index 00000000000..4dac36a717f
--- /dev/null
+++ b/sample-apps/blog-recommendation/src/pig/tutorial_blog_popularity.pig
@@ -0,0 +1,55 @@
+REGISTER '$VESPA_HADOOP_JAR'
+
+-- UDF to create valid Vespa document operation in JSON format
+DEFINE VespaUpdateOperationDoc
+       com.yahoo.vespa.hadoop.pig.VespaDocumentOperation(
+            'operation=update',
+            'docid=id:blog-search:blog_post::<post_id>'
+       );
+
+-- UDF to send data to a Vespa endpoint
+DEFINE VespaStorage
+       com.yahoo.vespa.hadoop.pig.VespaStorage();
+
+-- Load data from any source - here we load using JsonLoader
+data = LOAD '$DATA_PATH' USING 
+  JsonLoader('date_gmt:chararray, 
+              language:chararray, 
+              author:chararray, 
+              url:chararray, 
+              title:chararray, 
+              blog:chararray, 
+              post_id:chararray, 
+              tags:{T:(tag_name:chararray)}, 
+              blogname:chararray, 
+              date:chararray, 
+              content:chararray, 
+              categories:{T:(category_name:chararray)}, 
+              likes:{T:(dt:chararray, uid:chararray)}');
+
+data = FILTER data BY likes IS NOT NULL;
+
+data_likes = FOREACH data GENERATE 
+  blog, 
+  post_id, 
+  blogname,
+  FLATTEN(likes) AS (dt, uid);
+
+-- data_likes_limit = LIMIT data_likes 10;
+
+likes = FOREACH (GROUP data_likes ALL) 
+     GENERATE COUNT(data_likes) as total_number;
+
+blog_popularity = FOREACH (GROUP data_likes BY blog) GENERATE 
+  group as blog,
+  (double)COUNT(data_likes)/(double)likes.total_number AS popularity;
+
+data_update = JOIN data_likes BY blog, blog_popularity BY blog;
+data_update = FOREACH data_update GENERATE
+  post_id, popularity;
+
+-- Create valid Vespa put operations in JSON format
+data_for_feed_json = FOREACH data_update GENERATE VespaUpdateOperationDoc(*);
+
+-- Store into Vespa
+STORE data_for_feed_json INTO '$ENDPOINT' USING VespaStorage();
diff --git a/sample-apps/blog-recommendation/src/pig/tutorial_feed_content_and_tensor_vespa.pig b/sample-apps/blog-recommendation/src/pig/tutorial_feed_content_and_tensor_vespa.pig
new file mode 100644
index 00000000000..77943fd842a
--- /dev/null
+++ b/sample-apps/blog-recommendation/src/pig/tutorial_feed_content_and_tensor_vespa.pig
@@ -0,0 +1,116 @@
+REGISTER '$VESPA_HADOOP_JAR'
+
+-- Create valid Vespa put operations
+DEFINE VespaPutOperationDoc
+       com.yahoo.vespa.hadoop.pig.VespaDocumentOperation(
+            'operation=put',
+            'docid=id:blog-recommendation:blog_post::<post_id>',
+            'create-tensor-fields=user_item_cf',
+            'simple-array-fields=tags,categories'
+       );
+
+DEFINE VespaPutOperationUser
+       com.yahoo.vespa.hadoop.pig.VespaDocumentOperation(
+            'operation=put',
+            'docid=id:blog-recommendation:user::<user_id>',
+            'create-tensor-fields=user_item_cf',
+            'simple-array-fields=has_read_items'
+       );
+
+-- Transform tabular data to a Vespa document operation JSON format
+DEFINE VespaStorage
+       com.yahoo.vespa.hadoop.pig.VespaStorage();
+
+-- Load data 
+data = LOAD '$DATA_PATH' USING 
+  JsonLoader('date_gmt:chararray, 
+              language:chararray, 
+              author:chararray, 
+              url:chararray, 
+              title:chararray, 
+              blog:chararray, 
+              post_id:chararray, 
+              tags:{T:(tag_name:chararray)}, 
+              blogname:chararray, 
+              date:chararray, 
+              content:chararray, 
+              categories:{T:(category_name:chararray)}, 
+              likes:{T:(dt:chararray, uid:chararray)}');
+
+data_for_feed = FOREACH data GENERATE 
+  date_gmt, 
+  language, 
+  author, 
+  url, 
+  title, 
+  blog, 
+  post_id, 
+  tags, 
+  blogname,
+  content, 
+  categories;
+
+-- Load Blog post CF latent factors
+data_doc = LOAD '$BLOG_POST_FACTORS' USING 
+  JsonLoader('post_id:chararray, 
+              user_item_cf:[double]');
+
+-- Join data and latent factors
+data_content_and_doc_tensor = JOIN data_for_feed BY post_id LEFT, data_doc BY post_id;
+data_content_and_doc_tensor = FOREACH data_content_and_doc_tensor GENERATE
+  date_gmt AS date_gmt, 
+  language AS language, 
+  author AS author, 
+  url AS url, 
+  title AS title, 
+  blog AS blog, 
+  data_for_feed::post_id as post_id, 
+  tags AS tags, 
+  blogname AS blogname,
+  content AS content, 
+  categories AS categories,
+  user_item_cf AS user_item_cf,
+  (user_item_cf IS NOT NULL ? 1 : 0) AS has_user_item_cf;
+
+-- Generate valid Vespa JSON format
+data_content_and_doc_tensor_feed = FOREACH data_content_and_doc_tensor GENERATE VespaPutOperationDoc(*);
+
+-- Load User CF latent factors
+data_user = LOAD '$USER_FACTORS' USING 
+  JsonLoader('user_id:chararray, 
+              user_item_cf:[double]');
+data_user = FOREACH data_user GENERATE  
+  user_id AS user_id,
+  user_item_cf AS user_item_cf;
+  
+-- Articles already liked
+data_likes = FOREACH data GENERATE post_id, FLATTEN(likes) AS (dt, uid); 
+
+post_liked_per_user = GROUP data_likes BY uid;
+post_liked_per_user = FOREACH post_liked_per_user GENERATE 
+  group AS user_id, 
+  data_likes.post_id AS has_read_items;
+
+-- Join user data
+data_user = JOIN post_liked_per_user BY user_id FULL, 
+                 data_user BY user_id;
+
+data_user = FOREACH data_user GENERATE 
+  (post_liked_per_user::user_id IS NOT NULL ? post_liked_per_user::user_id : data_user::user_id) AS user_id,
+  user_item_cf AS user_item_cf,
+  (user_item_cf IS NOT NULL ? 1 : 0) AS has_user_item_cf,
+  has_read_items AS has_read_items;
+
+data_user = FILTER data_user BY user_id IS NOT NULL;
+
+-- Generate valid Vespa JSON format
+data_user_for_feed = FOREACH data_user GENERATE VespaPutOperationUser(*);
+
+joint_content_tensors = UNION data_content_and_doc_tensor_feed, data_user_for_feed;
+
+-- Store into Vespa
+STORE joint_content_tensors INTO '$ENDPOINT' USING VespaStorage();
+
+
+
+
diff --git a/sample-apps/blog-recommendation/src/pig/tutorial_feed_content_vespa.pig b/sample-apps/blog-recommendation/src/pig/tutorial_feed_content_vespa.pig
new file mode 100644
index 00000000000..d20ccf505a9
--- /dev/null
+++ b/sample-apps/blog-recommendation/src/pig/tutorial_feed_content_vespa.pig
@@ -0,0 +1,51 @@
+REGISTER '$VESPA_HADOOP_JAR'
+-- REGISTER vespa-hadoop.jar
+
+-- UDF to create valid Vespa document operation in JSON format
+DEFINE VespaPutOperationDoc
+       com.yahoo.vespa.hadoop.pig.VespaDocumentOperation(
+            'operation=put',
+            'docid=id:blog-search:blog_post::<post_id>',
+            'simple-array-fields=tags,categories'
+       );
+
+-- UDF to send data to a Vespa endpoint
+DEFINE VespaStorage
+       com.yahoo.vespa.hadoop.pig.VespaStorage();
+
+-- Load data from any source - here we load using JsonLoader
+data = LOAD '$DATA_PATH' USING 
+  JsonLoader('date_gmt:chararray, 
+              language:chararray, 
+              author:chararray, 
+              url:chararray, 
+              title:chararray, 
+              blog:chararray, 
+              post_id:chararray, 
+              tags:{T:(tag_name:chararray)}, 
+              blogname:chararray, 
+              date:chararray, 
+              content:chararray, 
+              categories:{T:(category_name:chararray)}, 
+              likes:{T:(dt:chararray, uid:chararray)}');
+
+-- Select fields that will be sent to Vespa.
+-- This should follow blog_post.sd
+data_for_feed = FOREACH data GENERATE 
+  date_gmt, 
+  language, 
+  author, 
+  url, 
+  title, 
+  blog, 
+  post_id, 
+  tags, 
+  blogname,
+  content, 
+  categories;
+
+-- Create valid Vespa put operations in JSON format
+data_for_feed_json = FOREACH data_for_feed GENERATE VespaPutOperationDoc(*);
+
+-- Store into Vespa
+STORE data_for_feed_json INTO '$ENDPOINT' USING VespaStorage();
diff --git a/sample-apps/blog-recommendation/src/spark/collaborative_filtering_example.scala b/sample-apps/blog-recommendation/src/spark/collaborative_filtering_example.scala
new file mode 100644
index 00000000000..1a2c8f92730
--- /dev/null
+++ b/sample-apps/blog-recommendation/src/spark/collaborative_filtering_example.scala
@@ -0,0 +1,59 @@
+import org.apache.spark.mllib.recommendation.ALS
+import org.apache.spark.mllib.recommendation.MatrixFactorizationModel
+import org.apache.spark.mllib.recommendation.Rating
+import scala.util.parsing.json.JSONObject
+
+// Load and parse the data
+val data = sc.textFile("blog-recommendation/trainPostsFinal_user_item_cf")
+val ratings = data.map(_.split('\t') match { case Array(user, item, rate) =>
+  Rating(user.toInt, item.toInt, rate.toDouble)
+})
+
+// Build the recommendation model using ALS
+val rank = 10
+val numIterations = 10
+val model = ALS.train(ratings, rank, numIterations, 0.01)
+
+// Evaluate the model on rating data
+val usersProducts = ratings.map { case Rating(user, product, rate) =>
+  (user, product)
+}
+val predictions =
+  model.predict(usersProducts).map { case Rating(user, product, rate) =>
+    ((user, product), rate)
+  }
+val ratesAndPreds = ratings.map { case Rating(user, product, rate) =>
+  ((user, product), rate)
+}.join(predictions)
+val MSE = ratesAndPreds.map { case ((user, product), (r1, r2)) =>
+  val err = (r1 - r2)
+  err * err
+}.mean()
+println("Mean Squared Error = " + MSE)
+
+def writeModelFeaturesAsTensor (modelFeatures:(Int, Array[Double]), id_string:String) = {
+
+  val id = modelFeatures._1
+  val latentVector = modelFeatures._2
+  var latentVectorMap:Map[String,Double] = Map()
+  var output:Map[String,Any] = Map()
+
+  for ( i <- 0 until latentVector.length ){
+
+    latentVectorMap += (("user_item_cf:" + i.toString, latentVector(i)))      
+
+  }
+
+  output += ((id_string, id))
+  output += (("user_item_cf", scala.util.parsing.json.JSONObject(latentVectorMap)))
+
+  JSONObject(output)
+
+}
+
+val product_features = model.productFeatures.map(x => writeModelFeaturesAsTensor(x, "post_id"))
+product_features.saveAsTextFile("blog-recommendation/user_item_cf/product_features")
+val user_features = model.userFeatures.map(x => writeModelFeaturesAsTensor(x, "user_id"))
+user_features.saveAsTextFile("blog-recommendation/user_item_cf/user_features")
+
+
diff --git a/sample-apps/blog-recommendation/src/spark/data_exploration.scala b/sample-apps/blog-recommendation/src/spark/data_exploration.scala
new file mode 100644
index 00000000000..228834cfb4b
--- /dev/null
+++ b/sample-apps/blog-recommendation/src/spark/data_exploration.scala
@@ -0,0 +1,63 @@
+// sc is an existing SparkContext.
+val sqlContext = new org.apache.spark.sql.SQLContext(sc)
+
+val original_train_post_path = "blog-recommendation-support/data/original_data/trainPosts.json"
+val original_train_post_thin_path = "blog-recommendation-support/data/original_data/trainPostsThin.json"
+val original_test_post_thin_path = "blog-recommendation-support/data/original_data/testPostsThin.json"
+
+val original_train_post = sqlContext.read.json(original_train_post_path)
+val original_train_post_thin = sqlContext.read.json(original_train_post_thin_path)
+val original_test_post_thin = sqlContext.read.json(original_test_post_thin_path)
+
+val count_original_train = original_train_post.count()
+val count_original_train_thin = original_train_post_thin.count()
+val count_original_test_thin = original_test_post_thin.count()
+
+// The inferred schema can be visualized using the printSchema() method.
+original_train_post.printSchema()
+original_train_post_thin.printSchema()
+original_test_post_thin.printSchema()
+
+// No intersection between train and test data
+original_train_post_thin.join(original_test_post_thin, original_train_post_thin("post_id") == original_test_post_thin("post_id")).count(2)
+
+// original_train_minimal_df
+var original_train_minimal_df = original_train_post.select($"date_gmt", $"post_id", size($"likes").as("number_likes"), $"likes")
+// no duplicate post_id
+original_train_minimal_df.select("post_id").dropDuplicates().count() - original_train_minimal_df.select("post_id").count()
+
+// CHECK THIS DECISION - I SHOULD NOT EXLUDE POST_ID WITH ZERO LIKES
+// OTHERWISE THERE WILL BE NO DOCUMENT IN THE TEST SET THAT NO ONE HAS LIKED,
+// WHICH MAKES THE EXERCISE MUCH EASIER
+// only post_id with at least one like
+// original_train_minimal_df = original_train_minimal_df.filter("number_likes > 0")
+
+// Set some post_id aside to be present only on the test set 
+var sets = original_train_minimal_df.randomSplit(Array(0.95, 0.05), 123)
+
+var training_set = sets(0)
+var test_set = sets(1)
+
+// flat dataframe so that each line is a combination of post_id and user
+training_set = training_set.select($"post_id", explode($"likes").as("likes_flat"))
+training_set = training_set.select("post_id", "likes_flat.uid")
+
+test_set = test_set.select($"post_id", explode($"likes").as("likes_flat"))
+test_set = test_set.select("post_id", "likes_flat.uid")
+
+// randomly move some (post_id, uid) from training set to test set
+sets = training_set.randomSplit(Array(0.85, 0.15), 123)
+
+training_set = sets(0)
+var additional_test_set = sets(1)
+
+// concatenate test_set and additional_test_set
+test_set = test_set.unionAll(additional_test_set)
+
+// see number of likes distribution
+val like_dist = original_train_minimal_df.groupBy("number_likes").count().orderBy(asc("number_likes")).collect()
+like_dist.map(println)
+
+
+
+
diff --git a/sample-apps/blog-recommendation/src/spark/expected_percentile.scala b/sample-apps/blog-recommendation/src/spark/expected_percentile.scala
new file mode 100644
index 00000000000..986a3eb79f4
--- /dev/null
+++ b/sample-apps/blog-recommendation/src/spark/expected_percentile.scala
@@ -0,0 +1,39 @@
+val test_file_path = "data/cv/test_set_exploded"
+val blog_recom_file_path = "data/recommendations"
+val size_recommendation_list = 100
+
+val sqlContext = new org.apache.spark.sql.SQLContext(sc)
+
+val test_set = sc.textFile(test_file_path).
+  map(_.split("\t")).map(p => (p(0).toString, p(1).toString)).
+  toDF("post_id", "user_id")
+
+val recommendations = sc.textFile(blog_recom_file_path).
+  map(_.split("\t")).map(p => (p(0).toString, p(1).toString, p(2).toString)).
+  toDF("user_id", "rank", "post_id")
+
+// val recommendations = sqlContext.createDataFrame(Seq(
+//   ("16966742", "5", "1009088"),
+//   ("30463255", "10", "1044974")
+// )).toDF("user_id", "rank", "post_id")
+
+// join data
+var joined_data = test_set.
+  join(recommendations, 
+       test_set("post_id") === recommendations("post_id") && 
+        test_set("user_id") === recommendations("user_id")).
+  select(test_set("post_id"), 
+         test_set("user_id"), 
+         recommendations("rank"))
+
+// transform and add a column
+joined_data = joined_data.withColumn("percentile", joined_data("rank")/size_recommendation_list)
+
+val expected_percentile = joined_data.
+  // groupBy($"user_id").
+  groupBy().
+  agg(sum($"percentile").as("sum_percentile"), 
+      count($"post_id").as("number_read")).
+  withColumn("expected_percentile", $"sum_percentile" / $"number_read")
+
+expected_percentile.show()
+\ No newline at end of file
diff --git a/sample-apps/blog-recommendation/src/spark/full_dataset_cf.scala b/sample-apps/blog-recommendation/src/spark/full_dataset_cf.scala
new file mode 100644
index 00000000000..0b76e8b8b1c
--- /dev/null
+++ b/sample-apps/blog-recommendation/src/spark/full_dataset_cf.scala
@@ -0,0 +1,60 @@
+import org.apache.spark.mllib.recommendation.ALS
+import org.apache.spark.mllib.recommendation.MatrixFactorizationModel
+import org.apache.spark.mllib.recommendation.Rating
+import scala.util.parsing.json.JSONObject
+
+// Prepare data
+
+val data_path = "data/original_data/trainPosts.json"
+
+val sqlContext = new org.apache.spark.sql.SQLContext(sc)
+
+val full_dataset = sqlContext.read.json(data_path)
+
+var data = full_dataset.select($"post_id", explode($"likes").as("likes_flat"))
+data = data.select($"likes_flat.uid".as("uid"), $"post_id")
+
+data = data.filter("uid is not null and uid != '' and post_id is not null and post_id != ''")
+
+val ratings = data.rdd.map(x => (x(0).toString, x(1).toString) match { 
+  case (user, item) =>  Rating(user.toInt, item.toInt, 1)
+})
+
+// Train the model
+
+val rank = 10
+val numIterations = 10
+val model = ALS.train(ratings, rank, numIterations, 0.01)
+
+// Convert latent vectors from model to Vespa Tensor model
+
+def writeModelFeaturesAsTensor (modelFeatures:(Int, Array[Double]), id_string:String) = {
+
+  val id = modelFeatures._1
+  val latentVector = modelFeatures._2
+  var latentVectorMap:Map[String,Double] = Map()
+  var output:Map[String,Any] = Map()
+
+  for ( i <- 0 until latentVector.length ){
+
+    latentVectorMap += (("user_item_cf:" + i.toString, latentVector(i)))      
+
+  }
+
+  output += ((id_string, id))
+  output += (("user_item_cf", scala.util.parsing.json.JSONObject(latentVectorMap)))
+
+  JSONObject(output)
+
+}
+
+// Write user and item latent factors to disk
+
+val product_features = model.productFeatures.map(x => writeModelFeaturesAsTensor(x, "post_id"))
+product_features.saveAsTextFile("data/user_item_cf/product_features")
+val user_features = model.userFeatures.map(x => writeModelFeaturesAsTensor(x, "user_id"))
+user_features.saveAsTextFile("data/user_item_cf/user_features")
+
+
+
+
diff --git a/sample-apps/blog-recommendation/src/spark/train_test_set_division.scala b/sample-apps/blog-recommendation/src/spark/train_test_set_division.scala
new file mode 100644
index 00000000000..2fc67734386
--- /dev/null
+++ b/sample-apps/blog-recommendation/src/spark/train_test_set_division.scala
@@ -0,0 +1,45 @@
+import org.apache.spark.sql.functions.udf
+
+// Inputs
+val input_file_path = "data/original_data/trainPosts.json"
+val test_perc_stage1 = 0.05
+val test_perc_stage2 = 0.15
+val training_file_path = "data/cv/training_set_exploded"
+val test_file_path = "data/cv/test_set_exploded"
+val seed = 123
+
+val sqlContext = new org.apache.spark.sql.SQLContext(sc)
+
+// Load full dataset
+val full_dataset = sqlContext.read.json(input_file_path)
+val full_dataset_simple = full_dataset.select($"post_id", size($"likes").as("number_likes"), $"likes")
+
+// Set some blog posts aside to be present only on the test set 
+var sets = full_dataset_simple.randomSplit(Array(1 - test_perc_stage1, test_perc_stage1), seed)
+
+var training_set = sets(0)
+val training_set_null = training_set.filter("number_likes == 0")
+var training_set_exploded = training_set.select($"post_id", explode($"likes").as("likes_flat"))
+training_set_exploded = training_set_exploded.select("post_id", "likes_flat.uid")
+
+var test_set = sets(1)
+val test_set_null = test_set.filter("number_likes == 0")
+var test_set_exploded = test_set.select($"post_id", explode($"likes").as("likes_flat"))
+test_set_exploded = test_set_exploded.select("post_id", "likes_flat.uid")
+
+// randomly move some (post_id, uid) from training set to test set
+sets = training_set_exploded.randomSplit(Array(1 - test_perc_stage2, test_perc_stage2), seed)
+
+training_set_exploded = sets(0)
+
+var additional_test_set_exploded = sets(1)
+test_set_exploded = test_set_exploded.unionAll(additional_test_set_exploded)
+
+// concatenate exploded set with null set
+val getNull = udf(() => None: Option[String])
+training_set_exploded = training_set_exploded.unionAll(training_set_null.select("post_id").withColumn("uid", getNull()))
+test_set_exploded = test_set_exploded.unionAll(test_set_null.select("post_id").withColumn("uid", getNull()))
+
+// Write to disk
+training_set_exploded.rdd.map(x => x(0) + "\t" + x(1)).saveAsTextFile(training_file_path)
+test_set_exploded.rdd.map(x => x(0) + "\t" + x(1)).saveAsTextFile(test_file_path)
+\ No newline at end of file
diff --git a/sample-apps/blog-recommendation/training_data_example.json b/sample-apps/blog-recommendation/training_data_example.json
new file mode 100644
index 00000000000..b25e4613fbe
--- /dev/null
+++ b/sample-apps/blog-recommendation/training_data_example.json
@@ -0,0 +1,247 @@
+[
+  {
+    "date_gmt": "2012-08-11 09:44:22",
+    "language": "en",
+    "author": "232",
+    "url": "https://joen.wordpress.com/?p=1339",
+    "title": "Selma and me",
+    "blog": "230",
+    "post_id": "522793",
+    "tags": [
+      "selma"
+    ],
+    "blogname": "Lens Cap",
+    "date": "2012-08-11 09:44:22",
+    "content": "<a href=\"http://joen.files.wordpress.com/2012/08/wpid-imag03021.jpg\"><img title=\"\" class=\"alignnone\" alt=\"image\" src=\"http://joen.files.wordpress.com/2012/08/wpid-imag0302.jpg\" /></a>",
+    "categories": [
+      "Photopool"
+    ],
+    "likes": [
+      {
+        "dt": "2012-08-11 14:26:51",
+        "uid": "5"
+      },
+      {
+        "dt": "2012-08-11 14:06:24",
+        "uid": "144"
+      },
+      {
+        "dt": "2012-08-11 14:02:42",
+        "uid": "414033"
+      },
+      {
+        "dt": "2012-08-11 10:56:57",
+        "uid": "6894686"
+      },
+      {
+        "dt": "2012-08-11 14:16:09",
+        "uid": "20416304"
+      }
+    ]
+  },
+  {
+    "date_gmt": "2012-07-17 16:05:35",
+    "language": "en",
+    "author": "322",
+    "url": "http://markjaquith.wordpress.com/?p=3894",
+    "title": "WP Help 1.0",
+    "blog": "316",
+    "post_id": "618834",
+    "tags": [],
+    "blogname": "Mark on WordPress",
+    "date": "2012-07-17 12:05:35",
+    "content": "My <a href=\"http://wordpress.org/extend/plugins/wp-help/\">WP Help</a> plugin just got a huge update. Version 1.0 is really worth checking out. I'm quite proud of it.\r\n\r\n<img src=\"http://markjaquith.files.wordpress.com/2012/07/screen-shot-2012-07-17-at-12-04-21-pm.png\" alt=\"\" title=\"Screen Shot 2012-07-17 at 12.04.21 PM\" width=\"584\" height=\"284\" class=\"aligncenter size-full wp-image-3896\" />\r\n\r\n<h3>WP Help</h3>\r\n\r\nWP Help is a plugin for creating documentation for display within the WordPress admin. Many WordPress installs are customized, and it's really helpful to have a centralized resource for documenting those features. You can create documents about creating content, editing content, moderating comments, or whatever you want! If you have clients who can't seem to remember how to do X, you should install WP Help and document it for them. WP Help is powered by WordPress Custom Post Types, so you create content using the full WordPress editor.\r\n\r\n<h3>Document Syncing</h3>\r\n\r\nOh yeah. This is the feature you've all been waiting for. If you have a standard set of help documents you want to use on multiple sites, this lets you do that. Create the documents, grab the (secret) sync URL for that site, and then plug that URL in to other sites. Those other sites will automatically pull down those documents, and keep them up-to-date (even handling new documents, deleted documents, renamed documents, and re-parented documents). Any internal links in the original document will be rewritten to be local to the destination WP Help install. So go ahead and use the WP internal linking functionality on your source site and know that those links will work on all the destination sites!\r\n\r\n<h3>Menu Placement</h3>\r\n\r\nThe menu item for the help documents can now be placed in one of four locations:\r\n\r\n<ol>\r\n<li>As a Dashboard submenu</li>\r\n<li>Top level, above the Dashboard</li>\r\n<li>Top level, below the Dashboard</li>\r\n<li>Top level, at the bottom</li>\r\n</ol>\r\n\r\nYou get a live preview of this (yeah, I know, super fancy).\r\n\r\n<h3>Menu Name</h3>\r\n\r\nThe menu name (and page title) can be changed. Just doubleclick the page title, edit it, and hit return. Boom. Again, this has a live preview.\r\n\r\n<h3>Topics List Name</h3>\r\n\r\nLikewise the topics list header can be renamed. Doubleclick, edit, return. Live preview.\r\n\r\n<h3>Edit Links &amp; Management Links</h3>\r\n\r\nIf you can edit help documents, you'll get an edit link. When editing, there is a handy \"Manage\" link to jump you back to the documents management interface. Navigating in general has been improved quite a bit.\r\n\r\n<h3>Dashboard Widget</h3>\r\n\r\nThere is now a simple dashboard widget, listing all of your help documents.\r\n\r\n<h3>Better Default Access</h3>\r\n\r\nA lot of you said that you have Contributor-level users who need documentation just like authors do. So now they can view documentation by default (there's a hook if you want to change the capability required to view help documents).\r\n\r\n<h3>Lots of Little Tweaks</h3>\r\n\r\nThere are numerous little tweaks to improve your experience. <a href=\"http://wordpress.org/extend/plugins/wp-help/\">Check out out</a>!\r\n\r\n<h3>Roadmap</h3>\r\n\r\nThings I'm considering:\r\n\r\n<ul>\r\n<li>Restricting individual documents to users with a certain level of access</li>\r\n<li>Multiple sync sources</li>\r\n</ul>\r\n\r\nAny other ideas?",
+    "categories": [
+      "WordPress"
+    ],
+    "likes": [
+      {
+        "dt": "2012-07-17 21:12:13",
+        "uid": "1234"
+      },
+      {
+        "dt": "2012-07-17 16:14:08",
+        "uid": "249119"
+      },
+      {
+        "dt": "2012-07-17 18:52:43",
+        "uid": "5818061"
+      },
+      {
+        "dt": "2012-07-20 18:00:27",
+        "uid": "10773334"
+      },
+      {
+        "dt": "2012-07-18 10:27:04",
+        "uid": "23406451"
+      },
+      {
+        "dt": "2012-07-17 18:50:47",
+        "uid": "37851431"
+      }
+    ]
+  },
+  {
+    "date_gmt": "2012-07-13 20:49:59",
+    "language": "en",
+    "author": "18342963",
+    "url": "http://justin.wordpress.com/?p=1214",
+    "title": "@ Café Meta",
+    "blog": "415",
+    "post_id": "1648262",
+    "tags": [
+      "checkin"
+    ],
+    "blogname": "Justin G. Shreve",
+    "date": "2012-07-13 16:49:59",
+    "content": "<p><a href=\"Café Meta\"><img src=\"https://is0.4sqi.net/pix/Nox6drHMgdWnt_7m8w1HaINMksVAtKXpLiD0Ocou-rg.jpg\"/></a></p>\n<p>Cheese filled pork rolls with mashed potatoes.</p>",
+    "categories": [
+      "foursquare"
+    ],
+    "likes": [
+      {
+        "dt": "2012-07-14 18:05:10",
+        "uid": "10916751"
+      }
+    ]
+  },
+  {
+    "date_gmt": "2012-08-03 23:55:23",
+    "language": "en",
+    "author": "18342963",
+    "url": "http://justin.wordpress.com/?p=1218",
+    "title": "@ El Campesino",
+    "blog": "415",
+    "post_id": "1194797",
+    "tags": [
+      "checkin"
+    ],
+    "blogname": "Justin G. Shreve",
+    "date": "2012-08-03 19:55:23",
+    "content": "<p><a href=\"http://4sq.com/7wXFrq\"><img src=\"https://is1.4sqi.net/pix/AwOlZW7UMMcfWhzioK7YCZIr-f2o1jJ-aTYTpxAuhHo.jpg\"/></a></p>\n<p>Celebrating a family birthday #burritofriday style.</p>",
+    "categories": [
+      "foursquare"
+    ],
+    "likes": []
+  },
+  {
+    "date_gmt": "2012-08-08 14:35:13",
+    "language": "en",
+    "author": "18342963",
+    "url": "http://justin.wordpress.com/?p=1220",
+    "title": "@ Wild Goats Cafe",
+    "blog": "415",
+    "post_id": "285252",
+    "tags": [
+      "checkin"
+    ],
+    "blogname": "Justin G. Shreve",
+    "date": "2012-08-08 10:35:13",
+    "content": "<p><a href=\"http://4sq.com/aDGp5P\"><img src=\"https://is0.4sqi.net/pix/eylAdOrAsEbe3T0W7345n8nRfoiFv6GuYd4xQ4nu4g0.jpg\"/></a></p>\n<p>Italian Omelet\n with Grits</p>",
+    "categories": [
+      "foursquare"
+    ],
+    "likes": []
+  },
+  {
+    "date_gmt": "2012-08-09 00:00:54",
+    "language": "en",
+    "author": "18342963",
+    "url": "http://justin.wordpress.com/?p=1222",
+    "title": "@ Oxford, OH",
+    "blog": "415",
+    "post_id": "250389",
+    "tags": [
+      "checkin"
+    ],
+    "blogname": "Justin G. Shreve",
+    "date": "2012-08-08 20:00:54",
+    "content": "<p><a href=\"http://4sq.com/95WcwY\"><img src=\"https://is1.4sqi.net/pix/M75ofin4EhckhAszHPHLMzhC4mYWELShAnJKI12Re_c.jpg\"/></a></p>\n<p>Welcome to new home meal</p>",
+    "categories": [
+      "foursquare"
+    ],
+    "likes": []
+  },
+  {
+    "date_gmt": "2012-08-09 19:16:40",
+    "language": "en",
+    "author": "18342963",
+    "url": "http://justin.wordpress.com/?p=1226",
+    "title": "@ IKEA",
+    "blog": "415",
+    "post_id": "1590141",
+    "tags": [
+      "checkin"
+    ],
+    "blogname": "Justin G. Shreve",
+    "date": "2012-08-09 15:16:40",
+    "content": "<p><a href=\"http://4sq.com/6IXrIh\"><img src=\"https://is0.4sqi.net/pix/FmrWa9WdnQi8q7LBd5iaI_oqRhBGyEHaxr4k_6qHETg.jpg\"/></a></p>\n<p></p>",
+    "categories": [
+      "foursquare"
+    ],
+    "likes": []
+  },
+  {
+    "date_gmt": "2012-08-11 17:00:36",
+    "language": "en",
+    "author": "18342963",
+    "url": "http://justin.wordpress.com/?p=1228",
+    "title": "@ Oxford, OH",
+    "blog": "415",
+    "post_id": "583639",
+    "tags": [
+      "checkin"
+    ],
+    "blogname": "Justin G. Shreve",
+    "date": "2012-08-11 13:00:36",
+    "content": "<p><a href=\"http://4sq.com/95WcwY\"><img src=\"https://is0.4sqi.net/pix/X6hSjNcr74mn5aWhEQ73M6Yzw2WFTcP9B4Y0bHrFrrw.jpg\"/></a></p>\n<p>Part of the property</p>",
+    "categories": [
+      "foursquare"
+    ],
+    "likes": []
+  },
+  {
+    "date_gmt": "2012-08-11 18:32:48",
+    "language": "en",
+    "author": "18342963",
+    "url": "http://justin.wordpress.com/?p=1230",
+    "title": "@ Phan-Shin",
+    "blog": "415",
+    "post_id": "1451283",
+    "tags": [
+      "checkin"
+    ],
+    "blogname": "Justin G. Shreve",
+    "date": "2012-08-11 14:32:48",
+    "content": "<p><a href=\"http://4sq.com/90TF72\"><img src=\"https://is1.4sqi.net/pix/TMuuSbyftmgRT98vbeMgVCcMSBnsZzg-PNGRP4EqQqY.jpg\"/></a></p>\n<p>Last meal before leaving. General Tso's</p>",
+    "categories": [
+      "foursquare"
+    ],
+    "likes": []
+  },
+  {
+    "date_gmt": "2012-07-12 17:03:49",
+    "language": "en",
+    "author": "486",
+    "url": "http://rick.wordpress.com/?p=1635",
+    "title": "On Baghdad by the Bay, July 12th at 8PM PDT; Interview with Dogcatcher and Wave Array",
+    "blog": "479",
+    "post_id": "1166238",
+    "tags": [
+      "bay area music",
+      "music"
+    ],
+    "blogname": "STET",
+    "date": "2012-07-12 10:03:49",
+    "content": "<strong>Stream the show archive: [audio http://rick.files.wordpress.com/2012/07/on-baghdad-by-the-bay-july-12th-interview-with-dogcatcher-and-wave-array.mp3] <a href=\"http://rick.files.wordpress.com/2012/07/on-baghdad-by-the-bay-july-12th-interview-with-dogcatcher-and-wave-array.mp3\">Or download the MP3</a>.</strong>\r\n\r\n<a href=\"http://dogcatcherband.com/\"><img src=\"http://rick.files.wordpress.com/2012/07/dogcatcher4panel.jpg?w=450\" alt=\"\" title=\"Dogcatcher4panel\" width=\"450\" height=\"193\" class=\"alignnone size-large wp-image-1636\" /></a>\r\n<strong>Dogcatcher</strong> \r\n<a href=\"http://dogcatcherband.com/\">dogcatcherband.com</a>\r\n8pm\r\nThere’s something magical that happens when four musicians come together from four completely different backgrounds. For Mountain View’s Dogcatcher, the result is a mellow, subtly-intense blend of soul, jazz, folk and indie-rock. Their first album, KILR, was a quiet folk-oriented solo album Heine wrote while living in San Diego and digesting the Iraq War he’d recently returned from. After KILR, Heine returned to his hometown, Mountain View, and formed a proper group, plucking musicians out of totally different bands to create an eclectic, genre-defying sound. On Dogcatcher’s second album, It’s Easy, the four members demonstrate that they have truly come into their own. Heine’s soft, hushed voice and simple-stated lyrics compliment the easy, hip-shaking nature of the music. But for listeners looking for lyrics with depth, Heine will not disappoint. Rather than aim for overt expressions of emotion, Heine always goes for the understated, which makes the emotionality behind the music that much more powerful with repeat listens. \r\n\r\n[youtube http://www.youtube.com/watch?v=ZbI_KdDK8-c?rel=0&amp;w=560&amp;h=315]\r\n<strong>Wave Array </strong>\r\n<a href=\"http://wavearraymusic.com/\">wavearraymusic.com</a>\r\n9:20\r\nWave Array is a four-piece rock band hailing from the East Bay/San Francisco, CA that is stretching the limits of alternative/indie rock into the psychedelic and experimental realms. Using vibrant guitar and bass work, driving rhythms, and palpable melodies, the quartet aims to resonate frequencies of human emotion and thought buried beneath the preoccupation of everyday life. Wave Array's eclectic compositions of melodic vamps and hard hitting jams have already sparked the interest of the Bay Area music community as well as local and nation-wide radio. Their self-produced debut album, Cheapjack Moon, is available on iTunes and Amazon.",
+    "categories": [
+      "Radio Valencia"
+    ],
+    "likes": [
+      {
+        "dt": "2012-07-12 18:27:02",
+        "uid": "34944702"
+      }
+    ]
+  }
+]
+\ No newline at end of file
diff --git a/sample-apps/boolean-search/README.md b/sample-apps/boolean-search/README.md
index a13f54b7fb5..6f7f427b8f9 100644
--- a/sample-apps/boolean-search/README.md
+++ b/sample-apps/boolean-search/README.md
@@ -16,5 +16,5 @@ type predicate to the .sd-file. (Remember to set the arity parameter.)
 
 2. **Search** using yql expressions, e.g. `select * from sources * where predicate(target, {"name":"Wile E. Coyote"},{});`
     ```sh
-curl "<endpoint url>/search/?query=sddocname:ad&yql=select%20*%20from%20sources%20*%20where%20predicate(target%2C%20%7B%22name%22%3A%22Wile%20E.%20Coyote%22%7D%2C%7B%7D)%3B"
+    curl "<endpoint url>/search/?query=sddocname:ad&yql=select%20*%20from%20sources%20*%20where%20predicate(target%2C%20%7B%22name%22%3A%22Wile%20E.%20Coyote%22%7D%2C%7B%7D)%3B"
     ```
diff --git a/searchcore/src/apps/fdispatch/.gitignore b/searchcore/src/apps/fdispatch/.gitignore
index 7ef8af69759..6cb8d53bdc1 100644
--- a/searchcore/src/apps/fdispatch/.gitignore
+++ b/searchcore/src/apps/fdispatch/.gitignore
@@ -1,2 +1 @@
-/fdispatch
-fdispatch-bin
+vespa-fdispatch-bin
diff --git a/searchcore/src/apps/fdispatch/CMakeLists.txt b/searchcore/src/apps/fdispatch/CMakeLists.txt
index 728f4e46883..63e1e6ca515 100644
--- a/searchcore/src/apps/fdispatch/CMakeLists.txt
+++ b/searchcore/src/apps/fdispatch/CMakeLists.txt
@@ -2,7 +2,7 @@
 vespa_add_executable(searchcore_fdispatch_app
     SOURCES
     fdispatch.cpp
-    OUTPUT_NAME fdispatch-bin
+    OUTPUT_NAME vespa-fdispatch-bin
     INSTALL sbin
     DEPENDS
     searchcore_fdispatch_program
diff --git a/searchcore/src/apps/proton/.gitignore b/searchcore/src/apps/proton/.gitignore
index 4c244afd919..c2cc445d918 100644
--- a/searchcore/src/apps/proton/.gitignore
+++ b/searchcore/src/apps/proton/.gitignore
@@ -1,4 +1,3 @@
 .depend
 Makefile
-proton
-proton-bin
+vespa-proton-bin
diff --git a/searchcore/src/apps/proton/CMakeLists.txt b/searchcore/src/apps/proton/CMakeLists.txt
index 13f04d2d6f4..c5398feed2e 100644
--- a/searchcore/src/apps/proton/CMakeLists.txt
+++ b/searchcore/src/apps/proton/CMakeLists.txt
@@ -3,7 +3,7 @@ vespa_add_executable(searchcore_proton_app
     SOURCES
     downpersistence.cpp
     proton.cpp
-    OUTPUT_NAME proton-bin
+    OUTPUT_NAME vespa-proton-bin
     INSTALL sbin
     DEPENDS
     searchcore_server
diff --git a/searchcore/src/apps/proton/proton.cpp b/searchcore/src/apps/proton/proton.cpp
index 98b4db85d36..57c4b62cddb 100644
--- a/searchcore/src/apps/proton/proton.cpp
+++ b/searchcore/src/apps/proton/proton.cpp
@@ -1,6 +1,7 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include <vespa/searchcore/proton/server/proton.h>
+#include <vespa/storage/storageserver/storagenode.h>
 #include <vespa/searchlib/util/statefile.h>
 #include <vespa/searchlib/util/sigbushandler.h>
 #include <vespa/searchlib/util/ioerrorhandler.h>
diff --git a/searchcore/src/apps/verify_ranksetup/.gitignore b/searchcore/src/apps/verify_ranksetup/.gitignore
index 269b313f1f0..d32b68f0f38 100644
--- a/searchcore/src/apps/verify_ranksetup/.gitignore
+++ b/searchcore/src/apps/verify_ranksetup/.gitignore
@@ -1,4 +1,3 @@
 .depend
 Makefile
-verify_ranksetup
-verify_ranksetup-bin
+vespa-verify-ranksetup-bin
diff --git a/searchcore/src/apps/verify_ranksetup/CMakeLists.txt b/searchcore/src/apps/verify_ranksetup/CMakeLists.txt
index de777f88182..7dd6d0490da 100644
--- a/searchcore/src/apps/verify_ranksetup/CMakeLists.txt
+++ b/searchcore/src/apps/verify_ranksetup/CMakeLists.txt
@@ -2,7 +2,7 @@
 vespa_add_executable(searchcore_verify_ranksetup_app
     SOURCES
     verify_ranksetup.cpp
-    OUTPUT_NAME verify_ranksetup-bin
+    OUTPUT_NAME vespa-verify-ranksetup-bin
     INSTALL bin
     DEPENDS
     searchcore_fconfig
diff --git a/searchcore/src/apps/vespa-gen-testdocs/vespa-gen-testdocs.cpp b/searchcore/src/apps/vespa-gen-testdocs/vespa-gen-testdocs.cpp
index 835c06db3cf..7966a705802 100644
--- a/searchcore/src/apps/vespa-gen-testdocs/vespa-gen-testdocs.cpp
+++ b/searchcore/src/apps/vespa-gen-testdocs/vespa-gen-testdocs.cpp
@@ -8,7 +8,7 @@
 #include <vespa/fastos/app.h>
 #include <iostream>
 #include <algorithm>
-#include <string>
+#include <sstream>
 #include <openssl/sha.h>
 
 #include <vespa/log/log.h>
diff --git a/searchcore/src/apps/vespa-proton-cmd/vespa-proton-cmd.cpp b/searchcore/src/apps/vespa-proton-cmd/vespa-proton-cmd.cpp
index bfbe27cb26c..58844dc969c 100644
--- a/searchcore/src/apps/vespa-proton-cmd/vespa-proton-cmd.cpp
+++ b/searchcore/src/apps/vespa-proton-cmd/vespa-proton-cmd.cpp
@@ -5,6 +5,7 @@
 #include <vespa/config/common/configsystem.h>
 #include <vespa/fnet/frt/frt.h>
 #include <vespa/vespalib/util/host_name.h>
+#include <vespa/vespalib/util/stringfmt.h>
 #include <vespa/fastos/app.h>
 #include <algorithm>
 #include <string>
diff --git a/searchcore/src/apps/vespa-transactionlog-inspect/vespa-transactionlog-inspect.cpp b/searchcore/src/apps/vespa-transactionlog-inspect/vespa-transactionlog-inspect.cpp
index 95c6d39c51a..0fdfb09ecef 100644
--- a/searchcore/src/apps/vespa-transactionlog-inspect/vespa-transactionlog-inspect.cpp
+++ b/searchcore/src/apps/vespa-transactionlog-inspect/vespa-transactionlog-inspect.cpp
@@ -8,6 +8,7 @@
 #include <vespa/searchlib/transactionlog/translogclient.h>
 #include <vespa/searchlib/transactionlog/translogserver.h>
 #include <vespa/vespalib/util/programoptions.h>
+#include <vespa/vespalib/util/xmlstream.h>
 #include <vespa/document/config/config-documenttypes.h>
 #include <iostream>
 #include <vespa/config/helper/configgetter.hpp>
diff --git a/searchcore/src/tests/proton/common/cachedselect_test.cpp b/searchcore/src/tests/proton/common/cachedselect_test.cpp
index fb5e40767ab..1117234f18f 100644
--- a/searchcore/src/tests/proton/common/cachedselect_test.cpp
+++ b/searchcore/src/tests/proton/common/cachedselect_test.cpp
@@ -23,6 +23,7 @@
 #include <vespa/document/fieldvalue/stringfieldvalue.h>
 #include <vespa/document/fieldvalue/intfieldvalue.h>
 #include <vespa/document/fieldvalue/document.h>
+#include <vespa/document/datatype/documenttype.h>
 
 #include <vespa/log/log.h>
 LOG_SETUP("cachedselect_test");
diff --git a/searchcore/src/tests/proton/common/selectpruner_test.cpp b/searchcore/src/tests/proton/common/selectpruner_test.cpp
index 4ecefde024c..2d6584bdc0c 100644
--- a/searchcore/src/tests/proton/common/selectpruner_test.cpp
+++ b/searchcore/src/tests/proton/common/selectpruner_test.cpp
@@ -3,9 +3,11 @@
 #include <vespa/vespalib/testkit/testapp.h>
 #include <vespa/document/repo/configbuilder.h>
 #include <vespa/document/repo/documenttyperepo.h>
+#include <vespa/document/datatype/documenttype.h>
 #include <vespa/searchcore/proton/common/selectpruner.h>
 #include <vespa/document/select/parser.h>
 #include <vespa/document/select/cloningvisitor.h>
+#include <vespa/document/fieldvalue/document.h>
 #include <vespa/searchlib/attribute/attributefactory.h>
 #include <vespa/searchlib/test/mock_attribute_manager.h>
 
@@ -92,7 +94,7 @@ rsString(const ResultSet &s)
 {
     std::ostringstream os;
     bool first = true;
-    uint32_t erange = Result::enumRange();
+    uint32_t erange = Result::enumRange;
     for (uint32_t e = 0; e < erange; ++e) {
         if (s.hasEnum(e)) {
             if (!first)
diff --git a/searchcore/src/tests/proton/docsummary/docsummary.cpp b/searchcore/src/tests/proton/docsummary/docsummary.cpp
index 5567eb970bb..6c36b004133 100644
--- a/searchcore/src/tests/proton/docsummary/docsummary.cpp
+++ b/searchcore/src/tests/proton/docsummary/docsummary.cpp
@@ -952,13 +952,13 @@ Test::requireThatUrisAreUsed()
                        startIndexField("urisingle").
                        startSubField("all").
                        addUrlTokenizedString(
-                               "http://www.yahoo.com:81/fluke?ab=2#4").
+                               "http://www.example.com:81/fluke?ab=2#4").
                        endSubField().
                        startSubField("scheme").
                        addUrlTokenizedString("http").
                        endSubField().
                        startSubField("host").
-                       addUrlTokenizedString("www.yahoo.com").
+                       addUrlTokenizedString("www.example.com").
                        endSubField().
                        startSubField("port").
                        addUrlTokenizedString("81").
@@ -977,13 +977,13 @@ Test::requireThatUrisAreUsed()
                        startElement(1).
                        startSubField("all").
                        addUrlTokenizedString(
-                               "http://www.yahoo.com:82/fluke?ab=2#8").
+                               "http://www.example.com:82/fluke?ab=2#8").
                        endSubField().
                        startSubField("scheme").
                        addUrlTokenizedString("http").
                        endSubField().
                        startSubField("host").
-                       addUrlTokenizedString("www.yahoo.com").
+                       addUrlTokenizedString("www.example.com").
                        endSubField().
                        startSubField("port").
                        addUrlTokenizedString("82").
@@ -1027,13 +1027,13 @@ Test::requireThatUrisAreUsed()
                        startElement(4).
                        startSubField("all").
                        addUrlTokenizedString(
-                               "http://www.yahoo.com:83/fluke?ab=2#12").
+                               "http://www.example.com:83/fluke?ab=2#12").
                        endSubField().
                        startSubField("scheme").
                        addUrlTokenizedString("http").
                        endSubField().
                        startSubField("host").
-                       addUrlTokenizedString("www.yahoo.com").
+                       addUrlTokenizedString("www.example.com").
                        endSubField().
                        startSubField("port").
                        addUrlTokenizedString("83").
@@ -1086,14 +1086,14 @@ Test::requireThatUrisAreUsed()
                              bc.createFieldCacheRepo(getResultConfig())->getFieldCache("class0"),
                              getMarkupFields());
 
-    EXPECT_TRUE(assertString("http://www.yahoo.com:81/fluke?ab=2#4",
+    EXPECT_TRUE(assertString("http://www.example.com:81/fluke?ab=2#4",
                             "urisingle", dsa, 1));
     GeneralResultPtr res = getResult(dsa, 1);
     {
         vespalib::Slime slime;
         decode(res->GetEntry("uriarray"), slime);
         EXPECT_TRUE(slime.get().valid());
-        EXPECT_EQUAL("http://www.yahoo.com:82/fluke?ab=2#8",  asVstring(slime.get()[0]));
+        EXPECT_EQUAL("http://www.example.com:82/fluke?ab=2#8",  asVstring(slime.get()[0]));
         EXPECT_EQUAL("http://www.flickr.com:82/fluke?ab=2#9", asVstring(slime.get()[1]));
     }
     {
@@ -1104,7 +1104,7 @@ Test::requireThatUrisAreUsed()
         EXPECT_EQUAL(7L, slime.get()[1]["weight"].asLong());
         vespalib::string arr0s = asVstring(slime.get()[0]["item"]);
         vespalib::string arr1s = asVstring(slime.get()[1]["item"]);
-        EXPECT_EQUAL("http://www.yahoo.com:83/fluke?ab=2#12", arr0s);
+        EXPECT_EQUAL("http://www.example.com:83/fluke?ab=2#12", arr0s);
         EXPECT_EQUAL("http://www.flickr.com:85/fluke?ab=2#13", arr1s);
     }
 }
diff --git a/searchcore/src/tests/proton/docsummary/summaryfieldconverter_test.cpp b/searchcore/src/tests/proton/docsummary/summaryfieldconverter_test.cpp
index 878833fa58b..b9b74527dca 100644
--- a/searchcore/src/tests/proton/docsummary/summaryfieldconverter_test.cpp
+++ b/searchcore/src/tests/proton/docsummary/summaryfieldconverter_test.cpp
@@ -369,9 +369,9 @@ Document Test::makeDocument() {
     doc.setValue("position2", LongFieldValue(ZCurve::encode(4, 2)));
 
     StructFieldValue uri(getDataType("url"));
-    uri.setValue("all", annotateTerm("http://www.yahoo.com:42/foobar?q#frag"));
+    uri.setValue("all", annotateTerm("http://www.example.com:42/foobar?q#frag"));
     uri.setValue("scheme", annotateTerm("http"));
-    uri.setValue("host", annotateTerm("www.yahoo.com"));
+    uri.setValue("host", annotateTerm("www.example.com"));
     uri.setValue("port", annotateTerm("42"));
     uri.setValue("path", annotateTerm("foobar"));
     uri.setValue("query", annotateTerm("q"));
@@ -379,10 +379,10 @@ Document Test::makeDocument() {
     doc.setValue("uri", uri);
 
     ArrayFieldValue uri_array(getDataType("Array<url>"));
-    uri.setValue("all", annotateTerm("http://www.yahoo.com:80/foobar?q#frag"));
+    uri.setValue("all", annotateTerm("http://www.example.com:80/foobar?q#frag"));
     uri.setValue("port", annotateTerm("80"));
     uri_array.add(uri);
-    uri.setValue("all", annotateTerm("https://www.yahoo.com:443/foo?q#frag"));
+    uri.setValue("all", annotateTerm("https://www.example.com:443/foo?q#frag"));
     uri.setValue("scheme", annotateTerm("https"));
     uri.setValue("path", annotateTerm("foo"));
     uri.setValue("port", annotateTerm("443"));
diff --git a/searchcore/src/tests/proton/document_iterator/document_iterator_test.cpp b/searchcore/src/tests/proton/document_iterator/document_iterator_test.cpp
index 0ccd5b8d6ee..8261ae80e02 100644
--- a/searchcore/src/tests/proton/document_iterator/document_iterator_test.cpp
+++ b/searchcore/src/tests/proton/document_iterator/document_iterator_test.cpp
@@ -2,6 +2,7 @@
 
 #include <vespa/document/fieldset/fieldsets.h>
 #include <vespa/document/fieldvalue/fieldvalues.h>
+#include <vespa/document/datatype/documenttype.h>
 #include <vespa/persistence/spi/bucket.h>
 #include <vespa/persistence/spi/docentry.h>
 #include <vespa/persistence/spi/result.h>
diff --git a/searchcore/src/tests/proton/documentdb/document_scan_iterator/document_scan_iterator_test.cpp b/searchcore/src/tests/proton/documentdb/document_scan_iterator/document_scan_iterator_test.cpp
index 6a2c283dc8d..db95898e349 100644
--- a/searchcore/src/tests/proton/documentdb/document_scan_iterator/document_scan_iterator_test.cpp
+++ b/searchcore/src/tests/proton/documentdb/document_scan_iterator/document_scan_iterator_test.cpp
@@ -4,6 +4,7 @@
 #include <vespa/vespalib/test/insertion_operators.h>
 #include <vespa/vespalib/testkit/testapp.h>
 #include <vespa/document/base/documentid.h>
+#include <vespa/vespalib/util/stringfmt.h>
 
 using namespace document;
 using namespace proton;
diff --git a/searchcore/src/tests/proton/documentdb/documentdb_test.cpp b/searchcore/src/tests/proton/documentdb/documentdb_test.cpp
index 5b550f9c49d..d757e48e7d0 100644
--- a/searchcore/src/tests/proton/documentdb/documentdb_test.cpp
+++ b/searchcore/src/tests/proton/documentdb/documentdb_test.cpp
@@ -1,10 +1,6 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include <tests/proton/common/dummydbowner.h>
-#include <vespa/document/repo/documenttyperepo.h>
-#include <vespa/messagebus/emptyreply.h>
-#include <vespa/messagebus/testlib/receptor.h>
-#include <vespa/searchcommon/common/schema.h>
 #include <vespa/searchcore/proton/attribute/flushableattribute.h>
 #include <vespa/searchcore/proton/common/feedtoken.h>
 #include <vespa/searchcore/proton/common/hw_info.h>
@@ -26,6 +22,9 @@
 #include <vespa/searchcorespi/index/indexflushtarget.h>
 #include <vespa/searchlib/index/dummyfileheadercontext.h>
 #include <vespa/searchlib/transactionlog/translogserver.h>
+#include <vespa/messagebus/emptyreply.h>
+#include <vespa/messagebus/testlib/receptor.h>
+#include <vespa/document/datatype/documenttype.h>
 #include <vespa/vespalib/data/slime/slime.h>
 #include <vespa/vespalib/testkit/test_kit.h>
 
diff --git a/searchcore/src/tests/proton/feedoperation/feedoperation_test.cpp b/searchcore/src/tests/proton/feedoperation/feedoperation_test.cpp
index 6077ca5fe76..1c8165d9f17 100644
--- a/searchcore/src/tests/proton/feedoperation/feedoperation_test.cpp
+++ b/searchcore/src/tests/proton/feedoperation/feedoperation_test.cpp
@@ -25,6 +25,7 @@
 #include <vespa/document/fieldvalue/fieldvalues.h>
 #include <vespa/document/repo/configbuilder.h>
 #include <vespa/document/repo/documenttyperepo.h>
+#include <vespa/document/datatype/documenttype.h>
 #include <vespa/vespalib/testkit/testapp.h>
 
 using document::BucketId;
diff --git a/searchcore/src/tests/proton/metrics/documentdb_job_trackers/documentdb_job_trackers_test.cpp b/searchcore/src/tests/proton/metrics/documentdb_job_trackers/documentdb_job_trackers_test.cpp
index 45931307c0e..816e214aa0c 100644
--- a/searchcore/src/tests/proton/metrics/documentdb_job_trackers/documentdb_job_trackers_test.cpp
+++ b/searchcore/src/tests/proton/metrics/documentdb_job_trackers/documentdb_job_trackers_test.cpp
@@ -1,12 +1,13 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-#include <vespa/log/log.h>
-LOG_SETUP("documentdb_job_trackers_test");
 
 #include <vespa/searchcore/proton/metrics/documentdb_job_trackers.h>
 #include <vespa/searchcore/proton/metrics/job_tracked_flush_target.h>
 #include <vespa/searchcore/proton/test/dummy_flush_target.h>
 #include <vespa/vespalib/testkit/testapp.h>
 
+#include <vespa/log/log.h>
+LOG_SETUP("documentdb_job_trackers_test");
+
 using namespace proton;
 using namespace searchcorespi;
 
@@ -61,7 +62,7 @@ TEST_F("require that job metrics are updated", Fixture)
     // Update metrics 2 times to ensure that all jobs are running
     // in the last interval we actually care about.
     f._trackers.updateMetrics(f._metrics);
-    FastOS_Thread::Sleep(100);
+    std::this_thread::sleep_for(std::chrono::milliseconds(100));
     f._trackers.updateMetrics(f._metrics);
 
     EXPECT_APPROX(1.0, f._metrics.attributeFlush.getLast(), EPS);
diff --git a/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp b/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp
index 19843cb9531..8dea47e850f 100644
--- a/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp
+++ b/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp
@@ -1,5 +1,6 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 #include <vespa/document/repo/documenttyperepo.h>
+#include <vespa/document/datatype/documenttype.h>
 #include <vespa/persistence/spi/documentselection.h>
 #include <vespa/searchcore/proton/persistenceengine/bucket_guard.h>
 #include <vespa/searchcore/proton/persistenceengine/ipersistenceengineowner.h>
diff --git a/searchcore/src/tests/proton/server/documentretriever_test.cpp b/searchcore/src/tests/proton/server/documentretriever_test.cpp
index bd2fe12e894..b0541f9ac71 100644
--- a/searchcore/src/tests/proton/server/documentretriever_test.cpp
+++ b/searchcore/src/tests/proton/server/documentretriever_test.cpp
@@ -5,6 +5,7 @@
 #include <vespa/document/bucket/bucketid.h>
 #include <vespa/document/datatype/datatype.h>
 #include <vespa/document/datatype/positiondatatype.h>
+#include <vespa/document/datatype/documenttype.h>
 #include <vespa/document/fieldvalue/arrayfieldvalue.h>
 #include <vespa/document/fieldvalue/document.h>
 #include <vespa/document/fieldvalue/doublefieldvalue.h>
@@ -29,11 +30,7 @@
 #include <vespa/searchlib/attribute/integerbase.h>
 #include <vespa/searchlib/attribute/predicate_attribute.h>
 #include <vespa/searchlib/attribute/stringbase.h>
-#include <vespa/searchlib/docstore/cachestats.h>
-#include <vespa/searchlib/docstore/idocumentstore.h>
-#include <vespa/vespalib/stllike/string.h>
 #include <vespa/vespalib/testkit/testapp.h>
-#include <vespa/vespalib/util/stringfmt.h>
 
 using document::ArrayFieldValue;
 using document::FieldValue;
diff --git a/searchcore/src/tests/proton/verify_ranksetup/verify_ranksetup_test.cpp b/searchcore/src/tests/proton/verify_ranksetup/verify_ranksetup_test.cpp
index 2c5e119ecbe..8439c2fb54f 100644
--- a/searchcore/src/tests/proton/verify_ranksetup/verify_ranksetup_test.cpp
+++ b/searchcore/src/tests/proton/verify_ranksetup/verify_ranksetup_test.cpp
@@ -9,7 +9,7 @@
 #include <map>
 #include <initializer_list>
 
-const char *prog = "../../../apps/verify_ranksetup/verify_ranksetup-bin";
+const char *prog = "../../../apps/verify_ranksetup/vespa-verify-ranksetup-bin";
 const std::string gen_dir("generated");
 
 const char *valid_feature = "value(0)";
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_directory.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_directory.cpp
index eb462ebebe2..9b2fc71702b 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_directory.cpp
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_directory.cpp
@@ -5,6 +5,7 @@
 #include <vespa/searchlib/util/filekit.h>
 #include <vespa/vespalib/io/fileutil.h>
 #include <vespa/vespalib/stllike/asciistream.h>
+#include <cassert>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".proton.attribute.attribute_directory");
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp
index 60cc292cbeb..a357fb92980 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp
@@ -5,6 +5,7 @@
 #include <vespa/searchcore/proton/common/attrupdate.h>
 #include <vespa/searchlib/attribute/attributevector.hpp>
 #include <vespa/searchlib/common/isequencedtaskexecutor.h>
+#include <vespa/document/datatype/documenttype.h>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".proton.server.attributeadapter");
diff --git a/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.cpp b/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.cpp
index d15eddb3ac4..10c2adfd1f2 100644
--- a/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.cpp
+++ b/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.cpp
@@ -3,6 +3,8 @@
 #include "attributefieldvaluenode.h"
 #include "selectcontext.h"
 #include <vespa/searchcommon/attribute/attributecontent.h>
+#include <vespa/searchlib/attribute/attributevector.h>
+
 
 namespace proton {
 
@@ -20,11 +22,10 @@ using search::attribute::BasicType;
 using search::attribute::CollectionType;
 using search::attribute::IAttributeVector;
 
-
 AttributeFieldValueNode::
 AttributeFieldValueNode(const vespalib::string& doctype,
                         const vespalib::string& field,
-                        const search::AttributeVector::SP &attribute)
+                        const std::shared_ptr<search::AttributeVector> &attribute)
     : FieldValueNode(doctype, field),
       _attribute(attribute)
 {
@@ -94,5 +95,11 @@ AttributeFieldValueNode::traceValue(const Context &context,
 }
 
 
+document::select::ValueNode::UP
+AttributeFieldValueNode::clone() const
+{
+    return wrapParens(new AttributeFieldValueNode(getDocType(), getFieldName(), _attribute));
+}
+
 } // namespace proton
 
diff --git a/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.h b/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.h
index 38e532053d4..c0ffbaea987 100644
--- a/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.h
+++ b/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.h
@@ -1,38 +1,24 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 #pragma once
 
-#include <vespa/document/select/valuenode.h>
-#include <vespa/searchlib/attribute/attributevector.h>
-
-namespace proton
-{
+#include <vespa/document/select/valuenodes.h>
 
+namespace search { class AttributeVector; }
+namespace proton {
 
 class AttributeFieldValueNode : public document::select::FieldValueNode
 {
-    search::AttributeVector::SP _attribute;
+    using Context = document::select::Context;
+    std::shared_ptr<search::AttributeVector> _attribute;
 
 public:
     AttributeFieldValueNode(const vespalib::string& doctype,
                             const vespalib::string& field,
-                            const search::AttributeVector::SP &attribute);
-
-    virtual std::unique_ptr<document::select::Value>
-    getValue(const document::select::Context &context) const override;
+                            const std::shared_ptr<search::AttributeVector> &attribute);
 
-    virtual std::unique_ptr<document::select::Value>
-    traceValue(const document::select::Context &context,
-               std::ostream& out) const override;
-
-    document::select::ValueNode::UP
-    clone() const override
-    {
-        return wrapParens(new AttributeFieldValueNode(getDocType(),
-                                                      getFieldName(),
-                                                      _attribute));
-    }
+    std::unique_ptr<document::select::Value> getValue(const Context &context) const override;
+    std::unique_ptr<document::select::Value> traceValue(const Context &context, std::ostream& out) const override;
+    document::select::ValueNode::UP clone() const override;
 };
 
 } // namespace proton
-
-
diff --git a/searchcore/src/vespa/searchcore/proton/common/attrupdate.cpp b/searchcore/src/vespa/searchcore/proton/common/attrupdate.cpp
index d54603fd898..51cb9a17b5b 100644
--- a/searchcore/src/vespa/searchcore/proton/common/attrupdate.cpp
+++ b/searchcore/src/vespa/searchcore/proton/common/attrupdate.cpp
@@ -20,6 +20,7 @@
 
 #include <vespa/searchlib/attribute/attributevector.hpp>
 #include <vespa/searchlib/attribute/changevector.hpp>
+#include <sstream>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".attrupdate");
diff --git a/searchcore/src/vespa/searchcore/proton/common/cachedselect.cpp b/searchcore/src/vespa/searchcore/proton/common/cachedselect.cpp
index 861131642e4..9a15c257dcc 100644
--- a/searchcore/src/vespa/searchcore/proton/common/cachedselect.cpp
+++ b/searchcore/src/vespa/searchcore/proton/common/cachedselect.cpp
@@ -1,12 +1,11 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "cachedselect.h"
-#include <vespa/document/select/valuenode.h>
-#include <vespa/document/select/cloningvisitor.h>
 #include "attributefieldvaluenode.h"
+#include "selectpruner.h"
 #include <vespa/searchlib/attribute/iattributemanager.h>
+#include <vespa/searchlib/attribute/attributevector.h>
 #include <vespa/document/select/parser.h>
-#include "selectpruner.h"
 
 #include <vespa/log/log.h>
 LOG_SETUP(".proton.common.cachedselect");
@@ -85,7 +84,7 @@ AttrVisitor::visitFieldValueNode(const FieldValueNode &expr)
             _valueNode = expr.clone();
             return;
         }
-        AttributeVector::SP av(ag->getSP());
+        std::shared_ptr<search::AttributeVector> av(ag->getSP());
         if (av->getCollectionType() == CollectionType::SINGLE) {
             ++_svAttrs;
             AttrMap::iterator it(_amap.find(name));
diff --git a/searchcore/src/vespa/searchcore/proton/common/selectpruner.cpp b/searchcore/src/vespa/searchcore/proton/common/selectpruner.cpp
index c166dfbe634..7e92be7507e 100644
--- a/searchcore/src/vespa/searchcore/proton/common/selectpruner.cpp
+++ b/searchcore/src/vespa/searchcore/proton/common/selectpruner.cpp
@@ -2,15 +2,14 @@
 
 #include "selectpruner.h"
 #include <vespa/document/base/exceptions.h>
-#include <vespa/document/base/fieldpath.h>
+#include <vespa/document/datatype/documenttype.h>
 #include <vespa/document/repo/documenttyperepo.h>
 #include <vespa/document/select/compare.h>
-#include <vespa/document/select/operator.h>
 #include <vespa/document/select/constant.h>
 #include <vespa/document/select/branch.h>
 #include <vespa/document/select/doctype.h>
 #include <vespa/document/select/invalidconstant.h>
-#include <vespa/document/select/constant.h>
+#include <vespa/document/select/valuenodes.h>
 #include <vespa/searchlib/attribute/iattributemanager.h>
 
 using document::select::And;
diff --git a/searchcore/src/vespa/searchcore/proton/common/selectpruner.h b/searchcore/src/vespa/searchcore/proton/common/selectpruner.h
index 5d4bbc038b9..7b6c9954124 100644
--- a/searchcore/src/vespa/searchcore/proton/common/selectpruner.h
+++ b/searchcore/src/vespa/searchcore/proton/common/selectpruner.h
@@ -9,8 +9,9 @@
 
 namespace search { class IAttributeManager; }
 
-namespace proton
-{
+namespace document { class DocumentTypeRepo; }
+
+namespace proton {
 
 class SelectPrunerBase
 {
@@ -50,100 +51,36 @@ public:
                  bool hasFields);
 
     SelectPruner(const SelectPruner *rhs);
-
-    virtual
-    ~SelectPruner();
-
-    uint32_t
-    getFieldNodes() const
-    {
-        return _fieldNodes;
-    }
-
-    uint32_t
-    getAttrFieldNodes() const
-    {
-        return _attrFieldNodes;
-    }
-
-    const document::select::ResultSet &
-    getResultSet() const
-    {
-        return _resultSet;
-    }
-
-    bool
-    isFalse() const;
-
-    bool
-    isTrue() const;
-
-    bool
-    isInvalid() const;
-
-    bool
-    isConst() const;
-
-    void
-    trace(std::ostream &t);
-
-    void
-    process(const document::select::Node &node);
+    virtual ~SelectPruner();
+
+    uint32_t getFieldNodes() const { return _fieldNodes; }
+    uint32_t getAttrFieldNodes() const { return _attrFieldNodes; }
+    const document::select::ResultSet & getResultSet() const { return _resultSet; }
+    bool isFalse() const;
+    bool isTrue() const;
+    bool isInvalid() const;
+    bool isConst() const;
+    void trace(std::ostream &t);
+    void process(const document::select::Node &node);
 private:
-    virtual void
-    visitAndBranch(const document::select::And &expr) override;
-
-    virtual void
-    visitComparison(const document::select::Compare &expr) override;
-
-    virtual void
-    visitDocumentType(const document::select::DocType &expr) override;
-
-    virtual void
-    visitNotBranch(const document::select::Not &expr) override;
-
-    virtual void
-    visitOrBranch(const document::select::Or &expr) override;
-
-    virtual void
-    visitArithmeticValueNode(const document::select::ArithmeticValueNode &expr) override;
-
-    virtual void
-    visitFunctionValueNode(const document::select::FunctionValueNode &expr) override;
-
-    virtual void
-    visitFieldValueNode(const document::select::FieldValueNode &expr) override;
-
-    void
-    invertNode();
-
-    const document::select::Operator &
-    getOperator(const document::select::Operator &op);
-
-    void
-    addNodeCount(const SelectPruner &rhs);
-
-    void
-    setInvalidVal();
-
-    void
-    setInvalidConst();
-
-    void
-    setTernaryConst(bool val);
-
-    void
-    resolveTernaryConst(bool wantInverted);
-
-    bool
-    isInvalidVal() const;
-
-    bool
-    isNullVal() const;
-
-    void
-    swap(SelectPruner &rhs);
+    void visitAndBranch(const document::select::And &expr) override;
+    void visitComparison(const document::select::Compare &expr) override;
+    void visitDocumentType(const document::select::DocType &expr) override;
+    void visitNotBranch(const document::select::Not &expr) override;
+    void visitOrBranch(const document::select::Or &expr) override;
+    void visitArithmeticValueNode(const document::select::ArithmeticValueNode &expr) override;
+    void visitFunctionValueNode(const document::select::FunctionValueNode &expr) override;
+    void visitFieldValueNode(const document::select::FieldValueNode &expr) override;
+    void invertNode();
+    const document::select::Operator &getOperator(const document::select::Operator &op);
+    void addNodeCount(const SelectPruner &rhs);
+    void setInvalidVal();
+    void setInvalidConst();
+    void setTernaryConst(bool val);
+    void resolveTernaryConst(bool wantInverted);
+    bool isInvalidVal() const;
+    bool isNullVal() const;
+    void swap(SelectPruner &rhs);
 };
 
 } // namespace proton
-
diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp
index 67a02c5d3ee..0253e943883 100644
--- a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp
+++ b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp
@@ -10,6 +10,7 @@
 #include <vespa/searchlib/docstore/logdocumentstore.h>
 #include <vespa/searchsummary/docsummary/docsumconfig.h>
 #include <vespa/vespalib/util/exceptions.h>
+#include <sstream>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".proton.docsummary.summarymanager");
diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/moveoperation.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/moveoperation.cpp
index 2a369cb821b..2cbfee1d5c9 100644
--- a/searchcore/src/vespa/searchcore/proton/feedoperation/moveoperation.cpp
+++ b/searchcore/src/vespa/searchcore/proton/feedoperation/moveoperation.cpp
@@ -1,8 +1,8 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "moveoperation.h"
-#include <vespa/vespalib/util/stringfmt.h>
 #include <vespa/document/fieldvalue/document.h>
+#include <cassert>
 
 using document::BucketId;
 using document::Document;
diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/pruneremoveddocumentsoperation.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/pruneremoveddocumentsoperation.cpp
index aa56daddeb2..1a18fee265f 100644
--- a/searchcore/src/vespa/searchcore/proton/feedoperation/pruneremoveddocumentsoperation.cpp
+++ b/searchcore/src/vespa/searchcore/proton/feedoperation/pruneremoveddocumentsoperation.cpp
@@ -2,6 +2,7 @@
 
 #include "pruneremoveddocumentsoperation.h"
 #include <vespa/vespalib/util/stringfmt.h>
+#include <cassert>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".proton.feedoperation.pruneremoveddocumentsoperation");
diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/prepare_restart_flush_strategy.cpp b/searchcore/src/vespa/searchcore/proton/flushengine/prepare_restart_flush_strategy.cpp
index ed029751271..2244759e866 100644
--- a/searchcore/src/vespa/searchcore/proton/flushengine/prepare_restart_flush_strategy.cpp
+++ b/searchcore/src/vespa/searchcore/proton/flushengine/prepare_restart_flush_strategy.cpp
@@ -3,6 +3,7 @@
 #include "prepare_restart_flush_strategy.h"
 #include "flush_target_candidates.h"
 #include "tls_stats_map.h"
+#include <sstream>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".proton.flushengine.prepare_restart_flush_strategy");
diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.cpp b/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.cpp
index 9df67a6bf11..79629410b00 100644
--- a/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.cpp
+++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.cpp
@@ -4,6 +4,7 @@
 #include <vespa/document/select/gid_filter.h>
 #include <vespa/document/select/node.h>
 #include <vespa/vespalib/objects/nbostream.h>
+#include <vespa/vespalib/stllike/hash_map.h>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".proton.persistenceengine.document_iterator");
diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/transport_latch.cpp b/searchcore/src/vespa/searchcore/proton/persistenceengine/transport_latch.cpp
index 78eb1cfb3f6..94bac320ef3 100644
--- a/searchcore/src/vespa/searchcore/proton/persistenceengine/transport_latch.cpp
+++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/transport_latch.cpp
@@ -1,6 +1,7 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "transport_latch.h"
+#include <vespa/vespalib/util/stringfmt.h>
 
 using storage::spi::Result;
 
diff --git a/searchcore/src/vespa/searchcore/proton/server/clusterstatehandler.cpp b/searchcore/src/vespa/searchcore/proton/server/clusterstatehandler.cpp
index 0cc6823f300..60f35934831 100644
--- a/searchcore/src/vespa/searchcore/proton/server/clusterstatehandler.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/clusterstatehandler.cpp
@@ -3,7 +3,7 @@
 #include "clusterstatehandler.h"
 #include "iclusterstatechangedhandler.h"
 #include <vespa/vespalib/util/closuretask.h>
-#include <algorithm>
+#include <sstream>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".proton.server.clusterstatehandler");
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentretriever.cpp b/searchcore/src/vespa/searchcore/proton/server/documentretriever.cpp
index a3566de66b9..e40d6480cc3 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentretriever.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/documentretriever.cpp
@@ -2,6 +2,7 @@
 
 #include "documentretriever.h"
 #include <vespa/document/datatype/positiondatatype.h>
+#include <vespa/document/datatype/documenttype.h>
 #include <vespa/document/repo/documenttyperepo.h>
 #include <vespa/searchcore/proton/attribute/document_field_retriever.h>
 #include <vespa/vespalib/geo/zcurve.h>
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentretrieverbase.cpp b/searchcore/src/vespa/searchcore/proton/server/documentretrieverbase.cpp
index 07a04905beb..63cfded1962 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentretrieverbase.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/documentretrieverbase.cpp
@@ -2,6 +2,7 @@
 
 #include "documentretrieverbase.h"
 #include <vespa/document/repo/documenttyperepo.h>
+#include <vespa/document/datatype/documenttype.h>
 #include <vespa/vespalib/stllike/lrucache_map.hpp>
 
 using document::DocumentId;
diff --git a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp
index 2f36f5140df..aa1541d766e 100644
--- a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp
@@ -12,15 +12,13 @@
 #include <vespa/documentapi/messagebus/messages/feedreply.h>
 #include <vespa/documentapi/messagebus/messages/removedocumentreply.h>
 #include <vespa/documentapi/messagebus/messages/updatedocumentreply.h>
-#include <vespa/searchcore/proton/common/bucketfactory.h>
-#include <vespa/searchcore/proton/feedoperation/moveoperation.h>
-#include <vespa/searchcore/proton/feedoperation/operations.h>
 #include <vespa/searchcore/proton/persistenceengine/transport_latch.h>
 #include <vespa/searchcore/proton/bucketdb/ibucketdbhandler.h>
 #include <vespa/searchcorespi/index/ithreadingservice.h>
 #include <vespa/vespalib/util/closuretask.h>
 #include <vespa/searchcore/proton/persistenceengine/i_resource_write_filter.h>
 #include <vespa/vespalib/util/exceptions.h>
+#include <vespa/document/datatype/documenttype.h>
 #include <vespa/log/log.h>
 LOG_SETUP(".proton.server.feedhandler");
 
diff --git a/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp
index 2b8209d9e4e..b444cd99952 100644
--- a/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp
@@ -10,9 +10,10 @@
 #include <vespa/config-summarymap.h>
 #include <vespa/config-rank-profiles.h>
 #include <vespa/searchsummary/config/config-juniperrc.h>
-#include <fstream>
 #include <vespa/fastos/file.h>
 #include <vespa/config/helper/configgetter.hpp>
+#include <fstream>
+#include <sstream>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".proton.server.fileconfigmanager");
diff --git a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.cpp b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.cpp
index bb45103a13f..3d655f3decf 100644
--- a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.cpp
@@ -1,13 +1,10 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "lid_space_compaction_job.h"
-#include "ifrozenbuckethandler.h"
 #include "imaintenancejobrunner.h"
 #include "i_disk_mem_usage_notifier.h"
 #include <vespa/searchcore/proton/common/eventlogger.h>
-
-#include <vespa/log/log.h>
-LOG_SETUP(".proton.server.lid_space_compaction_job");
+#include <cassert>
 
 using search::DocumentMetaData;
 using search::LidUsageStats;
@@ -76,9 +73,7 @@ LidSpaceCompactionJob::compactLidSpace(const LidUsageStats &stats)
     CompactLidSpaceOperation op(_handler.getSubDbId(), wantedLidLimit);
     _opStorer.storeOperation(op);
     _handler.handleCompactLidSpace(op);
-    if (LOG_WOULD_LOG(event)) {
-        EventLogger::lidSpaceCompactionComplete(_handler.getName(), wantedLidLimit);
-    }
+    EventLogger::lidSpaceCompactionComplete(_handler.getName(), wantedLidLimit);
     _shouldCompactLidSpace = false;
 }
 
diff --git a/searchcore/src/vespa/searchcore/proton/server/matchview.cpp b/searchcore/src/vespa/searchcore/proton/server/matchview.cpp
index 86ee2fe2ac0..c567930d68d 100644
--- a/searchcore/src/vespa/searchcore/proton/server/matchview.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/matchview.cpp
@@ -4,6 +4,7 @@
 #include "searchcontext.h"
 #include <vespa/searchlib/engine/searchrequest.h>
 #include <vespa/searchlib/engine/searchreply.h>
+#include <vespa/vespalib/util/stringfmt.h>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".proton.server.matchview");
diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp
index be8df97b683..271c127bcee 100644
--- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp
@@ -21,6 +21,7 @@
 #include <vespa/searchlib/util/fileheadertk.h>
 #include <vespa/searchcommon/common/schemaconfigurer.h>
 #include <vespa/document/base/exceptions.h>
+#include <vespa/document/datatype/documenttype.h>
 #include <vespa/vespalib/io/fileutil.h>
 #include <vespa/vespalib/util/closuretask.h>
 #include <vespa/vespalib/util/host_name.h>
diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp
index ac24469e4d2..e27a710022f 100644
--- a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp
@@ -2,23 +2,21 @@
 
 #include "ireplayconfig.h"
 #include "storeonlyfeedview.h"
-#include <vespa/searchcore/proton/common/bucketfactory.h>
-#include <vespa/searchcore/proton/common/commit_time_tracker.h>
-#include <vespa/searchcore/proton/common/feedtoken.h>
-#include <vespa/searchcore/proton/metrics/feed_metrics.h>
-#include <vespa/searchcore/proton/documentmetastore/ilidreusedelayer.h>
-#include <vespa/vespalib/stllike/string.h>
-#include <vespa/vespalib/text/stringtokenizer.h>
-#include <vespa/vespalib/util/closuretask.h>
-#include <vespa/searchcore/proton/feedoperation/moveoperation.h>
 #include "forcecommitcontext.h"
 #include "operationdonecontext.h"
 #include "removedonecontext.h"
 #include "updatedonecontext.h"
 #include "putdonecontext.h"
-#include <vespa/searchlib/common/lambdatask.h>
+#include <vespa/searchcore/proton/common/commit_time_tracker.h>
+#include <vespa/searchcore/proton/common/feedtoken.h>
+#include <vespa/searchcore/proton/metrics/feed_metrics.h>
+#include <vespa/searchcore/proton/documentmetastore/ilidreusedelayer.h>
 #include <vespa/searchlib/common/scheduletaskcallback.h>
+#include <vespa/document/datatype/documenttype.h>
 #include <vespa/vespalib/util/exceptions.h>
+#include <vespa/vespalib/text/stringtokenizer.h>
+#include <vespa/vespalib/util/closuretask.h>
+
 #include <vespa/log/log.h>
 LOG_SETUP(".proton.server.storeonlyfeedview");
 
diff --git a/searchcore/src/vespa/searchcore/proton/test/bucketdocuments.h b/searchcore/src/vespa/searchcore/proton/test/bucketdocuments.h
index b2fc3ff6dbc..f0d10d9ecbd 100644
--- a/searchcore/src/vespa/searchcore/proton/test/bucketdocuments.h
+++ b/searchcore/src/vespa/searchcore/proton/test/bucketdocuments.h
@@ -2,6 +2,7 @@
 #pragma once
 
 #include "document.h"
+#include <cassert>
 
 namespace proton {
 
diff --git a/searchcore/src/vespa/searchcore/proton/test/bucketstatecalculator.h b/searchcore/src/vespa/searchcore/proton/test/bucketstatecalculator.h
index 281c814fc77..4947c7500ed 100644
--- a/searchcore/src/vespa/searchcore/proton/test/bucketstatecalculator.h
+++ b/searchcore/src/vespa/searchcore/proton/test/bucketstatecalculator.h
@@ -2,11 +2,12 @@
 #pragma once
 
 #include <vespa/searchcore/proton/server/ibucketstatecalculator.h>
-#include <vespa/document/bucket/bucketid.h>
+#include <vespa/document/bucket/bucketidlist.h>
 
 namespace proton::test {
 
-typedef document::BucketId::List BucketIdVector;
+using BucketIdVector = document::bucket::BucketIdList;
+
 typedef std::set<document::BucketId>    BucketIdSet;
 
 class BucketStateCalculator : public IBucketStateCalculator
diff --git a/searchcore/src/vespa/searchcore/proton/test/userdocuments.h b/searchcore/src/vespa/searchcore/proton/test/userdocuments.h
index d1ea6f6ab4c..6b5a911352b 100644
--- a/searchcore/src/vespa/searchcore/proton/test/userdocuments.h
+++ b/searchcore/src/vespa/searchcore/proton/test/userdocuments.h
@@ -2,10 +2,9 @@
 #pragma once
 
 #include "bucketdocuments.h"
+#include <map>
 
-namespace proton {
-
-namespace test {
+namespace proton::test {
 
 /**
  * Collection of documents for a set of users,
@@ -48,7 +47,4 @@ public:
     void clear() { _docs.clear(); }
 };
 
-} // namespace test
-
-} // namespace proton
-
+}
diff --git a/searchlib/CMakeLists.txt b/searchlib/CMakeLists.txt
index c15ed681d8f..b9e0a8f8943 100644
--- a/searchlib/CMakeLists.txt
+++ b/searchlib/CMakeLists.txt
@@ -58,11 +58,10 @@ vespa_define_module(
 
     APPS
     src/apps/docstore
-    src/apps/expgolomb
-    src/apps/fileheaderinspect
-    src/apps/loadattribute
     src/apps/tests
     src/apps/uniform
+    src/apps/vespa-attribute-inspect
+    src/apps/vespa-fileheader-inspect
     src/apps/vespa-index-inspect
     src/apps/vespa-ranking-expression-analyzer
 
@@ -156,7 +155,6 @@ vespa_define_module(
     src/tests/fef/table
     src/tests/fef/termfieldmodel
     src/tests/fef/termmatchdatamerger
-    src/tests/fileheaderinspect
     src/tests/fileheadertk
     src/tests/forcelink
     src/tests/grouping
@@ -221,4 +219,5 @@ vespa_define_module(
     src/tests/util/slime_output_raw_buf_adapter
     src/tests/util/statebuf
     src/tests/util/statefile
+    src/tests/vespa-fileheader-inspect
 )
diff --git a/searchlib/src/apps/expgolomb/.gitignore b/searchlib/src/apps/expgolomb/.gitignore
deleted file mode 100644
index 0886ab154a2..00000000000
--- a/searchlib/src/apps/expgolomb/.gitignore
+++ /dev/null
@@ -1,3 +0,0 @@
-.depend
-Makefile
-expgolomb
diff --git a/searchlib/src/apps/expgolomb/expgolomb.cpp b/searchlib/src/apps/expgolomb/expgolomb.cpp
deleted file mode 100644
index 81aa566305c..00000000000
--- a/searchlib/src/apps/expgolomb/expgolomb.cpp
+++ /dev/null
@@ -1,168 +0,0 @@
-// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include <vespa/fastos/app.h>
-#include <vespa/searchlib/bitcompression/compression.h>
-#include <vector>
-
-class ExpGolombApp : public FastOS_Application
-{
-    void usage();
-    int testExpGolomb64(int kValue);
-    int testExpGolomb64le(int kValue);
-    int Main() override;
-};
-
-
-
-void
-ExpGolombApp::usage()
-{
-    printf("Usage: expgolomb testeg64 <kValue>]\n");
-    fflush(stdout);
-}
-
-
-int
-ExpGolombApp::testExpGolomb64(int kValue)
-{
-    std::vector<uint64_t> myrand;
-    for (int i = 0; i < 10000; ++i) {
-        uint64_t rval = rand();
-        rval <<= 30;
-        rval |= rand();
-        myrand.push_back(rval);
-    }
-    for (int i = 0; i < 10000; ++i) {
-        uint64_t rval = rand();
-        rval <<= 30;
-        rval |= rand();
-        uint32_t bits = (rand() & 63);
-        rval &= ((UINT64_C(1) << bits) - 1);
-        myrand.push_back(rval);
-    }
-    typedef search::bitcompression::EncodeContext64BE EC;
-
-    EC e;
-    search::ComprFileWriteContext wc(e);
-    wc.allocComprBuf(32768, 32768);
-    e.setupWrite(wc);
-
-    int rsize = myrand.size();
-    for (int i = 0; i < rsize; ++i) {
-        e.encodeExpGolomb(myrand[i], kValue);
-        if (e._valI >= e._valE)
-            wc.writeComprBuffer(false);
-    }
-    e.flush();
-
-    UC64_DECODECONTEXT(o);
-    unsigned int length;
-    uint64_t val64;
-    UC64BE_SETUPBITS_NS(o, static_cast<const uint64_t *>(wc._comprBuf), 0, EC);
-
-    bool failure = false;
-    for (int i = 0; i < rsize; ++i) {
-        UC64BE_DECODEEXPGOLOMB(oVal, oCompr, oPreRead, oCacheInt,
-                               kValue, EC);
-        if (val64 != myrand[i]) {
-            printf("FAILURE: TestExpGolomb64, val64=%"
-                   PRIu64 ", myrand[%d]=%" PRIu64 "\n",
-                   val64, i, myrand[i]);
-            failure = true;
-        }
-    }
-    if (!failure)
-        printf("SUCCESS: TestExpGolomb64\n");
-    return failure ? 1 : 0;
-}
-
-int
-ExpGolombApp::testExpGolomb64le(int kValue)
-{
-    std::vector<uint64_t> myrand;
-    for (int i = 0; i < 10000; ++i) {
-        uint64_t rval = rand();
-        rval <<= 30;
-        rval |= rand();
-        myrand.push_back(rval);
-    }
-    for (int i = 0; i < 10000; ++i) {
-        uint64_t rval = rand();
-        rval <<= 30;
-        rval |= rand();
-        uint32_t bits = (rand() & 63);
-        rval &= ((UINT64_C(1) << bits) - 1);
-        myrand.push_back(rval);
-    }
-    typedef search::bitcompression::EncodeContext64LE EC;
-
-    EC e;
-    search::ComprFileWriteContext wc(e);
-    wc.allocComprBuf(32768, 32768);
-    e.setupWrite(wc);
-
-    int rsize = myrand.size();
-    for (int i = 0; i < rsize; ++i) {
-        e.encodeExpGolomb(myrand[i], kValue);
-        if (e._valI >= e._valE)
-            wc.writeComprBuffer(false);
-    }
-    e.flush();
-
-    UC64_DECODECONTEXT(o);
-    unsigned int length;
-    uint64_t val64;
-    UC64LE_SETUPBITS_NS(o, static_cast<const uint64_t *>(wc._comprBuf), 0, EC);
-
-    bool failure = false;
-    for (int i = 0; i < rsize; ++i) {
-        UC64LE_DECODEEXPGOLOMB(oVal, oCompr, oPreRead, oCacheInt,
-                               kValue, EC);
-        if (val64 != myrand[i]) {
-            printf("FAILURE: TestExpGolomb64le, val64=%"
-                   PRIu64 ", myrand[%d]=%" PRIu64 "\n",
-                   val64, i, myrand[i]);
-            failure = true;
-        }
-    }
-    if (!failure)
-        printf("SUCCESS: TestExpGolomb64le\n");
-    return failure ? 1 : 0;
-}
-
-
-int
-ExpGolombApp::Main()
-{
-    printf("Hello world\n");
-    if (_argc >= 2) {
-        if (strcmp(_argv[1], "testeg64") == 0) {
-            if (_argc < 3) {
-                fprintf(stderr, "Too few arguments\n");
-                usage();
-                return 1;
-            }
-            return testExpGolomb64(atoi(_argv[2]));
-        } else if (strcmp(_argv[1], "testeg64le") == 0) {
-            if (_argc < 3) {
-                fprintf(stderr, "Too few arguments\n");
-                usage();
-                return 1;
-            }
-            return testExpGolomb64le(atoi(_argv[2]));
-        } else {
-            fprintf(stderr, "Wrong arguments\n");
-            usage();
-            return 1;
-        }
-    } else {
-        fprintf(stderr, "Too few arguments\n");
-        usage();
-        return 1;
-    }
-    return 0;
-}
-
-FASTOS_MAIN(ExpGolombApp);
-
-
diff --git a/searchlib/src/apps/fileheaderinspect/.gitignore b/searchlib/src/apps/fileheaderinspect/.gitignore
deleted file mode 100644
index 5616f8e735c..00000000000
--- a/searchlib/src/apps/fileheaderinspect/.gitignore
+++ /dev/null
@@ -1,3 +0,0 @@
-.depend
-Makefile
-vespa-header-inspect
diff --git a/searchlib/src/apps/loadattribute/.gitignore b/searchlib/src/apps/loadattribute/.gitignore
deleted file mode 100644
index 4f008fbf84e..00000000000
--- a/searchlib/src/apps/loadattribute/.gitignore
+++ /dev/null
@@ -1,3 +0,0 @@
-.depend
-Makefile
-loadattribute
diff --git a/searchlib/src/apps/loadattribute/CMakeLists.txt b/searchlib/src/apps/loadattribute/CMakeLists.txt
deleted file mode 100644
index 6712519e59a..00000000000
--- a/searchlib/src/apps/loadattribute/CMakeLists.txt
+++ /dev/null
@@ -1,9 +0,0 @@
-# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-vespa_add_executable(searchlib_loadattribute_app
-    SOURCES
-    loadattribute.cpp
-    OUTPUT_NAME loadattribute
-    INSTALL bin
-    DEPENDS
-    searchlib
-)
diff --git a/searchlib/src/apps/uniform/.gitignore b/searchlib/src/apps/uniform/.gitignore
index ff18dbaa7fd..d1fbbf62d24 100644
--- a/searchlib/src/apps/uniform/.gitignore
+++ b/searchlib/src/apps/uniform/.gitignore
@@ -1,3 +1,3 @@
 .depend
 Makefile
-uniform
+searchlib_uniform_app
diff --git a/searchlib/src/apps/uniform/CMakeLists.txt b/searchlib/src/apps/uniform/CMakeLists.txt
index 9f9c2139f42..7b835a64e8c 100644
--- a/searchlib/src/apps/uniform/CMakeLists.txt
+++ b/searchlib/src/apps/uniform/CMakeLists.txt
@@ -2,8 +2,6 @@
 vespa_add_executable(searchlib_uniform_app
     SOURCES
     uniform.cpp
-    OUTPUT_NAME uniform
-    INSTALL bin
     DEPENDS
     searchlib
 )
diff --git a/searchlib/src/apps/vespa-attribute-inspect/.gitignore b/searchlib/src/apps/vespa-attribute-inspect/.gitignore
new file mode 100644
index 00000000000..62bcd8ab443
--- /dev/null
+++ b/searchlib/src/apps/vespa-attribute-inspect/.gitignore
@@ -0,0 +1,3 @@
+.depend
+Makefile
+vespa-attribute-inspect
diff --git a/searchlib/src/apps/fileheaderinspect/CMakeLists.txt b/searchlib/src/apps/vespa-attribute-inspect/CMakeLists.txt
index 322bf6fefcb..4d170b9f325 100644
--- a/searchlib/src/apps/fileheaderinspect/CMakeLists.txt
+++ b/searchlib/src/apps/vespa-attribute-inspect/CMakeLists.txt
@@ -1,8 +1,8 @@
 # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-vespa_add_executable(searchlib_vespa-header-inspect_app
+vespa_add_executable(searchlib_vespa-attribute-inspect_app
     SOURCES
-    fileheaderinspect.cpp
-    OUTPUT_NAME vespa-header-inspect
+    vespa-attribute-inspect.cpp
+    OUTPUT_NAME vespa-attribute-inspect
     INSTALL bin
     DEPENDS
     searchlib
diff --git a/searchlib/src/apps/loadattribute/loadattribute.rb b/searchlib/src/apps/vespa-attribute-inspect/loadattribute.rb
index d1fb5a5632c..d1fb5a5632c 100644
--- a/searchlib/src/apps/loadattribute/loadattribute.rb
+++ b/searchlib/src/apps/vespa-attribute-inspect/loadattribute.rb
diff --git a/searchlib/src/apps/loadattribute/loadattribute.cpp b/searchlib/src/apps/vespa-attribute-inspect/vespa-attribute-inspect.cpp
index e82ab3ac645..189074aa5d3 100644
--- a/searchlib/src/apps/loadattribute/loadattribute.cpp
+++ b/searchlib/src/apps/vespa-attribute-inspect/vespa-attribute-inspect.cpp
@@ -98,7 +98,7 @@ LoadAttribute::printContent(const AttributePtr & ptr, std::ostream & os)
 void
 LoadAttribute::usage()
 {
-    std::cout << "usage: loadattribute [-p (print content to <attribute>.out)]" << std::endl;
+    std::cout << "usage: vespa-attribute-inspect [-p (print content to <attribute>.out)]" << std::endl;
     std::cout << "                     [-a (apply a single update)]" << std::endl;
     std::cout << "                     [-s (save attribute to <attribute>.save.dat)]" << std::endl;
     std::cout << "                     <attribute>" << std::endl;
diff --git a/searchlib/src/apps/vespa-fileheader-inspect/.gitignore b/searchlib/src/apps/vespa-fileheader-inspect/.gitignore
new file mode 100644
index 00000000000..5b53dd5e3ac
--- /dev/null
+++ b/searchlib/src/apps/vespa-fileheader-inspect/.gitignore
@@ -0,0 +1,3 @@
+.depend
+Makefile
+vespa-fileheader-inspect
diff --git a/searchlib/src/apps/expgolomb/CMakeLists.txt b/searchlib/src/apps/vespa-fileheader-inspect/CMakeLists.txt
index 230718907dd..cbbe8c05c2b 100644
--- a/searchlib/src/apps/expgolomb/CMakeLists.txt
+++ b/searchlib/src/apps/vespa-fileheader-inspect/CMakeLists.txt
@@ -1,8 +1,8 @@
 # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-vespa_add_executable(searchlib_expgolomb_app
+vespa_add_executable(searchlib_vespa-fileheader-inspect_app
     SOURCES
-    expgolomb.cpp
-    OUTPUT_NAME expgolomb
+    vespa-fileheader-inspect.cpp
+    OUTPUT_NAME vespa-fileheader-inspect
     INSTALL bin
     DEPENDS
     searchlib
diff --git a/searchlib/src/apps/fileheaderinspect/fileheaderinspect.cpp b/searchlib/src/apps/vespa-fileheader-inspect/vespa-fileheader-inspect.cpp
index 2407be0b4c2..02cbb230960 100644
--- a/searchlib/src/apps/fileheaderinspect/fileheaderinspect.cpp
+++ b/searchlib/src/apps/vespa-fileheader-inspect/vespa-fileheader-inspect.cpp
@@ -7,7 +7,7 @@
 #include <vespa/vespalib/data/fileheader.h>
 #include <vespa/vespalib/stllike/asciistream.h>
 #include <vespa/log/log.h>
-LOG_SETUP("fileheaderinspect");
+LOG_SETUP("vespa-fileheader-inspect");
 
 using namespace vespalib;
 
diff --git a/searchlib/src/tests/aggregator/perdocexpr.cpp b/searchlib/src/tests/aggregator/perdocexpr.cpp
index 6f374f1bea4..512252fd732 100644
--- a/searchlib/src/tests/aggregator/perdocexpr.cpp
+++ b/searchlib/src/tests/aggregator/perdocexpr.cpp
@@ -5,14 +5,13 @@
 #include <vespa/searchlib/attribute/extendableattributes.h>
 #include <vespa/vespalib/objects/objectdumper.h>
 #include <vespa/vespalib/testkit/testapp.h>
-#include <stdexcept>
 #include <vespa/document/base/testdocman.h>
 #include <vespa/vespalib/util/md5.h>
-#include <vespa/vespalib/util/stringfmt.h>
 #include <vespa/searchlib/expression/getdocidnamespacespecificfunctionnode.h>
 #include <vespa/searchlib/expression/documentfieldnode.h>
 #include <cmath>
 #include <iostream>
+#include <list>
 
 #define MU std::make_unique
 
diff --git a/searchlib/src/tests/diskindex/pagedict4/pagedict4test.cpp b/searchlib/src/tests/diskindex/pagedict4/pagedict4test.cpp
index a7e2d5738ce..b96b2b4ff05 100644
--- a/searchlib/src/tests/diskindex/pagedict4/pagedict4test.cpp
+++ b/searchlib/src/tests/diskindex/pagedict4/pagedict4test.cpp
@@ -12,6 +12,7 @@
 #include <vespa/searchlib/diskindex/pagedict4randread.h>
 #include <vespa/searchlib/common/tunefileinfo.h>
 #include <vespa/fastos/app.h>
+#include <sstream>
 
 #include <vespa/log/log.h>
 LOG_SETUP("pagedict4test");
diff --git a/searchlib/src/tests/docstore/document_store_visitor/document_store_visitor_test.cpp b/searchlib/src/tests/docstore/document_store_visitor/document_store_visitor_test.cpp
index 6c628b761b4..df1fffa2a0e 100644
--- a/searchlib/src/tests/docstore/document_store_visitor/document_store_visitor_test.cpp
+++ b/searchlib/src/tests/docstore/document_store_visitor/document_store_visitor_test.cpp
@@ -7,6 +7,7 @@
 #include <vespa/searchlib/index/dummyfileheadercontext.h>
 #include <vespa/searchlib/common/bitvector.h>
 #include <vespa/document/repo/documenttyperepo.h>
+#include <vespa/document/datatype/documenttype.h>
 #include <vespa/document/repo/configbuilder.h>
 #include <vespa/vespalib/io/fileutil.h>
 #include <vespa/vespalib/stllike/asciistream.h>
diff --git a/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp b/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp
index fc69cf12d74..46e7333ebb7 100644
--- a/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp
+++ b/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp
@@ -3,6 +3,7 @@
 #include <vespa/vespalib/testkit/test_kit.h>
 #include <vespa/document/repo/configbuilder.h>
 #include <vespa/document/repo/documenttyperepo.h>
+#include <vespa/document/datatype/documenttype.h>
 #include <vespa/searchlib/docstore/chunkformats.h>
 #include <vespa/searchlib/docstore/logdocumentstore.h>
 #include <vespa/searchlib/docstore/storebybucket.h>
diff --git a/searchlib/src/tests/engine/transportserver/transportserver_test.cpp b/searchlib/src/tests/engine/transportserver/transportserver_test.cpp
index b7932ea2381..d8fd8a52b0e 100644
--- a/searchlib/src/tests/engine/transportserver/transportserver_test.cpp
+++ b/searchlib/src/tests/engine/transportserver/transportserver_test.cpp
@@ -4,7 +4,7 @@
 #include <vespa/document/base/documentid.h>
 #include <vespa/searchlib/common/packets.h>
 #include <vespa/searchlib/engine/transportserver.h>
-
+#include <vespa/vespalib/util/stringfmt.h>
 #include <vespa/fnet/fnet.h>
 #include <vespa/searchlib/engine/errorcodes.h>
 #include <vespa/log/log.h>
diff --git a/searchlib/src/tests/features/element_completeness/element_completeness_test.cpp b/searchlib/src/tests/features/element_completeness/element_completeness_test.cpp
index 44077d06d15..16670c0ff30 100644
--- a/searchlib/src/tests/features/element_completeness/element_completeness_test.cpp
+++ b/searchlib/src/tests/features/element_completeness/element_completeness_test.cpp
@@ -7,6 +7,7 @@
 #include <vespa/searchlib/features/element_completeness_feature.h>
 #include <vespa/searchlib/fef/test/ftlib.h>
 #include <vespa/searchlib/fef/test/dummy_dependency_handler.h>
+#include <vespa/vespalib/util/stringfmt.h>
 
 using namespace search::fef;
 using namespace search::fef::test;
diff --git a/searchlib/src/tests/features/element_similarity_feature/element_similarity_feature_test.cpp b/searchlib/src/tests/features/element_similarity_feature/element_similarity_feature_test.cpp
index d693062bb00..f9e1fc902c8 100644
--- a/searchlib/src/tests/features/element_similarity_feature/element_similarity_feature_test.cpp
+++ b/searchlib/src/tests/features/element_similarity_feature/element_similarity_feature_test.cpp
@@ -8,6 +8,7 @@
 #include <vespa/searchlib/fef/test/ftlib.h>
 #include <initializer_list>
 #include <vespa/searchlib/fef/test/dummy_dependency_handler.h>
+#include <vespa/vespalib/util/stringfmt.h>
 
 using namespace search::fef;
 using namespace search::fef::test;
diff --git a/searchlib/src/tests/features/item_raw_score/item_raw_score_test.cpp b/searchlib/src/tests/features/item_raw_score/item_raw_score_test.cpp
index 51405064ac6..44f4fd46765 100644
--- a/searchlib/src/tests/features/item_raw_score/item_raw_score_test.cpp
+++ b/searchlib/src/tests/features/item_raw_score/item_raw_score_test.cpp
@@ -9,6 +9,7 @@
 #include <vespa/searchlib/fef/fef.h>
 #include <vespa/searchlib/fef/test/dummy_dependency_handler.h>
 #include <vespa/vespalib/stllike/asciistream.h>
+#include <vespa/vespalib/util/stringfmt.h>
 
 using search::feature_t;
 using namespace search::fef;
diff --git a/searchlib/src/tests/features/native_dot_product/native_dot_product_test.cpp b/searchlib/src/tests/features/native_dot_product/native_dot_product_test.cpp
index 93313c69a3f..a84e8f42847 100644
--- a/searchlib/src/tests/features/native_dot_product/native_dot_product_test.cpp
+++ b/searchlib/src/tests/features/native_dot_product/native_dot_product_test.cpp
@@ -8,6 +8,7 @@
 #include <vespa/searchlib/fef/fef.h>
 #include <vespa/searchlib/query/weight.h>
 #include <vespa/searchlib/fef/test/dummy_dependency_handler.h>
+#include <vespa/vespalib/util/stringfmt.h>
 
 using search::feature_t;
 using namespace search::fef;
diff --git a/searchlib/src/tests/features/prod_features_fieldtermmatch.cpp b/searchlib/src/tests/features/prod_features_fieldtermmatch.cpp
index 7ae7ba8b26a..3a3dba039ff 100644
--- a/searchlib/src/tests/features/prod_features_fieldtermmatch.cpp
+++ b/searchlib/src/tests/features/prod_features_fieldtermmatch.cpp
@@ -1,9 +1,10 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-#include <vespa/log/log.h>
-LOG_SETUP(".prod_features_fieldtermmatch");
 
 #include "prod_features.h"
 #include <vespa/searchlib/features/fieldtermmatchfeature.h>
+#include <vespa/vespalib/util/stringfmt.h>
+#include <vespa/log/log.h>
+LOG_SETUP(".prod_features_fieldtermmatch");
 
 using namespace search::features;
 using namespace search::fef;
diff --git a/searchlib/src/tests/features/raw_score/raw_score_test.cpp b/searchlib/src/tests/features/raw_score/raw_score_test.cpp
index 8b97a8edbcc..acf5066b2c3 100644
--- a/searchlib/src/tests/features/raw_score/raw_score_test.cpp
+++ b/searchlib/src/tests/features/raw_score/raw_score_test.cpp
@@ -7,6 +7,7 @@
 #include <vespa/searchlib/features/raw_score_feature.h>
 #include <vespa/searchlib/fef/fef.h>
 #include <vespa/searchlib/fef/test/dummy_dependency_handler.h>
+#include <vespa/vespalib/util/stringfmt.h>
 
 using search::feature_t;
 using namespace search::fef;
diff --git a/searchlib/src/tests/features/subqueries/subqueries_test.cpp b/searchlib/src/tests/features/subqueries/subqueries_test.cpp
index 233b3a65b9f..0dfe2858735 100644
--- a/searchlib/src/tests/features/subqueries/subqueries_test.cpp
+++ b/searchlib/src/tests/features/subqueries/subqueries_test.cpp
@@ -7,6 +7,7 @@
 #include <vespa/searchlib/features/subqueries_feature.h>
 #include <vespa/searchlib/fef/fef.h>
 #include <vespa/searchlib/fef/test/dummy_dependency_handler.h>
+#include <vespa/vespalib/util/stringfmt.h>
 
 using search::feature_t;
 using namespace search::fef;
diff --git a/searchlib/src/tests/features/text_similarity_feature/text_similarity_feature_test.cpp b/searchlib/src/tests/features/text_similarity_feature/text_similarity_feature_test.cpp
index 88b043cb2e8..bf5416291d7 100644
--- a/searchlib/src/tests/features/text_similarity_feature/text_similarity_feature_test.cpp
+++ b/searchlib/src/tests/features/text_similarity_feature/text_similarity_feature_test.cpp
@@ -8,6 +8,7 @@
 #include <vespa/searchlib/fef/test/ftlib.h>
 #include <initializer_list>
 #include <vespa/searchlib/fef/test/dummy_dependency_handler.h>
+#include <vespa/vespalib/util/stringfmt.h>
 
 using namespace search::fef;
 using namespace search::fef::test;
diff --git a/searchlib/src/tests/fef/properties/properties_test.cpp b/searchlib/src/tests/fef/properties/properties_test.cpp
index c7400d5815b..bd6818cf6e4 100644
--- a/searchlib/src/tests/fef/properties/properties_test.cpp
+++ b/searchlib/src/tests/fef/properties/properties_test.cpp
@@ -216,11 +216,15 @@ TEST("test stuff") {
     { // test index properties known by the framework
         { // vespa.eval.lazy_expressions
             EXPECT_EQUAL(eval::LazyExpressions::NAME, vespalib::string("vespa.eval.lazy_expressions"));
-            EXPECT_EQUAL(eval::LazyExpressions::DEFAULT_VALUE, vespalib::string("false"));
             Properties p;
-            EXPECT_TRUE(!eval::LazyExpressions::check(p));
-            p.add("vespa.eval.lazy_expressions", "true");
-            EXPECT_TRUE(eval::LazyExpressions::check(p));
+            EXPECT_TRUE(eval::LazyExpressions::check(p, true));
+            EXPECT_TRUE(!eval::LazyExpressions::check(p, false));
+            p = Properties().add("vespa.eval.lazy_expressions", "true");
+            EXPECT_TRUE(eval::LazyExpressions::check(p, true));
+            EXPECT_TRUE(eval::LazyExpressions::check(p, false));
+            p = Properties().add("vespa.eval.lazy_expressions", "false");
+            EXPECT_TRUE(!eval::LazyExpressions::check(p, true));
+            EXPECT_TRUE(!eval::LazyExpressions::check(p, false));
         }
         { // vespa.rank.firstphase
             EXPECT_EQUAL(rank::FirstPhase::NAME, vespalib::string("vespa.rank.firstphase"));
diff --git a/searchlib/src/tests/fileheaderinspect/CMakeLists.txt b/searchlib/src/tests/fileheaderinspect/CMakeLists.txt
deleted file mode 100644
index 4bc15f90ea7..00000000000
--- a/searchlib/src/tests/fileheaderinspect/CMakeLists.txt
+++ /dev/null
@@ -1,10 +0,0 @@
-# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-vespa_add_executable(searchlib_fileheaderinspect_test_app TEST
-    SOURCES
-    fileheaderinspect.cpp
-    DEPENDS
-    searchlib
-    AFTER
-    searchlib_vespa-header-inspect_app
-)
-vespa_add_test(NAME searchlib_fileheaderinspect_test_app COMMAND searchlib_fileheaderinspect_test_app)
diff --git a/searchlib/src/tests/index/docbuilder/docbuilder_test.cpp b/searchlib/src/tests/index/docbuilder/docbuilder_test.cpp
index a4402736f2d..f5eb6d385d9 100644
--- a/searchlib/src/tests/index/docbuilder/docbuilder_test.cpp
+++ b/searchlib/src/tests/index/docbuilder/docbuilder_test.cpp
@@ -124,13 +124,13 @@ Test::testBuilder()
             endField();
         b.startIndexField("iu").
             startSubField("all").
-            addUrlTokenizedString("http://www.yahoo.com:81/fluke?ab=2#4").
+            addUrlTokenizedString("http://www.example.com:81/fluke?ab=2#4").
             endSubField().
             startSubField("scheme").
             addUrlTokenizedString("http").
             endSubField().
             startSubField("host").
-            addUrlTokenizedString("www.yahoo.com").
+            addUrlTokenizedString("www.example.com").
             endSubField().
             startSubField("port").
             addUrlTokenizedString("81").
@@ -148,13 +148,13 @@ Test::testBuilder()
         b.startIndexField("iau").
             startElement(1).
             startSubField("all").
-            addUrlTokenizedString("http://www.yahoo.com:82/fluke?ab=2#8").
+            addUrlTokenizedString("http://www.example.com:82/fluke?ab=2#8").
             endSubField().
             startSubField("scheme").
             addUrlTokenizedString("http").
             endSubField().
             startSubField("host").
-            addUrlTokenizedString("www.yahoo.com").
+            addUrlTokenizedString("www.example.com").
             endSubField().
             startSubField("port").
             addUrlTokenizedString("82").
@@ -196,13 +196,13 @@ Test::testBuilder()
         b.startIndexField("iwu").
             startElement(4).
             startSubField("all").
-            addUrlTokenizedString("http://www.yahoo.com:83/fluke?ab=2#12").
+            addUrlTokenizedString("http://www.example.com:83/fluke?ab=2#12").
             endSubField().
             startSubField("scheme").
             addUrlTokenizedString("http").
             endSubField().
             startSubField("host").
-            addUrlTokenizedString("www.yahoo.com").
+            addUrlTokenizedString("www.example.com").
             endSubField().
             startSubField("port").
             addUrlTokenizedString("83").
@@ -309,8 +309,8 @@ Test::testBuilder()
         EXPECT_EQUAL("<item weight=\"1\">foo</item>", *itr++);
         EXPECT_EQUAL("</ic>", *itr++);
         EXPECT_EQUAL("<iu>", *itr++);
-        EXPECT_EQUAL("<all>http://www.yahoo.com:81/fluke?ab=2#4</all>", *itr++);
-        EXPECT_EQUAL("<host>www.yahoo.com</host>", *itr++);
+        EXPECT_EQUAL("<all>http://www.example.com:81/fluke?ab=2#4</all>", *itr++);
+        EXPECT_EQUAL("<host>www.example.com</host>", *itr++);
         EXPECT_EQUAL("<scheme>http</scheme>", *itr++);
         EXPECT_EQUAL("<path>/fluke</path>", *itr++);
         EXPECT_EQUAL("<port>81</port>", *itr++);
@@ -319,8 +319,8 @@ Test::testBuilder()
         EXPECT_EQUAL("</iu>", *itr++);
         EXPECT_EQUAL("<iau>", *itr++);
         EXPECT_EQUAL("<item>", *itr++);
-        EXPECT_EQUAL("<all>http://www.yahoo.com:82/fluke?ab=2#8</all>", *itr++);
-        EXPECT_EQUAL("<host>www.yahoo.com</host>", *itr++);
+        EXPECT_EQUAL("<all>http://www.example.com:82/fluke?ab=2#8</all>", *itr++);
+        EXPECT_EQUAL("<host>www.example.com</host>", *itr++);
         EXPECT_EQUAL("<scheme>http</scheme>", *itr++);
         EXPECT_EQUAL("<path>/fluke</path>", *itr++);
         EXPECT_EQUAL("<port>82</port>", *itr++);
@@ -339,8 +339,8 @@ Test::testBuilder()
         EXPECT_EQUAL("</iau>", *itr++);
         EXPECT_EQUAL("<iwu>", *itr++);
         EXPECT_EQUAL("<item weight=\"4\">", *itr++);
-        EXPECT_EQUAL("<all>http://www.yahoo.com:83/fluke?ab=2#12</all>", *itr++);
-        EXPECT_EQUAL("<host>www.yahoo.com</host>", *itr++);
+        EXPECT_EQUAL("<all>http://www.example.com:83/fluke?ab=2#12</all>", *itr++);
+        EXPECT_EQUAL("<host>www.example.com</host>", *itr++);
         EXPECT_EQUAL("<scheme>http</scheme>", *itr++);
         EXPECT_EQUAL("<path>/fluke</path>", *itr++);
         EXPECT_EQUAL("<port>83</port>", *itr++);
diff --git a/searchlib/src/tests/memoryindex/dictionary/dictionary_test.cpp b/searchlib/src/tests/memoryindex/dictionary/dictionary_test.cpp
index 8ebab883613..3dbf4efc368 100644
--- a/searchlib/src/tests/memoryindex/dictionary/dictionary_test.cpp
+++ b/searchlib/src/tests/memoryindex/dictionary/dictionary_test.cpp
@@ -1178,13 +1178,13 @@ TEST_F("requireThatUriIndexingIsWorking", DictionaryFixture<UriFixture>)
     f._b.startDocument("doc::10");
     f._b.startIndexField("iu").
         startSubField("all").
-        addUrlTokenizedString("http://www.yahoo.com:81/fluke?ab=2#4").
+        addUrlTokenizedString("http://www.example.com:81/fluke?ab=2#4").
         endSubField().
         startSubField("scheme").
         addUrlTokenizedString("http").
         endSubField().
         startSubField("host").
-        addUrlTokenizedString("www.yahoo.com").
+        addUrlTokenizedString("www.example.com").
         endSubField().
         startSubField("port").
         addUrlTokenizedString("81").
@@ -1202,13 +1202,13 @@ TEST_F("requireThatUriIndexingIsWorking", DictionaryFixture<UriFixture>)
     f._b.startIndexField("iau").
         startElement(1).
         startSubField("all").
-        addUrlTokenizedString("http://www.yahoo.com:82/fluke?ab=2#8").
+        addUrlTokenizedString("http://www.example.com:82/fluke?ab=2#8").
         endSubField().
         startSubField("scheme").
         addUrlTokenizedString("http").
         endSubField().
         startSubField("host").
-        addUrlTokenizedString("www.yahoo.com").
+        addUrlTokenizedString("www.example.com").
         endSubField().
         startSubField("port").
         addUrlTokenizedString("82").
@@ -1250,13 +1250,13 @@ TEST_F("requireThatUriIndexingIsWorking", DictionaryFixture<UriFixture>)
     f._b.startIndexField("iwu").
         startElement(4).
         startSubField("all").
-        addUrlTokenizedString("http://www.yahoo.com:83/fluke?ab=2#12").
+        addUrlTokenizedString("http://www.example.com:83/fluke?ab=2#12").
         endSubField().
         startSubField("scheme").
         addUrlTokenizedString("http").
         endSubField().
         startSubField("host").
-        addUrlTokenizedString("www.yahoo.com").
+        addUrlTokenizedString("www.example.com").
         endSubField().
         startSubField("port").
         addUrlTokenizedString("83").
@@ -1315,7 +1315,7 @@ TEST_F("requireThatUriIndexingIsWorking", DictionaryFixture<UriFixture>)
     }
     {
         uint32_t fieldId = f.getSchema().getIndexFieldId("iu");
-        PostingIterator itr(f._d.findFrozen("yahoo", fieldId),
+        PostingIterator itr(f._d.findFrozen("example", fieldId),
                             featureStoreRef(f._d, fieldId),
                             fieldId, matchData);
         itr.initFullRange();
@@ -1327,7 +1327,7 @@ TEST_F("requireThatUriIndexingIsWorking", DictionaryFixture<UriFixture>)
     }
     {
         uint32_t fieldId = f.getSchema().getIndexFieldId("iau");
-        PostingIterator itr(f._d.findFrozen("yahoo", fieldId),
+        PostingIterator itr(f._d.findFrozen("example", fieldId),
                             featureStoreRef(f._d, fieldId),
                             fieldId, matchData);
         itr.initFullRange();
@@ -1340,7 +1340,7 @@ TEST_F("requireThatUriIndexingIsWorking", DictionaryFixture<UriFixture>)
     }
     {
         uint32_t fieldId = f.getSchema().getIndexFieldId("iwu");
-        PostingIterator itr(f._d.findFrozen("yahoo", fieldId),
+        PostingIterator itr(f._d.findFrozen("example", fieldId),
                             featureStoreRef(f._d, fieldId),
                             fieldId, matchData);
         itr.initFullRange();
diff --git a/searchlib/src/tests/memoryindex/urlfieldinverter/urlfieldinverter_test.cpp b/searchlib/src/tests/memoryindex/urlfieldinverter/urlfieldinverter_test.cpp
index 233c95b58c8..5011310f412 100644
--- a/searchlib/src/tests/memoryindex/urlfieldinverter/urlfieldinverter_test.cpp
+++ b/searchlib/src/tests/memoryindex/urlfieldinverter/urlfieldinverter_test.cpp
@@ -30,13 +30,13 @@ makeDoc10Single(DocBuilder &b)
     b.startDocument("doc::10");
     b.startIndexField("url").
         startSubField("all").
-        addUrlTokenizedString("http://www.yahoo.com:81/fluke?ab=2#4").
+        addUrlTokenizedString("http://www.example.com:81/fluke?ab=2#4").
         endSubField().
         startSubField("scheme").
         addUrlTokenizedString("http").
         endSubField().
         startSubField("host").
-        addUrlTokenizedString("www.yahoo.com").
+        addUrlTokenizedString("www.example.com").
         endSubField().
         startSubField("port").
         addUrlTokenizedString("81").
@@ -63,13 +63,13 @@ makeDoc10Array(DocBuilder &b)
     b.startIndexField("url").
         startElement(1).
         startSubField("all").
-        addUrlTokenizedString("http://www.yahoo.com:82/fluke?ab=2#8").
+        addUrlTokenizedString("http://www.example.com:82/fluke?ab=2#8").
         endSubField().
         startSubField("scheme").
         addUrlTokenizedString("http").
         endSubField().
         startSubField("host").
-        addUrlTokenizedString("www.yahoo.com").
+        addUrlTokenizedString("www.example.com").
         endSubField().
         startSubField("port").
         addUrlTokenizedString("82").
@@ -119,13 +119,13 @@ makeDoc10WeightedSet(DocBuilder &b)
     b.startIndexField("url").
         startElement(4).
         startSubField("all").
-        addUrlTokenizedString("http://www.yahoo.com:83/fluke?ab=2#12").
+        addUrlTokenizedString("http://www.example.com:83/fluke?ab=2#12").
         endSubField().
         startSubField("scheme").
         addUrlTokenizedString("http").
         endSubField().
         startSubField("host").
-        addUrlTokenizedString("www.yahoo.com").
+        addUrlTokenizedString("www.example.com").
         endSubField().
         startSubField("port").
         addUrlTokenizedString("83").
@@ -258,16 +258,16 @@ TEST_F("requireThatSingleUrlFieldWorks", Fixture(CollectionType::SINGLE))
                  "w=81,a=10,"
                  "w=ab,a=10,"
                  "w=com,a=10,"
+                 "w=example,a=10,"
                  "w=fluke,a=10,"
                  "w=http,a=10,"
                  "w=www,a=10,"
-                 "w=yahoo,a=10,"
                  "f=1,"
                  "w=http,a=10,"
                  "f=2,"
                  "w=com,a=10,"
+                 "w=example,a=10,"
                  "w=www,a=10,"
-                 "w=yahoo,a=10,"
                  "f=3,"
                  "w=81,a=10,"
                  "f=4,"
@@ -281,8 +281,8 @@ TEST_F("requireThatSingleUrlFieldWorks", Fixture(CollectionType::SINGLE))
                  "w=EnDhOsT,a=10,"
                  "w=StArThOsT,a=10,"
                  "w=com,a=10,"
-                 "w=www,a=10,"
-                 "w=yahoo,a=10",
+                 "w=example,a=10,"
+                 "w=www,a=10",
                  f._inserter.toStr());
 }
 
@@ -298,18 +298,18 @@ TEST_F("requireThatArrayUrlFieldWorks", Fixture(CollectionType::ARRAY))
                  "w=9,a=10,"
                  "w=ab,a=10,"
                  "w=com,a=10,"
+                 "w=example,a=10,"
                  "w=flickr,a=10,"
                  "w=fluke,a=10,"
                  "w=http,a=10,"
                  "w=www,a=10,"
-                 "w=yahoo,a=10,"
                  "f=1,"
                  "w=http,a=10,"
                  "f=2,"
                  "w=com,a=10,"
+                 "w=example,a=10,"
                  "w=flickr,a=10,"
                  "w=www,a=10,"
-                 "w=yahoo,a=10,"
                  "f=3,"
                  "w=82,a=10,"
                  "f=4,"
@@ -324,9 +324,9 @@ TEST_F("requireThatArrayUrlFieldWorks", Fixture(CollectionType::ARRAY))
                  "w=EnDhOsT,a=10,"
                  "w=StArThOsT,a=10,"
                  "w=com,a=10,"
+                 "w=example,a=10,"
                  "w=flickr,a=10,"
-                 "w=www,a=10,"
-                 "w=yahoo,a=10",
+                 "w=www,a=10",
                  f._inserter.toStr());
 }
 
@@ -342,18 +342,18 @@ TEST_F("requireThatWeightedSetFieldWorks", Fixture(CollectionType::WEIGHTEDSET))
                  "w=85,a=10,"
                  "w=ab,a=10,"
                  "w=com,a=10,"
+                 "w=example,a=10,"
                  "w=flickr,a=10,"
                  "w=fluke,a=10,"
                  "w=http,a=10,"
                  "w=www,a=10,"
-                 "w=yahoo,a=10,"
                  "f=1,"
                  "w=http,a=10,"
                  "f=2,"
                  "w=com,a=10,"
+                 "w=example,a=10,"
                  "w=flickr,a=10,"
                  "w=www,a=10,"
-                 "w=yahoo,a=10,"
                  "f=3,"
                  "w=83,a=10,"
                  "w=85,a=10,"
@@ -369,9 +369,9 @@ TEST_F("requireThatWeightedSetFieldWorks", Fixture(CollectionType::WEIGHTEDSET))
                  "w=EnDhOsT,a=10,"
                  "w=StArThOsT,a=10,"
                  "w=com,a=10,"
+                 "w=example,a=10,"
                  "w=flickr,a=10,"
-                 "w=www,a=10,"
-                 "w=yahoo,a=10",
+                 "w=www,a=10",
                  f._inserter.toStr());
 }
 
@@ -386,16 +386,16 @@ TEST_F("requireThatAnnotatedSingleUrlFieldWorks", Fixture(CollectionType::SINGLE
                  "w=81,a=10,"
                  "w=ab,a=10,"
                  "w=com,a=10,"
+                 "w=example,a=10,"
                  "w=fluke,a=10,"
                  "w=http,a=10,"
                  "w=www,a=10,"
-                 "w=yahoo,a=10,"
                  "f=1,"
                  "w=http,a=10,"
                  "f=2,"
                  "w=com,a=10,"
+                 "w=example,a=10,"
                  "w=www,a=10,"
-                 "w=yahoo,a=10,"
                  "f=3,"
                  "w=81,a=10,"
                  "f=4,"
@@ -410,8 +410,8 @@ TEST_F("requireThatAnnotatedSingleUrlFieldWorks", Fixture(CollectionType::SINGLE
                  "w=EnDhOsT,a=10,"
                  "w=StArThOsT,a=10,"
                  "w=com,a=10,"
-                 "w=www,a=10,"
-                 "w=yahoo,a=10",
+                 "w=example,a=10,"
+                 "w=www,a=10",
                  f._inserter.toStr());
 }
 
@@ -428,18 +428,18 @@ TEST_F("requireThatAnnotatedArrayUrlFieldWorks", Fixture(CollectionType::ARRAY))
                  "w=9,a=10,"
                  "w=ab,a=10,"
                  "w=com,a=10,"
+                 "w=example,a=10,"
                  "w=flickr,a=10,"
                  "w=fluke,a=10,"
                  "w=http,a=10,"
                  "w=www,a=10,"
-                 "w=yahoo,a=10,"
                  "f=1,"
                  "w=http,a=10,"
                  "f=2,"
                  "w=com,a=10,"
+                 "w=example,a=10,"
                  "w=flickr,a=10,"
                  "w=www,a=10,"
-                 "w=yahoo,a=10,"
                  "f=3,"
                  "w=82,a=10,"
                  "f=4,"
@@ -455,9 +455,9 @@ TEST_F("requireThatAnnotatedArrayUrlFieldWorks", Fixture(CollectionType::ARRAY))
                  "w=EnDhOsT,a=10,"
                  "w=StArThOsT,a=10,"
                  "w=com,a=10,"
+                 "w=example,a=10,"
                  "w=flickr,a=10,"
-                 "w=www,a=10,"
-                 "w=yahoo,a=10",
+                 "w=www,a=10",
                  f._inserter.toStr());
 }
 
@@ -476,18 +476,18 @@ TEST_F("requireThatAnnotatedWeightedSetFieldWorks",
                  "w=85,a=10(e=1,w=7,l=9[4]),"
                  "w=ab,a=10(e=0,w=4,l=9[6],e=1,w=7,l=9[6]),"
                  "w=com,a=10(e=0,w=4,l=9[3],e=1,w=7,l=9[3]),"
+                 "w=example,a=10(e=0,w=4,l=9[2]),"
                  "w=flickr,a=10(e=1,w=7,l=9[2]),"
                  "w=fluke,a=10(e=0,w=4,l=9[5],e=1,w=7,l=9[5]),"
                  "w=http,a=10(e=0,w=4,l=9[0],e=1,w=7,l=9[0]),"
                  "w=www,a=10(e=0,w=4,l=9[1],e=1,w=7,l=9[1]),"
-                 "w=yahoo,a=10(e=0,w=4,l=9[2]),"
                  "f=1,"
                  "w=http,a=10(e=0,w=4,l=1[0],e=1,w=7,l=1[0]),"
                  "f=2,"
                  "w=com,a=10(e=0,w=4,l=3[2],e=1,w=7,l=3[2]),"
+                 "w=example,a=10(e=0,w=4,l=3[1]),"
                  "w=flickr,a=10(e=1,w=7,l=3[1]),"
                  "w=www,a=10(e=0,w=4,l=3[0],e=1,w=7,l=3[0]),"
-                 "w=yahoo,a=10(e=0,w=4,l=3[1]),"
                  "f=3,"
                  "w=83,a=10(e=0,w=4,l=1[0]),"
                  "w=85,a=10(e=1,w=7,l=1[0]),"
@@ -504,9 +504,9 @@ TEST_F("requireThatAnnotatedWeightedSetFieldWorks",
                  "w=EnDhOsT,a=10(e=0,w=4,l=5[4],e=1,w=7,l=5[4]),"
                  "w=StArThOsT,a=10(e=0,w=4,l=5[0],e=1,w=7,l=5[0]),"
                  "w=com,a=10(e=0,w=4,l=5[3],e=1,w=7,l=5[3]),"
+                 "w=example,a=10(e=0,w=4,l=5[2]),"
                  "w=flickr,a=10(e=1,w=7,l=5[2]),"
-                 "w=www,a=10(e=0,w=4,l=5[1],e=1,w=7,l=5[1]),"
-                 "w=yahoo,a=10(e=0,w=4,l=5[2])",
+                 "w=www,a=10(e=0,w=4,l=5[1],e=1,w=7,l=5[1])",
                  f._inserter.toStr());
 }
 
diff --git a/searchlib/src/tests/transactionlogstress/translogstress.cpp b/searchlib/src/tests/transactionlogstress/translogstress.cpp
index 3719c5da1c7..b4285da447b 100644
--- a/searchlib/src/tests/transactionlogstress/translogstress.cpp
+++ b/searchlib/src/tests/transactionlogstress/translogstress.cpp
@@ -9,6 +9,7 @@
 #include <vespa/fastos/app.h>
 #include <iostream>
 #include <stdexcept>
+#include <sstream>
 
 #include <vespa/log/log.h>
 LOG_SETUP("translogstress");
diff --git a/searchlib/src/tests/fileheaderinspect/.gitignore b/searchlib/src/tests/vespa-fileheader-inspect/.gitignore
index 812991d07b5..4ff47722dca 100644
--- a/searchlib/src/tests/fileheaderinspect/.gitignore
+++ b/searchlib/src/tests/vespa-fileheader-inspect/.gitignore
@@ -3,4 +3,4 @@ Makefile
 fileheader.dat
 fileheaderinspect_test
 out
-searchlib_fileheaderinspect_test_app
+searchlib_vespa-fileheader-inspect_test_app
diff --git a/searchlib/src/tests/vespa-fileheader-inspect/CMakeLists.txt b/searchlib/src/tests/vespa-fileheader-inspect/CMakeLists.txt
new file mode 100644
index 00000000000..14bf5a82e3f
--- /dev/null
+++ b/searchlib/src/tests/vespa-fileheader-inspect/CMakeLists.txt
@@ -0,0 +1,10 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchlib_vespa-fileheader-inspect_test_app TEST
+    SOURCES
+    vespa-fileheader-inspect_test.cpp
+    DEPENDS
+    searchlib
+    AFTER
+    searchlib_vespa-fileheader-inspect_app
+)
+vespa_add_test(NAME searchlib_vespa-fileheader-inspect_test_app COMMAND searchlib_vespa-fileheader-inspect_test_app)
diff --git a/searchlib/src/tests/fileheaderinspect/DESC b/searchlib/src/tests/vespa-fileheader-inspect/DESC
index ee57a2fdde3..ee57a2fdde3 100644
--- a/searchlib/src/tests/fileheaderinspect/DESC
+++ b/searchlib/src/tests/vespa-fileheader-inspect/DESC
diff --git a/searchlib/src/tests/fileheaderinspect/FILES b/searchlib/src/tests/vespa-fileheader-inspect/FILES
index 7c32fb811d5..7c32fb811d5 100644
--- a/searchlib/src/tests/fileheaderinspect/FILES
+++ b/searchlib/src/tests/vespa-fileheader-inspect/FILES
diff --git a/searchlib/src/tests/fileheaderinspect/fileheaderinspect.cpp b/searchlib/src/tests/vespa-fileheader-inspect/vespa-fileheader-inspect_test.cpp
index 68bae5e57d8..b54b06e4d93 100644
--- a/searchlib/src/tests/fileheaderinspect/fileheaderinspect.cpp
+++ b/searchlib/src/tests/vespa-fileheader-inspect/vespa-fileheader-inspect_test.cpp
@@ -34,14 +34,14 @@ vespalib::string readFile(const vespalib::string &fileName) {
 }
 
 TEST("testError") {
-    EXPECT_TRUE(system("../../apps/fileheaderinspect/vespa-header-inspect notfound.dat") != 0);
+    EXPECT_TRUE(system("../../apps/vespa-fileheader-inspect/vespa-fileheader-inspect notfound.dat") != 0);
 }
 
 TEST("testEscape") {
     FileHeader header;
     header.putTag(FileHeader::Tag("fanart", "\fa\na\r\t"));
     ASSERT_TRUE(writeHeader(header, "fileheader.dat"));
-    EXPECT_TRUE(system("../../apps/fileheaderinspect/vespa-header-inspect -q fileheader.dat > out") == 0);
+    EXPECT_TRUE(system("../../apps/vespa-fileheader-inspect/vespa-fileheader-inspect -q fileheader.dat > out") == 0);
     EXPECT_EQUAL("fanart;string;\\fa\\na\\r\\t\n", readFile("out"));
 }
 
@@ -49,7 +49,7 @@ TEST("testDelimiter") {
     FileHeader header;
     header.putTag(FileHeader::Tag("string", "string"));
     ASSERT_TRUE(writeHeader(header, "fileheader.dat"));
-    EXPECT_TRUE(system("../../apps/fileheaderinspect/vespa-header-inspect -d i -q fileheader.dat > out") == 0);
+    EXPECT_TRUE(system("../../apps/vespa-fileheader-inspect/vespa-fileheader-inspect -d i -q fileheader.dat > out") == 0);
     EXPECT_EQUAL("str\\ingistr\\ingistr\\ing\n", readFile("out"));
 }
 
@@ -57,7 +57,7 @@ TEST("testQuiet") {
     FileHeader header;
     FileHeaderTk::addVersionTags(header);
     ASSERT_TRUE(writeHeader(header, "fileheader.dat"));
-    EXPECT_TRUE(system("../../apps/fileheaderinspect/vespa-header-inspect -q fileheader.dat > out") == 0);
+    EXPECT_TRUE(system("../../apps/vespa-fileheader-inspect/vespa-fileheader-inspect -q fileheader.dat > out") == 0);
     vespalib::string str = readFile("out");
     EXPECT_TRUE(!str.empty());
     for (uint32_t i = 0, numTags = header.getNumTags(); i < numTags; ++i) {
@@ -75,7 +75,7 @@ TEST("testVerbose") {
     FileHeader header;
     FileHeaderTk::addVersionTags(header);
     ASSERT_TRUE(writeHeader(header, "fileheader.dat"));
-    EXPECT_TRUE(system("../../apps/fileheaderinspect/vespa-header-inspect fileheader.dat > out") == 0);
+    EXPECT_TRUE(system("../../apps/vespa-fileheader-inspect/vespa-fileheader-inspect fileheader.dat > out") == 0);
     vespalib::string str = readFile("out");
     EXPECT_TRUE(!str.empty());
     for (uint32_t i = 0, numTags = header.getNumTags(); i < numTags; ++i) {
diff --git a/searchlib/src/vespa/searchlib/common/indexmetainfo.cpp b/searchlib/src/vespa/searchlib/common/indexmetainfo.cpp
index 4344895ad63..f92087f7352 100644
--- a/searchlib/src/vespa/searchlib/common/indexmetainfo.cpp
+++ b/searchlib/src/vespa/searchlib/common/indexmetainfo.cpp
@@ -3,6 +3,7 @@
 #include "indexmetainfo.h"
 #include <vespa/vespalib/util/stringfmt.h>
 #include <vespa/vespalib/util/guard.h>
+#include <cassert>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".indexmetainfo");
diff --git a/searchlib/src/vespa/searchlib/diskindex/fusion.cpp b/searchlib/src/vespa/searchlib/diskindex/fusion.cpp
index 5ebe6bdeefe..ea66a3435d1 100644
--- a/searchlib/src/vespa/searchlib/diskindex/fusion.cpp
+++ b/searchlib/src/vespa/searchlib/diskindex/fusion.cpp
@@ -3,18 +3,14 @@
 
 #include "fusion.h"
 #include "fieldreader.h"
-#include "fieldwriter.h"
 #include "dictionarywordreader.h"
 #include <vespa/vespalib/util/stringfmt.h>
-#include <vespa/searchlib/common/fslimits.h>
 #include <vespa/searchlib/util/filekit.h>
 #include <vespa/searchlib/util/dirtraverse.h>
 #include <vespa/vespalib/io/fileutil.h>
 #include <vespa/searchlib/common/documentsummary.h>
-#include <vespa/searchlib/common/tunefileinfo.h>
-#include <vespa/searchlib/index/postinglistparams.h>
-#include <vespa/searchlib/util/postingpriorityqueue.h>
 #include <vespa/vespalib/util/error.h>
+#include <sstream>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".diskindex.fusion");
diff --git a/searchlib/src/vespa/searchlib/docstore/chunkformat.cpp b/searchlib/src/vespa/searchlib/docstore/chunkformat.cpp
index cd44d34ea22..cddf8c96b2e 100644
--- a/searchlib/src/vespa/searchlib/docstore/chunkformat.cpp
+++ b/searchlib/src/vespa/searchlib/docstore/chunkformat.cpp
@@ -2,6 +2,7 @@
 
 #include "chunkformats.h"
 #include <vespa/document/util/compressor.h>
+#include <vespa/vespalib/util/stringfmt.h>
 
 namespace search {
 
diff --git a/searchlib/src/vespa/searchlib/docstore/chunkformats.cpp b/searchlib/src/vespa/searchlib/docstore/chunkformats.cpp
index c715f8a2129..0843d473121 100644
--- a/searchlib/src/vespa/searchlib/docstore/chunkformats.cpp
+++ b/searchlib/src/vespa/searchlib/docstore/chunkformats.cpp
@@ -3,6 +3,7 @@
 #include "chunkformats.h"
 #include <vespa/vespalib/util/crc.h>
 #include <vespa/vespalib/xxhash/xxhash.h>
+#include <vespa/vespalib/util/stringfmt.h>
 
 namespace search {
 
diff --git a/searchlib/src/vespa/searchlib/docstore/documentstore.cpp b/searchlib/src/vespa/searchlib/docstore/documentstore.cpp
index dd6a0f07197..fd7df29cd49 100644
--- a/searchlib/src/vespa/searchlib/docstore/documentstore.cpp
+++ b/searchlib/src/vespa/searchlib/docstore/documentstore.cpp
@@ -5,6 +5,8 @@
 #include "visitcache.h"
 #include "ibucketizer.h"
 #include <vespa/vespalib/stllike/cache.hpp>
+#include <vespa/vespalib/data/databuffer.h>
+#include <vespa/document/util/compressor.h>
 
 namespace search {
 
diff --git a/searchlib/src/vespa/searchlib/docstore/visitcache.cpp b/searchlib/src/vespa/searchlib/docstore/visitcache.cpp
index c424cb61d8c..8fac288a23a 100644
--- a/searchlib/src/vespa/searchlib/docstore/visitcache.cpp
+++ b/searchlib/src/vespa/searchlib/docstore/visitcache.cpp
@@ -4,9 +4,10 @@
 #include "ibucketizer.h"
 #include <vespa/vespalib/stllike/cache.hpp>
 #include <vespa/vespalib/stllike/hash_map.hpp>
+#include <vespa/vespalib/data/databuffer.h>
+#include <vespa/document/util/compressor.h>
 
-namespace search {
-namespace docstore {
+namespace search::docstore {
 
 using vespalib::ConstBufferRef;
 using vespalib::LockGuard;
@@ -241,5 +242,3 @@ VisitCache::Cache::onRemove(const K & key) {
 }
 
 }
-}
-
diff --git a/searchlib/src/vespa/searchlib/docstore/visitcache.h b/searchlib/src/vespa/searchlib/docstore/visitcache.h
index d47796d446a..6e5887349ee 100644
--- a/searchlib/src/vespa/searchlib/docstore/visitcache.h
+++ b/searchlib/src/vespa/searchlib/docstore/visitcache.h
@@ -6,11 +6,12 @@
 #include "cachestats.h"
 #include <vespa/vespalib/stllike/cache.h>
 #include <vespa/vespalib/stllike/hash_set.h>
+#include <vespa/vespalib/stllike/hash_map.h>
 #include <vespa/vespalib/util/alloc.h>
 #include <vespa/vespalib/objects/nbostream.h>
+#include <vespa/document/util/bytebuffer.h>
 
-namespace search {
-namespace docstore {
+namespace search::docstore {
 
 /**
  * Represents a unique set of keys that together acts as a key in a map.
@@ -160,4 +161,3 @@ private:
 };
 
 }
-}
diff --git a/searchlib/src/vespa/searchlib/expression/arrayatlookupfunctionnode.cpp b/searchlib/src/vespa/searchlib/expression/arrayatlookupfunctionnode.cpp
index 7d443219f29..eaa96587d6d 100644
--- a/searchlib/src/vespa/searchlib/expression/arrayatlookupfunctionnode.cpp
+++ b/searchlib/src/vespa/searchlib/expression/arrayatlookupfunctionnode.cpp
@@ -4,7 +4,7 @@
 #include "integerresultnode.h"
 #include "stringresultnode.h"
 #include <vespa/searchcommon/attribute/iattributecontext.h>
-
+#include <vespa/vespalib/util/stringfmt.h>
 
 namespace search {
 namespace expression {
diff --git a/searchlib/src/vespa/searchlib/expression/documentfieldnode.cpp b/searchlib/src/vespa/searchlib/expression/documentfieldnode.cpp
index 3a5375b672b..d9e4f6a3253 100644
--- a/searchlib/src/vespa/searchlib/expression/documentfieldnode.cpp
+++ b/searchlib/src/vespa/searchlib/expression/documentfieldnode.cpp
@@ -3,6 +3,7 @@
 #include "getdocidnamespacespecificfunctionnode.h"
 #include "getymumchecksumfunctionnode.h"
 #include <vespa/document/fieldvalue/fieldvalues.h>
+#include <vespa/document/datatype/documenttype.h>
 #include <vespa/vespalib/encoding/base64.h>
 
 #include <vespa/log/log.h>
diff --git a/searchlib/src/vespa/searchlib/expression/documentfieldnode.h b/searchlib/src/vespa/searchlib/expression/documentfieldnode.h
index a6c027beb22..2a202fe562e 100644
--- a/searchlib/src/vespa/searchlib/expression/documentfieldnode.h
+++ b/searchlib/src/vespa/searchlib/expression/documentfieldnode.h
@@ -4,6 +4,7 @@
 #include "documentaccessornode.h"
 #include "resultnode.h"
 #include "resultvector.h"
+#include <vespa/document/fieldvalue/iteratorhandler.h>
 
 namespace search {
 namespace expression {
@@ -38,11 +39,9 @@ public:
     DocumentFieldNode & operator = (const DocumentFieldNode & rhs);
     const vespalib::string & getFieldName() const override { return _fieldName; }
 private:
-    class Handler : public document::FieldValue::IteratorHandler {
+    class Handler : public document::fieldvalue::IteratorHandler {
     public:
         virtual void reset() = 0;
-    protected:
-        typedef document::FieldValue::IteratorHandler::Content Content;
     private:
         void onCollectionStart(const Content & c) override;
         void onStructStart(const Content & c) override;
diff --git a/searchlib/src/vespa/searchlib/expression/resultnode.cpp b/searchlib/src/vespa/searchlib/expression/resultnode.cpp
index 3306068efdf..bd8a5ff03d5 100644
--- a/searchlib/src/vespa/searchlib/expression/resultnode.cpp
+++ b/searchlib/src/vespa/searchlib/expression/resultnode.cpp
@@ -1,10 +1,18 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
 #include "resultnode.h"
-#include <stdexcept>
+#include <vespa/vespalib/util/stringfmt.h>
+#include <vespa/vespalib/util/exception.h>
 
 namespace search {
 namespace expression {
 
+int64_t
+ResultNode::onGetEnum(size_t index) const {
+    (void) index;
+    throw vespalib::Exception("search::expression::ResultNode onGetEnum is not implemented");
+}
+
 uint64_t ResultNode::radixAsc(const void * buf) const
 {
     (void) buf;
@@ -65,6 +73,11 @@ size_t ResultNode::getRawByteSize() const
     throw std::runtime_error("ResultNode::getRawByteSize() const must be overloaded by '" + vespalib::string(getClass().name()) + "'.");
 }
 
+const BucketResultNode&
+ResultNode::getNullBucket() const {
+    throw std::runtime_error(vespalib::make_string("No null bucket defined for this type"));
+}
+
 }
 }
 
diff --git a/searchlib/src/vespa/searchlib/expression/resultnode.h b/searchlib/src/vespa/searchlib/expression/resultnode.h
index 6343e90dc6b..26849d27ed3 100644
--- a/searchlib/src/vespa/searchlib/expression/resultnode.h
+++ b/searchlib/src/vespa/searchlib/expression/resultnode.h
@@ -3,7 +3,6 @@
 
 #include "expressionnode.h"
 #include "serializer.h"
-#include <vespa/vespalib/util/exception.h>
 #include <vespa/vespalib/util/buffer.h>
 
 namespace search {
@@ -49,10 +48,7 @@ public:
 
 private:
     virtual int64_t onGetInteger(size_t index) const = 0;
-    virtual int64_t onGetEnum(size_t index) const {
-        (void) index;
-        throw vespalib::Exception("search::expression::ResultNode onGetEnum is not implemented");
-    }
+    virtual int64_t onGetEnum(size_t index) const;
     virtual double    onGetFloat(size_t index) const = 0;
     virtual ConstBufferRef onGetString(size_t index, BufferRef buf) const = 0;
 
@@ -124,9 +120,8 @@ public:
     virtual ResultDeserializer & onDeserializeResult(ResultDeserializer & is);
     virtual size_t getRawByteSize() const;
     virtual bool isMultiValue() const { return false; }
-    virtual const BucketResultNode& getNullBucket() const { throw std::runtime_error(vespalib::make_string("No null bucket defined for this type")); }
+    virtual const BucketResultNode& getNullBucket() const;
 };
 
 }
 }
-
diff --git a/searchlib/src/vespa/searchlib/features/nativerankfeature.cpp b/searchlib/src/vespa/searchlib/features/nativerankfeature.cpp
index 92d00e9f907..924318a4a5a 100644
--- a/searchlib/src/vespa/searchlib/features/nativerankfeature.cpp
+++ b/searchlib/src/vespa/searchlib/features/nativerankfeature.cpp
@@ -3,8 +3,8 @@
 #include "nativerankfeature.h"
 #include "valuefeature.h"
 #include "utils.h"
-#include <vespa/searchlib/fef/fieldinfo.h>
 #include <vespa/searchlib/fef/properties.h>
+#include <sstream>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".features.nativerankfeature");
diff --git a/searchlib/src/vespa/searchlib/features/random_normal_feature.cpp b/searchlib/src/vespa/searchlib/features/random_normal_feature.cpp
index 127dd0feecf..44bbe0db6e3 100644
--- a/searchlib/src/vespa/searchlib/features/random_normal_feature.cpp
+++ b/searchlib/src/vespa/searchlib/features/random_normal_feature.cpp
@@ -21,7 +21,7 @@ RandomNormalExecutor::RandomNormalExecutor(uint64_t seed, double mean, double st
     _spare(0.0)
 
 {
-    LOG(debug, "RandomNormalExecutor: seed=%" PRIu64 ", mean=%f, stddev=%f", seed, mean, stddev);
+    LOG(debug, "RandomNormalExecutor: seed=%zu, mean=%f, stddev=%f", seed, mean, stddev);
     _rnd.srand48(seed);
 }
 
diff --git a/searchlib/src/vespa/searchlib/features/rankingexpressionfeature.cpp b/searchlib/src/vespa/searchlib/features/rankingexpressionfeature.cpp
index 07341ffd196..b14d44ee9b6 100644
--- a/searchlib/src/vespa/searchlib/features/rankingexpressionfeature.cpp
+++ b/searchlib/src/vespa/searchlib/features/rankingexpressionfeature.cpp
@@ -6,6 +6,7 @@
 #include <vespa/searchlib/fef/indexproperties.h>
 #include <vespa/searchlib/features/rankingexpression/feature_name_extractor.h>
 #include <vespa/eval/tensor/default_tensor_engine.h>
+#include <vespa/eval/eval/param_usage.h>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".features.rankingexpression");
@@ -244,7 +245,8 @@ RankingExpressionBlueprint::setup(const fef::IIndexEnvironment &env,
     // avoid costly compilation when only verifying setup
     if (env.getFeatureMotivation() != env.FeatureMotivation::VERIFY_SETUP) {
         if (do_compile) {
-            if (fef::indexproperties::eval::LazyExpressions::check(env.getProperties())) {
+            bool suggest_lazy = CompiledFunction::should_use_lazy_params(rank_function);
+            if (fef::indexproperties::eval::LazyExpressions::check(env.getProperties(), suggest_lazy)) {
                 _compile_token = CompileCache::compile(rank_function, PassParams::LAZY);
             } else {
                 _compile_token = CompileCache::compile(rank_function, PassParams::ARRAY);
diff --git a/searchlib/src/vespa/searchlib/fef/indexproperties.cpp b/searchlib/src/vespa/searchlib/fef/indexproperties.cpp
index b75a0f33393..d887d279e95 100644
--- a/searchlib/src/vespa/searchlib/fef/indexproperties.cpp
+++ b/searchlib/src/vespa/searchlib/fef/indexproperties.cpp
@@ -77,12 +77,11 @@ checkIfTrue(const Properties &props, const vespalib::string &name,
 namespace eval {
 
 const vespalib::string LazyExpressions::NAME("vespa.eval.lazy_expressions");
-const vespalib::string LazyExpressions::DEFAULT_VALUE("false");
 
 bool
-LazyExpressions::check(const Properties &props)
+LazyExpressions::check(const Properties &props, bool default_value)
 {
-    return checkIfTrue(props, NAME, DEFAULT_VALUE);
+    return lookupBool(props, NAME, default_value);
 }
 
 } // namespace eval
diff --git a/searchlib/src/vespa/searchlib/fef/indexproperties.h b/searchlib/src/vespa/searchlib/fef/indexproperties.h
index 4a97535879b..02942b1520a 100644
--- a/searchlib/src/vespa/searchlib/fef/indexproperties.h
+++ b/searchlib/src/vespa/searchlib/fef/indexproperties.h
@@ -26,8 +26,7 @@ namespace eval {
 // lazy evaluation of expressions. affects rank/summary/dump
 struct LazyExpressions {
     static const vespalib::string NAME;
-    static const vespalib::string DEFAULT_VALUE;
-    static bool check(const Properties &props);
+    static bool check(const Properties &props, bool default_value);
 };
 
 } // namespace eval
diff --git a/searchlib/src/vespa/searchlib/fef/test/ftlib.cpp b/searchlib/src/vespa/searchlib/fef/test/ftlib.cpp
index 1a359b4056e..08a0017c3e2 100644
--- a/searchlib/src/vespa/searchlib/fef/test/ftlib.cpp
+++ b/searchlib/src/vespa/searchlib/fef/test/ftlib.cpp
@@ -3,6 +3,7 @@
 #include "ftlib.h"
 #include "dummy_dependency_handler.h"
 #include <vespa/searchlib/features/utils.h>
+#include <vespa/vespalib/util/stringfmt.h>
 #include <boost/tokenizer.hpp>
 
 #include <vespa/log/log.h>
diff --git a/searchlib/src/vespa/searchlib/fef/test/matchdatabuilder.cpp b/searchlib/src/vespa/searchlib/fef/test/matchdatabuilder.cpp
index 1f59cfb52b8..02782565f80 100644
--- a/searchlib/src/vespa/searchlib/fef/test/matchdatabuilder.cpp
+++ b/searchlib/src/vespa/searchlib/fef/test/matchdatabuilder.cpp
@@ -4,6 +4,7 @@
 #include <vespa/searchlib/attribute/attributevector.h>
 #include <vespa/searchlib/attribute/attributevector.hpp>
 #include <vespa/searchlib/attribute/stringbase.h>
+#include <vespa/vespalib/util/stringfmt.h>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".fef.matchdatabuilder");
diff --git a/searchlib/src/vespa/searchlib/predicate/predicate_range_term_expander.h b/searchlib/src/vespa/searchlib/predicate/predicate_range_term_expander.h
index c6f7928f4d0..9b9c8eaaa3e 100644
--- a/searchlib/src/vespa/searchlib/predicate/predicate_range_term_expander.h
+++ b/searchlib/src/vespa/searchlib/predicate/predicate_range_term_expander.h
@@ -3,10 +3,9 @@
 #pragma once
 
 #include <vespa/vespalib/stllike/string.h>
-#include <vespa/vespalib/util/exceptions.h>
+#include <climits>
 
-namespace search {
-namespace predicate {
+namespace search::predicate {
 
 /**
  * Helper class for expanding a point in a predicate range query to
@@ -92,7 +91,4 @@ void PredicateRangeTermExpander::expand(const vespalib::string &key, int64_t sig
     }
 }
 
-
-}  // namespace search::predicate
-}  // namespace search
-
+}
diff --git a/searchlib/src/vespa/searchlib/query/querynoderesultbase.h b/searchlib/src/vespa/searchlib/query/querynoderesultbase.h
index c0d47f0f11d..05f0c9033d5 100644
--- a/searchlib/src/vespa/searchlib/query/querynoderesultbase.h
+++ b/searchlib/src/vespa/searchlib/query/querynoderesultbase.h
@@ -2,6 +2,7 @@
 #pragma once
 
 #include <vespa/vespalib/objects/cloneable.h>
+#include <memory>
 
 namespace search {
 
diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakewordset.cpp b/searchlib/src/vespa/searchlib/test/fakedata/fakewordset.cpp
index 9bf87ecac1a..b3b5f80a4a2 100644
--- a/searchlib/src/vespa/searchlib/test/fakedata/fakewordset.cpp
+++ b/searchlib/src/vespa/searchlib/test/fakedata/fakewordset.cpp
@@ -2,14 +2,12 @@
 
 #include "fakewordset.h"
 #include "fakeword.h"
-#include <vespa/searchlib/index/schemautil.h>
+#include <sstream>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".fakewordset");
 
-namespace search {
-
-namespace fakedata {
+namespace search::fakedata {
 
 using index::PostingListParams;
 using index::SchemaUtil;
@@ -146,7 +144,6 @@ FakeWordSet::getNumWords()
     return ret;
 }
 
-
 void
 FakeWordSet::addDocIdBias(uint32_t docIdBias)
 {
@@ -154,7 +151,4 @@ FakeWordSet::addDocIdBias(uint32_t docIdBias)
         applyDocIdBiasToVector(_words[i], docIdBias);
 }
 
-
-} // namespace fakedata
-
-} // namespace search
+}
diff --git a/searchlib/src/vespa/searchlib/test/imported_attribute_fixture.h b/searchlib/src/vespa/searchlib/test/imported_attribute_fixture.h
index 4b376df2bdd..eb278280156 100644
--- a/searchlib/src/vespa/searchlib/test/imported_attribute_fixture.h
+++ b/searchlib/src/vespa/searchlib/test/imported_attribute_fixture.h
@@ -15,6 +15,7 @@
 #include <vespa/searchlib/query/queryterm.h>
 #include <vespa/searchcommon/attribute/attributecontent.h>
 #include <vespa/vespalib/testkit/testapp.h>
+#include <vespa/vespalib/util/stringfmt.h>
 #include <algorithm>
 #include <future>
 #include <map>
diff --git a/searchlib/src/vespa/searchlib/util/filesizecalculator.cpp b/searchlib/src/vespa/searchlib/util/filesizecalculator.cpp
index 342ccb0104e..4be1eb668c9 100644
--- a/searchlib/src/vespa/searchlib/util/filesizecalculator.cpp
+++ b/searchlib/src/vespa/searchlib/util/filesizecalculator.cpp
@@ -31,22 +31,19 @@ FileSizeCalculator::extractFileSize(const vespalib::GenericHeader &header,
     uint64_t fileByteSize = fileBitSize / 8;
     if (!byteAligned(fileBitSize)) {
         LOG(error,
-            "Bad header file size tag for %s, fileBitSize=%" PRIu64
-            " which is not a multiple of 8",
+            "Bad header file size tag for %s, fileBitSize=%zu which is not a multiple of 8",
             fileName.c_str(), fileBitSize);
         return false;
     }
     if (fileByteSize < headerLen) {
         LOG(error,
-            "Bad header file size tag for %s, fileBitSize=%" PRIu64
-            " but header is %" PRIu64 "bits",
+            "Bad header file size tag for %s, fileBitSize=%zu but header is %zu bits",
             fileName.c_str(), fileBitSize, headerLen * 8);
         return false;
     }
     if (fileByteSize > fileSize) {
         LOG(error,
-            "Bad header file size tag for %s, fileBitSize=%" PRIu64
-            " but whole file size is %" PRIu64 "bits",
+            "Bad header file size tag for %s, fileBitSize=%zu but whole file size is %zu bits",
             fileName.c_str(), fileBitSize, fileSize * 8);
         return false;
     }
diff --git a/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp
index 8760a12fb82..4e11bbcd979 100644
--- a/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp
+++ b/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp
@@ -2,17 +2,14 @@
 
 #include "juniperdfw.h"
 #include "docsumwriter.h"
-#include "docsumfieldwriter.h"
 #include "docsumstate.h"
-#include "keywordextractor.h"
-#include "docsumformat.h"
 #include <vespa/searchlib/parsequery/stackdumpiterator.h>
 #include <vespa/searchlib/util/rawbuf.h>
 #include <vespa/searchlib/queryeval/split_float.h>
-
 #include <vespa/searchlib/fef/properties.h>
 #include <vespa/vespalib/objects/hexdump.h>
 #include <vespa/juniper/config.h>
+#include <sstream>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".searchlib.docsummary.dynamicteaserdfw");
diff --git a/searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.cpp
index e2dda337e73..697e2ed6722 100644
--- a/searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.cpp
+++ b/searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.cpp
@@ -6,12 +6,12 @@
 #include <vespa/vespalib/util/jsonwriter.h>
 #include <vespa/vespalib/data/slime/cursor.h>
 #include <vespa/vespalib/stllike/asciistream.h>
+#include <climits>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".searchlib.docsummary.geoposdfw");
 
-namespace search {
-namespace docsummary {
+namespace search::docsummary {
 
 using attribute::IAttributeVector;
 using attribute::IAttributeContext;
@@ -110,5 +110,3 @@ GeoPositionDFW::create(const char *attribute_name,
 
 
 }
-}
-
diff --git a/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.cpp
index 841840d8349..be9517eba03 100644
--- a/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.cpp
+++ b/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.cpp
@@ -2,16 +2,15 @@
 
 #include "positionsdfw.h"
 #include "docsumstate.h"
-#include "idocsumenvironment.h"
 #include <vespa/searchlib/common/location.h>
 #include <vespa/vespalib/stllike/asciistream.h>
 #include <cmath>
+#include <climits>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".searchlib.docsummary.positionsdfw");
 
-namespace search {
-namespace docsummary {
+namespace search::docsummary {
 
 using search::attribute::IAttributeContext;
 using search::attribute::IAttributeVector;
@@ -223,5 +222,4 @@ AbsDistanceDFW::UP createAbsDistanceDFW(const char *attribute_name,
     return ret;
 }
 
-}  // namespace docsummary
-}  // namespace search
+}
diff --git a/searchsummary/src/vespa/searchsummary/docsummary/urlresult.cpp b/searchsummary/src/vespa/searchsummary/docsummary/urlresult.cpp
index 0d3a212fc1f..d0beb2bd315 100644
--- a/searchsummary/src/vespa/searchsummary/docsummary/urlresult.cpp
+++ b/searchsummary/src/vespa/searchsummary/docsummary/urlresult.cpp
@@ -3,12 +3,12 @@
 #include "urlresult.h"
 #include "resultconfig.h"
 #include <zlib.h>
+#include <cassert>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".searchlib.docsummary.urlresult");
 
-namespace search {
-namespace docsummary {
+namespace search::docsummary {
 
 urlresult::urlresult(uint32_t partition, uint32_t docid, HitRank metric)
     : _partition(partition),
@@ -811,4 +811,3 @@ GeneralResult::_inplace_unpack(const char *buf, const size_t buflen)
 }
 
 }
-}
diff --git a/slobrok/src/tests/startsome/tstdst.cpp b/slobrok/src/tests/startsome/tstdst.cpp
index 577bf6f59a1..b645f79efa4 100644
--- a/slobrok/src/tests/startsome/tstdst.cpp
+++ b/slobrok/src/tests/startsome/tstdst.cpp
@@ -6,6 +6,7 @@
 #include <vespa/fnet/frt/invoker.h>
 #include <vespa/fnet/transport.h>
 #include <vespa/fnet/frt/target.h>
+#include <sstream>
 
 #include <vespa/log/log.h>
 LOG_SETUP("testrpcserver");
diff --git a/slobrok/src/vespa/slobrok/sblist.h b/slobrok/src/vespa/slobrok/sblist.h
index 53962363609..81bf5de325d 100644
--- a/slobrok/src/vespa/slobrok/sblist.h
+++ b/slobrok/src/vespa/slobrok/sblist.h
@@ -4,8 +4,7 @@
 #include "cfg.h"
 #include <vespa/vespalib/util/sync.h>
 
-namespace slobrok {
-namespace api {
+namespace slobrok::api {
 
 /**
  * @brief List of connection specs for service location brokers
@@ -50,6 +49,4 @@ private:
     size_t _retryCount;
 };
 
-} // namespace api
-} // namespace slobrok
-
+}
diff --git a/slobrok/src/vespa/slobrok/sbregister.cpp b/slobrok/src/vespa/slobrok/sbregister.cpp
index c930c35f8bb..0d1e506b613 100644
--- a/slobrok/src/vespa/slobrok/sbregister.cpp
+++ b/slobrok/src/vespa/slobrok/sbregister.cpp
@@ -5,11 +5,14 @@
 #include <vespa/fnet/frt/target.h>
 #include <vespa/vespalib/util/host_name.h>
 #include <vespa/vespalib/stllike/asciistream.h>
+#include <vespa/vespalib/util/exceptions.h>
 #include <sstream>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".slobrok.register");
 
+using vespalib::NetworkSetupFailureException;
+
 namespace {
 
 vespalib::string
@@ -47,8 +50,7 @@ discard(std::vector<vespalib::string> &vec, const vespalib::stringref & val)
 
 } // namespace <unnamed>
 
-namespace slobrok {
-namespace api {
+namespace slobrok::api {
 
 RegisterAPI::RegisterAPI(FRT_Supervisor &orb, const ConfiguratorFactory & config)
     : FNET_Task(orb.GetScheduler()),
@@ -69,7 +71,10 @@ RegisterAPI::RegisterAPI(FRT_Supervisor &orb, const ConfiguratorFactory & config
       _req(0)
 {
     _configurator->poll();
-    LOG_ASSERT(_slobrokSpecs.ok());
+    if ( ! _slobrokSpecs.ok()) {
+        throw NetworkSetupFailureException("Failed configuring the RegisterAPI. No valid slobrok specs from config",
+                                           VESPA_STRLOC);
+    }
     ScheduleNow();
 }
 
@@ -127,8 +132,8 @@ RegisterAPI::handleReqDone()
         _reqDone = false;
         if (_req->IsError()) {
             if (_req->GetErrorCode() != FRTE_RPC_METHOD_FAILED) {
-		LOG(debug, "register failed: %s (code %d)",
-		    _req->GetErrorMessage(), _req->GetErrorCode());
+                LOG(debug, "register failed: %s (code %d)",
+                    _req->GetErrorMessage(), _req->GetErrorCode());
                 // unexpected error; close our connection to this
                 // slobrok server and try again with a fresh slate
                 if (_target != 0) {
@@ -187,7 +192,7 @@ RegisterAPI::handleReconnect()
                 LOG(warning, "cannot connect to location broker at %s "
                     "(retry in %f seconds)", cps.c_str(), delay);
             } else {
-		LOG(debug, "slobrok retry in %f seconds", delay);
+                LOG(debug, "slobrok retry in %f seconds", delay);
 	    }
             return;
         }
@@ -220,7 +225,7 @@ RegisterAPI::handlePending()
         _req = _orb.AllocRPCRequest();
         _req->SetMethodName("slobrok.unregisterRpcServer");
         _req->GetParams()->AddString(name.c_str());
-	LOG(debug, "unregister [%s]", name.c_str());
+        LOG(debug, "unregister [%s]", name.c_str());
         _req->GetParams()->AddString(createSpec(_orb).c_str());
         _target->InvokeAsync(_req, 35.0, this);
     } else if (reg) {
@@ -228,7 +233,7 @@ RegisterAPI::handlePending()
         _req = _orb.AllocRPCRequest();
         _req->SetMethodName("slobrok.registerRpcServer");
         _req->GetParams()->AddString(name.c_str());
-	LOG(debug, "register [%s]", name.c_str());
+        LOG(debug, "register [%s]", name.c_str());
         _req->GetParams()->AddString(createSpec(_orb).c_str());
         _target->InvokeAsync(_req, 35.0, this);
     } else {
@@ -236,7 +241,7 @@ RegisterAPI::handlePending()
         // names after a long delay.
         _lock.Lock();
         _pending = _names;
-	LOG(debug, "done, reschedule in 30s");
+        LOG(debug, "done, reschedule in 30s");
         _busy = false;
         Schedule(30.0);
         _lock.Unlock();
@@ -248,7 +253,7 @@ RegisterAPI::PerformTask()
 {
     handleReqDone();
     if (_req != 0) {
-	LOG(debug, "req in progress");
+        LOG(debug, "req in progress");
         return; // current request still in progress, don't start anything new
     }
     handleReconnect();
@@ -312,5 +317,4 @@ RegisterAPI::RPCHooks::rpc_notifyUnregistered(FRT_RPCRequest *req)
     LOG(warning, "unregistered name %s", args[0]._string._str);
 }
 
-} // namespace api
-} // namespace slobrok
+}
diff --git a/slobrok/src/vespa/slobrok/server/rpc_server_manager.cpp b/slobrok/src/vespa/slobrok/server/rpc_server_manager.cpp
index 84ef6432f74..9d34d29fc0b 100644
--- a/slobrok/src/vespa/slobrok/server/rpc_server_manager.cpp
+++ b/slobrok/src/vespa/slobrok/server/rpc_server_manager.cpp
@@ -5,6 +5,7 @@
 #include "rpc_server_map.h"
 #include "remote_slobrok.h"
 #include "sbenv.h"
+#include <vespa/vespalib/util/stringfmt.h>
 #include <sstream>
 
 #include <vespa/log/log.h>
diff --git a/staging_vespalib/src/tests/benchmark/testbase.cpp b/staging_vespalib/src/tests/benchmark/testbase.cpp
index 93f5c9b975a..6de6f66f76b 100644
--- a/staging_vespalib/src/tests/benchmark/testbase.cpp
+++ b/staging_vespalib/src/tests/benchmark/testbase.cpp
@@ -1,6 +1,7 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 #include "testbase.h"
 #include <vespa/fastos/timestamp.h>
+#include <cassert>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".testbase");
diff --git a/staging_vespalib/src/tests/xmlserializable/xmlserializabletest.cpp b/staging_vespalib/src/tests/xmlserializable/xmlserializabletest.cpp
index 90fcdb8f94f..4af70f7e6fe 100644
--- a/staging_vespalib/src/tests/xmlserializable/xmlserializabletest.cpp
+++ b/staging_vespalib/src/tests/xmlserializable/xmlserializabletest.cpp
@@ -1,7 +1,7 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include <vespa/vespalib/testkit/testapp.h>
-#include <vespa/vespalib/util/xmlserializable.h>
+#include <vespa/vespalib/util/xmlstream.h>
 
 namespace vespalib {
 
diff --git a/staging_vespalib/src/vespa/vespalib/encoding/base64.cpp b/staging_vespalib/src/vespa/vespalib/encoding/base64.cpp
index 10bc2af51ce..54ac8363bf6 100644
--- a/staging_vespalib/src/vespa/vespalib/encoding/base64.cpp
+++ b/staging_vespalib/src/vespa/vespalib/encoding/base64.cpp
@@ -8,7 +8,8 @@
 
 #include <vespa/vespalib/encoding/base64.h>
 #include <vespa/vespalib/util/exceptions.h>
-#include <assert.h>
+#include <vespa/vespalib/util/stringfmt.h>
+#include <cassert>
 
 namespace vespalib {
 
@@ -129,9 +130,8 @@ Base64::decode(const char* inBuffer, int inLen, char* outBuffer, int outLen)
             continue; // Some illegal chars will be skipped.
         } else if (curchar == -1) {
             // Other illegal characters will generate failure
-            throw vespalib::IllegalArgumentException(vespalib::make_string(
-                    "Illegal base64 character %u found.",
-                    (unsigned int) *thischar), VESPA_STRLOC);
+            throw IllegalArgumentException(make_string("Illegal base64 character %u found.",
+                                                       (unsigned int) *thischar), VESPA_STRLOC);
         } else {
 
             // Four bytes from input (eqals three bytes in output)
@@ -162,4 +162,3 @@ Base64::decode(const char* inBuffer, int inLen, char* outBuffer, int outLen)
 }
 
 } // namespace vespalib
-
diff --git a/staging_vespalib/src/vespa/vespalib/net/http_server.cpp b/staging_vespalib/src/vespa/vespalib/net/http_server.cpp
index 32b9caab687..2c67a3c48c9 100644
--- a/staging_vespalib/src/vespa/vespalib/net/http_server.cpp
+++ b/staging_vespalib/src/vespa/vespalib/net/http_server.cpp
@@ -3,7 +3,7 @@
 #include "http_server.h"
 #include <vespa/vespalib/util/exceptions.h>
 #include <vespa/vespalib/util/host_name.h>
-#include <algorithm>
+#include <vespa/vespalib/util/stringfmt.h>
 
 namespace vespalib {
 
diff --git a/staging_vespalib/src/vespa/vespalib/objects/cloneable.h b/staging_vespalib/src/vespa/vespalib/objects/cloneable.h
index c0b12272797..ea9c8002d6e 100644
--- a/staging_vespalib/src/vespa/vespalib/objects/cloneable.h
+++ b/staging_vespalib/src/vespa/vespalib/objects/cloneable.h
@@ -5,8 +5,6 @@
  * @brief Superclass for objects implementing clone() deep copy.
  */
 
-#include <memory>
-
 namespace vespalib {
 
 class Cloneable {
diff --git a/staging_vespalib/src/vespa/vespalib/util/CMakeLists.txt b/staging_vespalib/src/vespa/vespalib/util/CMakeLists.txt
index ca440428e0a..3b36b863707 100644
--- a/staging_vespalib/src/vespa/vespalib/util/CMakeLists.txt
+++ b/staging_vespalib/src/vespa/vespalib/util/CMakeLists.txt
@@ -18,5 +18,6 @@ vespa_add_library(staging_vespalib_vespalib_util OBJECT
     shutdownguard.cpp
     timer.cpp
     xmlserializable.cpp
+    xmlstream.cpp
     DEPENDS
 )
diff --git a/staging_vespalib/src/vespa/vespalib/util/clock.h b/staging_vespalib/src/vespa/vespalib/util/clock.h
index 4a5611bbe5a..e17ec962fc9 100644
--- a/staging_vespalib/src/vespa/vespalib/util/clock.h
+++ b/staging_vespalib/src/vespa/vespalib/util/clock.h
@@ -33,7 +33,7 @@ public:
     Clock(double timePeriod=0.100);
     ~Clock();
 
-    fastos::TimeStamp getTimeNS(void) const {
+    fastos::TimeStamp getTimeNS() const {
         if (!_running) {
             setTime();
         }
@@ -41,7 +41,7 @@ public:
     }
     fastos::TimeStamp getTimeNSAssumeRunning() const { return _timeNS; }
 
-    void stop(void);
+    void stop();
 };
 
 }
diff --git a/staging_vespalib/src/vespa/vespalib/util/document_runnable.cpp b/staging_vespalib/src/vespa/vespalib/util/document_runnable.cpp
index 7c581c89af0..576343245e4 100644
--- a/staging_vespalib/src/vespa/vespalib/util/document_runnable.cpp
+++ b/staging_vespalib/src/vespa/vespalib/util/document_runnable.cpp
@@ -2,6 +2,7 @@
 
 #include "document_runnable.h"
 #include <vespa/vespalib/util/exceptions.h>
+#include <vespa/vespalib/util/stringfmt.h>
 
 namespace document {
 
diff --git a/staging_vespalib/src/vespa/vespalib/util/growablebytebuffer.cpp b/staging_vespalib/src/vespa/vespalib/util/growablebytebuffer.cpp
index 537b44c49eb..354d2e013ed 100644
--- a/staging_vespalib/src/vespa/vespalib/util/growablebytebuffer.cpp
+++ b/staging_vespalib/src/vespa/vespalib/util/growablebytebuffer.cpp
@@ -1,5 +1,6 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-#include <vespa/vespalib/util/growablebytebuffer.h>
+#include "growablebytebuffer.h"
+#include <arpa/inet.h>
 
 using namespace vespalib;
 
diff --git a/staging_vespalib/src/vespa/vespalib/util/growablebytebuffer.h b/staging_vespalib/src/vespa/vespalib/util/growablebytebuffer.h
index fd65f0134bc..fdf090d22f9 100644
--- a/staging_vespalib/src/vespa/vespalib/util/growablebytebuffer.h
+++ b/staging_vespalib/src/vespa/vespalib/util/growablebytebuffer.h
@@ -1,7 +1,6 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 #pragma once
 
-#include <vespa/fastos/types.h>
 #include <vespa/vespalib/util/memory.h>
 #include <vespa/vespalib/stllike/string.h>
 
diff --git a/staging_vespalib/src/vespa/vespalib/util/jsonstream.cpp b/staging_vespalib/src/vespa/vespalib/util/jsonstream.cpp
index e8f242879d8..c7a7ac34ef3 100644
--- a/staging_vespalib/src/vespa/vespalib/util/jsonstream.cpp
+++ b/staging_vespalib/src/vespa/vespalib/util/jsonstream.cpp
@@ -4,6 +4,7 @@
 #include "jsonexception.h"
 #include <vespa/vespalib/util/exceptions.h>
 #include <vespa/vespalib/stllike/asciistream.h>
+#include <vespa/vespalib/util/stringfmt.h>
 
 namespace vespalib {
 
@@ -18,7 +19,7 @@ JsonStream::getStateName(const State& s) {
     throw IllegalStateException("Control should not reach this point", VESPA_STRLOC);
 }
 
-JsonStream::JsonStream(vespalib::asciistream& as, bool createIndents)
+JsonStream::JsonStream(asciistream& as, bool createIndents)
     : _writer(as)
 {
     if (createIndents) _writer.setPretty();
@@ -28,7 +29,7 @@ JsonStream::JsonStream(vespalib::asciistream& as, bool createIndents)
 JsonStream::~JsonStream() {}
 
 JsonStream&
-JsonStream::operator<<(vespalib::stringref value)
+JsonStream::operator<<(stringref value)
 {
     if (_state.empty()) {
         fail("Stream already finalized. Can't add a string value.");
@@ -314,7 +315,7 @@ JsonStream::finalize()
 string
 JsonStream::getStateString() const
 {
-    vespalib::asciistream as;
+    asciistream as;
     for (auto it(_state.begin()), mt(_state.end()); it != mt; it++) {
         switch (it->state) {
             case State::OBJECT_EXPECTING_KEY:
@@ -343,7 +344,7 @@ JsonStream::getStateString() const
     return as.str();
 }
 
-vespalib::string
+string
 JsonStream::getJsonStreamState() const
 {
     asciistream report;
@@ -360,4 +361,4 @@ JsonStream::fail(stringref error) const
     throw JsonStreamException(report.str(), "", VESPA_STRLOC);
 }
 
-} // vespalib
+}
diff --git a/staging_vespalib/src/vespa/vespalib/util/librarypool.cpp b/staging_vespalib/src/vespa/vespalib/util/librarypool.cpp
index 8dd6995a6c1..d992b1b6db8 100644
--- a/staging_vespalib/src/vespa/vespalib/util/librarypool.cpp
+++ b/staging_vespalib/src/vespa/vespalib/util/librarypool.cpp
@@ -1,6 +1,8 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
 #include <vespa/vespalib/util/librarypool.h>
 #include <vespa/vespalib/util/exceptions.h>
+#include <vespa/vespalib/util/stringfmt.h>
 
 namespace vespalib {
 
@@ -17,14 +19,14 @@ LibraryPool::~LibraryPool()
 }
 
 void
-LibraryPool::loadLibrary(const vespalib::stringref & libName)
+LibraryPool::loadLibrary(const stringref & libName)
 {
     LockGuard guard(_lock);
     if (_libraries.find(libName) == _libraries.end()) {
         DynamicLibrarySP lib(new FastOS_DynamicLibrary);
-        vespalib::string file(libName);
+        string file(libName);
         if (!lib->Open(file.c_str())) {
-            vespalib::string error = lib->GetLastErrorString();
+            string error = lib->GetLastErrorString();
             throw IllegalArgumentException(make_string("Failed loading dynamic library '%s' due to '%s'.",
                       file.c_str(), error.c_str()));
         } else {
@@ -34,7 +36,7 @@ LibraryPool::loadLibrary(const vespalib::stringref & libName)
 }
 
 FastOS_DynamicLibrary *
-LibraryPool::get(const vespalib::stringref & name)
+LibraryPool::get(const stringref & name)
 {
     LockGuard guard(_lock);
     LibraryMap::const_iterator found(_libraries.find(name));
@@ -44,7 +46,7 @@ LibraryPool::get(const vespalib::stringref & name)
 }
 
 const FastOS_DynamicLibrary *
-LibraryPool::get(const vespalib::stringref & name) const
+LibraryPool::get(const stringref & name) const
 {
     LockGuard guard(_lock);
     LibraryMap::const_iterator found(_libraries.find(name));
diff --git a/staging_vespalib/src/vespa/vespalib/util/polymorphicarray.h b/staging_vespalib/src/vespa/vespalib/util/polymorphicarray.h
new file mode 100644
index 00000000000..6c4d8e37311
--- /dev/null
+++ b/staging_vespalib/src/vespa/vespalib/util/polymorphicarray.h
@@ -0,0 +1,82 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+//
+#pragma once
+
+#include "polymorphicarraybase.h"
+
+namespace vespalib {
+
+/**
+ * Describes an interface an array of polymorphic types.
+ * The intention is to allow efficient implementations when that is possible
+ * while still enjoying the flexibility of the polymorph interface.
+ * It is not a full feldged Array implementation as std::vector. It contains just
+ * the minimum required to allow for efficient implementations for document::ArrayFieldValue.
+ *
+ * You specify the base type the interface shall provide. This base type must define
+ * virtual void assign(const B & rhs);
+ * For use with ComplexArrayT your type also need
+ * virtual T * clone() const;
+ */
+template<typename B>
+class IArrayT : public IArrayBase {
+public:
+    class iterator {
+    public:
+        iterator(IArrayT &a, size_t i) : _a(&a), _i(i) {}
+        iterator operator+(size_t diff) const { return iterator(*_a, _i + diff); }
+        iterator &operator++() {
+            ++_i;
+            return *this;
+        }
+        iterator operator++(int) {
+            iterator other(*this);
+            ++_i;
+            return other;
+        }
+        bool operator==(const iterator &other) const { return (_a == other._a) && (_i == other._i); }
+        bool operator!=(const iterator &other) const { return (_i != other._i) || (_a != other._a); }
+        B &operator*() { return (*_a)[_i]; }
+        B *operator->() { return &(*_a)[_i]; }
+        friend ssize_t operator-(const iterator &a, const iterator &b) { return a._i - b._i; }
+    private:
+        IArrayT *_a;
+        size_t _i;
+    };
+
+    class const_iterator {
+    public:
+        const_iterator(const IArrayT &a, size_t i) : _a(&a), _i(i) {}
+        const_iterator operator+(size_t diff) const { return const_iterator(*_a, _i + diff); }
+        const_iterator &operator++() {
+            ++_i;
+            return *this;
+        }
+        const_iterator operator++(int) {
+            const_iterator other(*this);
+            ++_i;
+            return other;
+        }
+        bool operator==(const const_iterator &other) const { return (_a == other._a) && (_i == other._i); }
+        bool operator!=(const const_iterator &other) const { return (_i != other._i) || (_a != other._a); }
+        const B &operator*() const { return (*_a)[_i]; }
+        const B *operator->() const { return &(*_a)[_i]; }
+        size_t operator-(const const_iterator &b) const { return _i - b._i; }
+    private:
+        const IArrayT *_a;
+        size_t _i;
+    };
+
+    typedef std::unique_ptr<IArrayT> UP;
+    virtual const B &operator[](size_t i) const = 0;
+    virtual B &operator[](size_t i) = 0;
+    virtual IArrayT *clone() const override = 0;
+    virtual iterator erase(iterator it) = 0;
+    virtual const_iterator begin() const { return const_iterator(*this, 0); }
+    virtual const_iterator end() const { return const_iterator(*this, size()); }
+    virtual iterator begin() { return iterator(*this, 0); }
+    virtual iterator end() { return iterator(*this, size()); }
+    virtual void push_back(const B &v) = 0;
+};
+
+}
diff --git a/staging_vespalib/src/vespa/vespalib/util/polymorphicarraybase.h b/staging_vespalib/src/vespa/vespalib/util/polymorphicarraybase.h
new file mode 100644
index 00000000000..0fccd6cabb7
--- /dev/null
+++ b/staging_vespalib/src/vespa/vespalib/util/polymorphicarraybase.h
@@ -0,0 +1,18 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+//
+#pragma once
+
+namespace vespalib {
+
+class IArrayBase {
+public:
+    virtual ~IArrayBase() {}
+    virtual void resize(size_t sz) = 0;
+    virtual void reserve(size_t sz) = 0;
+    virtual void clear() = 0;
+    virtual IArrayBase *clone() const = 0;
+    virtual size_t size() const = 0;
+    bool empty() const { return size() == 0; }
+};
+
+}
diff --git a/staging_vespalib/src/vespa/vespalib/util/polymorphicarrays.h b/staging_vespalib/src/vespa/vespalib/util/polymorphicarrays.h
index 636586f839e..646ca522ba3 100644
--- a/staging_vespalib/src/vespa/vespalib/util/polymorphicarrays.h
+++ b/staging_vespalib/src/vespa/vespalib/util/polymorphicarrays.h
@@ -2,75 +2,12 @@
 //
 #pragma once
 
+#include "polymorphicarray.h"
 #include <vespa/vespalib/util/memory.h>
+#include <vector>
 
 namespace vespalib {
 
-/**
- * Describes an interface an array of polymorphic types.
- * The intention is to allow efficient implementations when that is possible
- * while still enjoying the flexibility of the polymorph interface.
- * It is not a full feldged Array implementation as std::vector. It contains just
- * the minimum required to allow for efficient implementations for document::ArrayFieldValue.
- *
- * You specify the base type the interface shall provide. This base type must define
- * virtual void assign(const B & rhs);
- * For use with ComplexArrayT your type also need
- * virtual T * clone() const;
- */
-template<typename B>
-class IArrayT
-{
-public:
-    class iterator {
-    public:
-        iterator(IArrayT & a, size_t i) : _a(&a), _i(i) { }
-        iterator operator+(size_t diff) const { return iterator(*_a, _i + diff); }
-        iterator& operator++() { ++_i; return *this; }
-        iterator operator++(int) { iterator other(*this); ++_i; return other; }
-        bool operator==(const iterator & other) const { return (_a == other._a) && (_i == other._i); }
-        bool operator!=(const iterator & other) const { return (_i != other._i) || (_a != other._a); }
-        B & operator*() { return (*_a)[_i]; }
-        B * operator->() { return &(*_a)[_i]; }
-        friend ssize_t operator - (const iterator & a, const iterator & b) { return a._i - b._i; }
-    private:
-        IArrayT * _a;
-        size_t  _i;
-    };
-    class const_iterator {
-    public:
-        const_iterator(const IArrayT & a, size_t i) : _a(&a), _i(i) { }
-        const_iterator operator+(size_t diff) const { return const_iterator(*_a, _i + diff); }
-        const_iterator& operator++() { ++_i; return *this; }
-        const_iterator operator++(int) { const_iterator other(*this); ++_i; return other; }
-        bool operator==(const const_iterator & other) const { return (_a == other._a) && (_i == other._i); }
-        bool operator!=(const const_iterator & other) const { return (_i != other._i) || (_a != other._a); }
-        const B & operator*() const { return (*_a)[_i]; }
-        const B * operator->() const { return &(*_a)[_i]; }
-        size_t operator - (const const_iterator & b) const { return _i - b._i; }
-    private:
-        const IArrayT * _a;
-        size_t  _i;
-    };
-    typedef std::unique_ptr<IArrayT> UP;
-
-    virtual ~IArrayT() { }
-    virtual const B & operator [] (size_t i) const = 0;
-    virtual B & operator [] (size_t i) = 0;
-    virtual void resize(size_t sz) = 0;
-    virtual void reserve(size_t sz) = 0;
-    virtual void clear() = 0;
-    virtual IArrayT * clone() const = 0;
-    virtual size_t size() const = 0;
-    virtual iterator erase(iterator it) = 0;
-    virtual const_iterator begin() const { return const_iterator(*this, 0); }
-    virtual const_iterator end() const { return const_iterator(*this, size()); }
-    virtual iterator begin() { return iterator(*this, 0); }
-    virtual iterator end() { return iterator(*this, size()); }
-    bool empty() const { return size() == 0; }
-    virtual void push_back(const B & v) = 0;
-};
-
 template <typename T, typename B>
 class PrimitiveArrayT : public IArrayT<B>
 {
diff --git a/staging_vespalib/src/vespa/vespalib/util/programoptions.cpp b/staging_vespalib/src/vespa/vespalib/util/programoptions.cpp
index 41ff561011b..6573390c712 100644
--- a/staging_vespalib/src/vespa/vespalib/util/programoptions.cpp
+++ b/staging_vespalib/src/vespa/vespalib/util/programoptions.cpp
@@ -1,7 +1,6 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-#include <vespa/vespalib/util/programoptions.h>
-
-#include <iostream>
+#include "programoptions.h"
+#include <vespa/vespalib/util/stringfmt.h>
 #include <vespa/vespalib/util/exceptions.h>
 #include <boost/lexical_cast.hpp>
 
diff --git a/staging_vespalib/src/vespa/vespalib/util/xmlserializable.cpp b/staging_vespalib/src/vespa/vespalib/util/xmlserializable.cpp
index 58a5489647d..357b3d94992 100644
--- a/staging_vespalib/src/vespa/vespalib/util/xmlserializable.cpp
+++ b/staging_vespalib/src/vespa/vespalib/util/xmlserializable.cpp
@@ -1,449 +1,12 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
-#include <vespa/vespalib/util/xmlserializable.hpp>
-#include <vespa/vespalib/encoding/base64.h>
-#include <vespa/vespalib/stllike/asciistream.h>
-#include <vector>
-#include <cassert>
+#include "xmlserializable.h"
+#include "xmlstream.h"
+#include <sstream>
 
 namespace vespalib {
 namespace xml {
 
-namespace {
-
-    std::vector<bool> getLegalIdentifierFirstCharacters() {
-        std::vector<bool> vec(256, false);
-        for (uint32_t i='a'; i<='z'; ++i) vec[i] = true;
-        for (uint32_t i='A'; i<='Z'; ++i) vec[i] = true;
-        vec[':'] = true;
-        vec['_'] = true;
-        return vec;
-    }
-
-    std::vector<bool> getLegalIdentifierCharacters() {
-        std::vector<bool> vec(getLegalIdentifierFirstCharacters());
-        vec['-'] = true;
-        vec['.'] = true;
-        for (uint32_t i='0'; i<='9'; ++i) {
-            vec[i] = true;
-        }
-        return vec;
-    }
-
-    std::vector<bool> getBinaryCharacters() {
-        std::vector<bool> vec(256, false);
-        for (uint32_t i=0; i<32; ++i) {
-            vec[i] = true;
-        }
-        vec['\t'] = false;
-        vec['\n'] = false;
-        vec['\r'] = false;
-        vec['\f'] = false;
-        return vec;
-    }
-
-    std::vector<bool> getEscapedXmlCharacters() {
-        std::vector<bool> vec(256, false);
-        for (uint32_t i=0; i<32; ++i) {
-            vec[i] = true;
-        }
-        vec['\n'] = false;
-        vec['<'] = true;
-        vec['>'] = true;
-        vec['&'] = true;
-        return vec;
-    }
-
-    std::vector<bool> legalIdentifierFirstChar(
-            getLegalIdentifierFirstCharacters());
-    std::vector<bool> legalIdentifierChars = getLegalIdentifierCharacters();
-    std::vector<bool> binaryChars = getBinaryCharacters();
-    std::vector<bool> escapedXmlChars = getEscapedXmlCharacters();
-
-    bool containsBinaryCharacters(const std::string& s) {
-        for (int i=0, n=s.size(); i<n; ++i) {
-            if (binaryChars[static_cast<uint8_t>(s[i])]) return true;
-        }
-        return false;
-    }
-
-    const std::string xmlAttributeEscape(const std::string& s) {
-        vespalib::asciistream ost;
-        for (uint32_t i=0, n=s.size(); i<n; ++i) {
-            if (s[i] == '"' || s[i] == '\n'
-                || escapedXmlChars[static_cast<uint8_t>(s[i])])
-            {
-                if      (s[i] == '<') ost << "&lt;";
-                else if (s[i] == '>') ost << "&gt;";
-                else if (s[i] == '&') ost << "&amp;";
-                else if (s[i] == '"') ost << "&quot;";
-                else {
-                    ost << "&#" << (int) s[i] << ";";
-                }
-            } else {
-                ost << s[i];
-            }
-        }
-        return ost.str();
-    }
-
-    void writeEscaped(std::ostream& out, const std::string& s) {
-        for (uint32_t i=0, n=s.size(); i<n; ++i) {
-            if (escapedXmlChars[static_cast<uint8_t>(s[i])]) {
-                if      (s[i] == '<') out << "&lt;";
-                else if (s[i] == '>') out << "&gt;";
-                else if (s[i] == '&') out << "&amp;";
-                else {
-                    out << "&#" << (int) s[i] << ";";
-                }
-            } else {
-                out << s[i];
-            }
-        }
-    }
-
-    void writeBase64Encoded(std::ostream& out, const std::string& s) {
-        out << vespalib::Base64::encode(&s[0], s.size());
-    }
-}
-
-bool isLegalName(const std::string& name) {
-    if (name.size() == 0) return false;
-    if (!legalIdentifierFirstChar[static_cast<uint8_t>(name[0])]) return false;
-    for (int i=1, n=name.size(); i<n; ++i) {
-        if (!legalIdentifierChars[static_cast<uint8_t>(name[i])]) return false;
-    }
-    return true;
-}
-
-void convertToLegalName(std::string& name) {
-    if (name.size() == 0) {
-        name == "__no_name__";
-    } else {
-        if (!legalIdentifierFirstChar[static_cast<uint8_t>(name[0])]) {
-            name[0] = '_';
-        }
-        for (int i=1, n=name.size(); i<n; ++i) {
-            if (!legalIdentifierChars[static_cast<uint8_t>(name[i])]) {
-                name[i] = '_';
-            }
-        }
-    }
-}
-
-XmlOutputStream::XmlOutputStream(std::ostream& ostream,
-                                 const std::string& indent)
-    : _indent(indent),
-      _wrappedStream(ostream),
-      _tagStack(),
-      _cachedTag(),
-      _cachedAttributes(),
-      _cachedContent()
-{
-}
-
-XmlAttribute::~XmlAttribute()
-{
-}
-
-XmlContent::~XmlContent()
-{
-}
-
-XmlOutputStream::~XmlOutputStream()
-{
-}
-
-XmlOutputStream&
-XmlOutputStream::operator<<(const XmlTag& tag)
-{
-    //std::cerr << "Trying to add tag " << tag.getName() << ". cached tag is "
-    //          << (void*) _cachedTag.get() << "\n";
-    if (_cachedTag.get() != 0) flush(false);
-    _cachedTag.reset(new XmlTag(tag));
-    _cachedContentType = XmlContent::AUTO;
-    //std::cerr << "Added tag " << _cachedTag->getName() << "\n";
-    return *this;
-}
-
-XmlOutputStream&
-XmlOutputStream::operator<<(const XmlAttribute& attribute)
-{
-    //std::cerr << "Adding attribute\n";
-    if (_cachedTag.get() == 0) {
-        throw vespalib::IllegalStateException("Cannot add attribute "
-                + attribute.getName() + ", as no tag is open");
-    }
-    _cachedAttributes.push_back(attribute);
-    return *this;
-}
-
-XmlOutputStream&
-XmlOutputStream::operator<<(const XmlEndTag&)
-{
-    //std::cerr << "Adding endtag\n";
-    if (_cachedTag.get()) {
-        flush(true);
-        _cachedContentType = XmlContent::ESCAPED;
-    } else if (_tagStack.empty()) {
-        throw vespalib::IllegalStateException("No open tags left to end");
-    } else {
-        for (uint32_t i=1; i<_tagStack.size(); ++i) {
-            _wrappedStream << _indent;
-        }
-        _wrappedStream << "</" << _tagStack.back() << ">";
-        _tagStack.pop_back();
-        if (!_tagStack.empty()) _wrappedStream << '\n';
-        _cachedContentType = XmlContent::ESCAPED;
-    }
-    return *this;
-}
-
-XmlOutputStream&
-XmlOutputStream::operator<<(const XmlContent& content)
-{
-    //std::cerr << "Adding content\n";
-    if (_cachedTag.get() == 0 && _tagStack.empty()) {
-        throw vespalib::IllegalStateException(
-                "No open tag to write content in");
-    }
-    if (_cachedTag.get() != 0) {
-        //std::cerr << "Content is '" << content.getContent() << "'\n";
-        if (content.getType() == XmlContent::AUTO) { // Do nothing.. Always ok
-        } else if (_cachedContentType == XmlContent::AUTO) {
-            _cachedContentType = content.getType();
-        } else if (_cachedContentType != content.getType()) {
-            throw vespalib::IllegalStateException(
-                "Have already added content of different type");
-        }
-        _cachedContent.push_back(content);
-    } else {
-        if (content.getType() == XmlContent::BASE64) {
-            throw vespalib::IllegalStateException(
-                "Cannot add Base64 encoded content after tag content");
-        }
-        for (uint32_t i=0; i<_tagStack.size(); ++i) {
-            _wrappedStream << _indent;
-        }
-        _wrappedStream << content.getContent() << '\n';
-    }
-    return *this;
-}
-
-XmlOutputStream&
-XmlOutputStream::operator<<(const XmlSerializable& serializable)
-{
-    //std::cerr << "Adding serializable\n";
-    serializable.printXml(*this);
-    return *this;
-}
-
-XmlOutputStream&
-XmlOutputStream::operator<<(const std::string& content)
-{
-    //std::cerr << "Adding content string\n";
-    return *this << XmlContent(content);
-}
-
-XmlOutputStream&
-XmlOutputStream::operator<<(char c)
-{
-    return *this << XmlContent(std::string(&c, 1));
-}
-
-XmlOutputStream&
-XmlOutputStream::operator<<(int32_t i)
-{
-    return *this << XmlContent(vespalib::make_string("%d", i));
-}
-
-XmlOutputStream&
-XmlOutputStream::operator<<(int64_t i)
-{
-    return *this << XmlContent(vespalib::make_string("%" PRId64, i));
-}
-
-XmlOutputStream&
-XmlOutputStream::operator<<(float f)
-{
-    return *this << XmlContent(vespalib::make_string("%g", f));
-}
-
-XmlOutputStream&
-XmlOutputStream::operator<<(double d)
-{
-    return *this << XmlContent(vespalib::make_string("%g", d));
-}
-
-void
-XmlOutputStream::flush(bool endTag)
-{
-    //std::cerr << "Flushing\n";
-    if (_cachedTag.get() == 0) {
-        throw vespalib::IllegalStateException("Cannot write non-existing tag");
-    }
-    for (uint32_t i=0; i<_tagStack.size(); ++i) {
-        _wrappedStream << _indent;
-    }
-    _wrappedStream << '<' << _cachedTag->getName();
-    for (std::list<XmlAttribute>::const_iterator it = _cachedAttributes.begin();
-         it != _cachedAttributes.end(); ++it)
-    {
-        _wrappedStream << ' ' << it->getName() << "=\""
-                       << xmlAttributeEscape(it->getValue()) << '"';
-    }
-    _cachedAttributes.clear();
-    if (_cachedContent.empty() && endTag) {
-        _wrappedStream << "/>\n";
-    } else if (_cachedContent.empty()) {
-        _wrappedStream << ">\n";
-        _tagStack.push_back(_cachedTag->getName());
-    } else {
-        if (_cachedContentType == XmlContent::AUTO) {
-            _cachedContentType = XmlContent::ESCAPED;
-            for (std::list<XmlContent>::const_iterator it
-                    = _cachedContent.begin(); it != _cachedContent.end(); ++it)
-            {
-                if (containsBinaryCharacters(it->getContent())) {
-                    _cachedContentType = XmlContent::BASE64;
-                    break;
-                }
-            }
-        }
-        if (_cachedContentType == XmlContent::BASE64) {
-            _wrappedStream << " binaryencoding=\"base64\"";
-        }
-        _wrappedStream << '>';
-        for (std::list<XmlContent>::const_iterator it = _cachedContent.begin();
-             it != _cachedContent.end(); ++it)
-        {
-            if (!endTag) {
-                _wrappedStream << '\n';
-                for (uint32_t i=0; i<=_tagStack.size(); ++i) {
-                    _wrappedStream << _indent;
-                }
-            }
-            switch (_cachedContentType) {
-                case XmlContent::ESCAPED: {
-                    writeEscaped(_wrappedStream, it->getContent());
-                    break;
-                }
-                case XmlContent::BASE64: {
-                    writeBase64Encoded(_wrappedStream, it->getContent());
-                    break;
-                }
-                default: assert(false);
-            }
-        }
-        _cachedContent.clear();
-        if (endTag) {
-            _wrappedStream << "</" << _cachedTag->getName() << ">\n";
-        } else {
-            _wrappedStream << '\n';
-            _tagStack.push_back(_cachedTag->getName());
-        }
-    }
-    _cachedTag.reset(0);
-}
-
-XmlTag::XmlTag(const XmlTag& tag)
-    : _name(tag._name),
-      _attributes(),
-      _content(),
-      _flags(tag._flags)
-{
-}
-
-XmlTag::~XmlTag() {}
-
-XmlTag::XmlTag(const std::string& name, XmlTagFlags flags)
-    : _name(name),
-      _attributes(),
-      _content(),
-      _flags(flags)
-{
-    if (_flags == CONVERT_ILLEGAL_CHARACTERS) {
-        convertToLegalName(_name);
-    }
-    if (!isLegalName(_name)) {
-        throw vespalib::IllegalArgumentException("Name '" + _name + "' contains "
-                "illegal XML characters and cannot be used as tag name");
-    }
-}
-
-XmlAttribute::XmlAttribute(const XmlAttribute& attribute)
-    : _name(attribute._name),
-      _value(attribute._value),
-      _next()
-{
-}
-
-XmlAttribute::XmlAttribute(const std::string& name, const char * value, uint32_t flags)
-    : _name(name),
-      _value(),
-      _next()
-{
-    vespalib::asciistream ost;
-    if (flags & HEX) ost << vespalib::hex << "0x";
-    ost << value;
-    _value = ost.str();
-    if (!isLegalName(name)) {
-        throw vespalib::IllegalArgumentException("Name '" + name + "' contains "
-                "illegal XML characters and cannot be used as attribute name");
-    }
-}
-
-XmlEndTag::XmlEndTag()
-{
-}
-
-XmlContent::XmlContent(Type type)
-    : _type(type),
-      _content(),
-      _nextContent(),
-      _nextTag()
-{
-}
-
-XmlContent::XmlContent()
-    : _type(AUTO),
-      _content(),
-      _nextContent(),
-      _nextTag()
-{
-}
-
-XmlContent::XmlContent(const XmlContent& content)
-    : _type(content._type),
-      _content(content._content),
-      _nextContent(),
-      _nextTag()
-{
-}
-
-XmlContent::XmlContent(const std::string& value)
-    : _type(AUTO),
-      _content(value),
-      _nextContent(),
-      _nextTag()
-{
-}
-
-XmlContentWrapper::XmlContentWrapper(const XmlContentWrapper& wrapper)
-    : XmlContent(wrapper)
-{
-}
-
-XmlContentWrapper::XmlContentWrapper(const char* value)
-    : XmlContent(std::string(value))
-{
-}
-
-XmlContentWrapper::XmlContentWrapper(const char* value, uint32_t size)
-    : XmlContent(std::string(value, size))
-{
-}
-
 std::string
 XmlSerializable::toXml(const std::string& indent) const
 {
@@ -452,22 +15,6 @@ XmlSerializable::toXml(const std::string& indent) const
     printXml(xos);
     return ost.str();
 }
-using CharP = char *;
-using ConstCharP = const char *;
-
-template XmlAttribute::XmlAttribute(const std::string &, const std::string &, unsigned int);
-template XmlAttribute::XmlAttribute(const std::string &, const vespalib::string &, unsigned int);
-template XmlAttribute::XmlAttribute(const std::string &, const vespalib::stringref &, unsigned int);
-template XmlAttribute::XmlAttribute(const std::string &, const CharP &, unsigned int);
-template XmlAttribute::XmlAttribute(const std::string &, const ConstCharP &, unsigned int);
-template XmlAttribute::XmlAttribute(const std::string &, const bool &, unsigned int);
-template XmlAttribute::XmlAttribute(const std::string &, const int16_t &, unsigned int);
-template XmlAttribute::XmlAttribute(const std::string &, const int32_t &, unsigned int);
-template XmlAttribute::XmlAttribute(const std::string &, const int64_t &, unsigned int);
-template XmlAttribute::XmlAttribute(const std::string &, const uint16_t &, unsigned int);
-template XmlAttribute::XmlAttribute(const std::string &, const uint32_t &, unsigned int);
-template XmlAttribute::XmlAttribute(const std::string &, const uint64_t &, unsigned int);
-template XmlAttribute::XmlAttribute(const std::string &, const double &, unsigned int);
 
 } // xml
 } // vespalib
diff --git a/staging_vespalib/src/vespa/vespalib/util/xmlserializable.h b/staging_vespalib/src/vespa/vespalib/util/xmlserializable.h
index b688c699d76..a4ddb12ed8d 100644
--- a/staging_vespalib/src/vespa/vespalib/util/xmlserializable.h
+++ b/staging_vespalib/src/vespa/vespalib/util/xmlserializable.h
@@ -1,166 +1,14 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-/**
- * @file xmlserializable.h
- * @ingroup util
- *
- * @brief Interfaces to be used for XML serialization.
- *
- * This file contains XML utility classes, to make XML serialization simple.
- * Rather than users writing their own XML, these tools let you define a tree
- * structure, and this library builds the XML for you. This ensures that you
- * write legal XML and that stuff that needs to be escaped is.
- * <p>
- * It defines a superclass for XML serializable classes, called XmlSerializable.
- * This is what classes that should be XML serializable will inherit.
- * <p>
- * When implementing the printXml() function in XmlSerializable, one will
- * use the various XML helper classes defined here to build a tree structure
- * creating the XML. These are: XmlTag, XmlEndTag, XmlAttribute, and XmlContent.
- * Some subclasses exist of XmlContent to facilitate various types of content.
- * <p>
- * The XmlOutputStream wraps a regular std::ostream. You write XML objects to it
- * and it is responsible for writing all the XML code. This way, the XML
- * serialization is done without interfering with regular output operators.
- * <p>
- * For example usage, refer to the unit test:
- * vespalib/tests/xmlserializable/xmlserializabletest.cpp
- *
- */
 
 #pragma once
 
-#include <iosfwd>
-#include <list>
-#include <memory>
+#include <string>
 
 namespace vespalib {
 namespace xml {
 
-class XmlAttribute;
-class XmlContent;
 class XmlOutputStream;
 
-bool isLegalName(const std::string& name);
-
-enum XmlTagFlags { NONE = 0, CONVERT_ILLEGAL_CHARACTERS = 1 };
-
-/**
- * @class document::XmlTag
- *
- * @brief Start a new tag with given name.
- */
-class XmlTag {
-    std::string _name;
-    std::unique_ptr<XmlAttribute> _attributes;
-    std::unique_ptr<XmlContent> _content;
-    XmlTagFlags _flags;
-public:
-    XmlTag(const XmlTag&);
-    XmlTag(const std::string& name, XmlTagFlags = NONE);
-    ~XmlTag();
-
-    const std::string& getName() const { return _name; }
-};
-
-/**
- * @class document::XmlEndTag
- *
- * @brief Indicates that current tag is closed.
- */
-class XmlEndTag {
-public:
-    XmlEndTag();
-};
-
-/**
- * @class document::XmlAttribute
- *
- * @brief Defined a single attribute within an XML tag.
- *
- * When adding an XML to an XML stream, the attribute will be added to the last
- * tag added. This can not be called after the last tag opened in the stream is
- * closed, so add all attributes before starting to add new XML child tags.
- */
-class XmlAttribute {
-    std::string _name;
-    std::string _value;
-    std::unique_ptr<XmlAttribute> _next;
-public:
-    enum Flag { NONE = 0x0, HEX = 0x1 };
-    XmlAttribute(const XmlAttribute&);
-    /** Add any value that can be written to an ostringstream. */
-    template<typename T>
-    XmlAttribute(const std::string& name, const T& value, uint32_t flags = NONE);
-    XmlAttribute(const std::string& name, const char * value, uint32_t flags = NONE);
-    ~XmlAttribute();
-
-    const std::string& getName() const { return _name; }
-    const std::string& getValue() const { return _value; }
-};
-
-
-/**
- * @class document::XmlContent
- *
- * XML content to be written to stream. By default it will autodetect whether to
- * escape or base64 encode content. XmlOutputStream functions taking primitives
- * will generate XmlContent instances.
- */
-class XmlContent {
-public:
-    enum Type { AUTO, ESCAPED, BASE64 };
-protected:
-    XmlContent(Type type);
-private:
-    Type _type;
-    std::string _content;
-    std::unique_ptr<XmlContent> _nextContent;
-    std::unique_ptr<XmlTag> _nextTag;
-
-public:
-    XmlContent();
-    XmlContent(const XmlContent&);
-    XmlContent(const std::string& value);
-    ~XmlContent();
-
-    Type getType() const { return _type; }
-    const std::string& getContent() const { return _content; }
-};
-
-/**
- * @class document::XmlEscapedContent
- *
- * Token used to tell that this content field should only be XML escaped.
- */
-class XmlEscapedContent : public XmlContent {
-public:
-    XmlEscapedContent() : XmlContent(ESCAPED) {}
-};
-
-/**
- * @class document::XmlBase64Content
- *
- * Token used to tell that this content field should always be base64 encoded.
- */
-class XmlBase64Content : public XmlContent {
-public:
-    XmlBase64Content() : XmlContent(BASE64) {}
-};
-
-/**
- * @class document::XmlContentWrapper
- *
- * A wrapper class for content that one doesn't want to copy or release
- * ownership of. This wrapper merely takes pointer to data, and assumes it
- * will stay alive as long as needed.
- */
-class XmlContentWrapper : public XmlContent {
-public:
-    XmlContentWrapper(const XmlContentWrapper&);
-    XmlContentWrapper(const char* value);
-    XmlContentWrapper(const char* value, uint32_t size);
-};
-
 /**
  * @class document::XmlSerializable
  *
@@ -178,51 +26,6 @@ public:
     virtual std::string toXml(const std::string& indent = "") const;
 };
 
-/**
- * @class document::XmlOutputStream
- *
- * @brief std::ostream wrapper, only accepting data that will become XML.
- *
- * After XmlEndTag() has been sent to the stream, the tag is guarantueed to have
- * been written. Call isFinalized() to ensure that you have closed all the tags
- * that have been opened. Within a tag, the stream will cache some information,
- * as more information might be required before knowing what to print.
- */
-class XmlOutputStream {
-    const std::string _indent;
-    std::ostream& _wrappedStream;
-    std::list<std::string> _tagStack;
-    std::unique_ptr<XmlTag> _cachedTag;
-    std::list<XmlAttribute> _cachedAttributes;
-    std::list<XmlContent> _cachedContent;
-    XmlContent::Type _cachedContentType;
-
-    void flush(bool endTag);
-
-public:
-
-    XmlOutputStream(std::ostream& ostream, const std::string& indent = "");
-    ~XmlOutputStream();
-
-    bool isFinalized() const
-        { return (_tagStack.empty() && _cachedTag.get() == 0); }
-
-    std::ostream& getWrappedStream() { return _wrappedStream; }
-
-    XmlOutputStream& operator<<(const XmlTag& tag);
-    XmlOutputStream& operator<<(const XmlAttribute& attribute);
-    XmlOutputStream& operator<<(const XmlEndTag& endtag);
-    XmlOutputStream& operator<<(const XmlContent& content);
-    XmlOutputStream& operator<<(const XmlSerializable& serializable);
-
-    XmlOutputStream& operator<<(const std::string& content);
-    XmlOutputStream& operator<<(char c);
-    XmlOutputStream& operator<<(int32_t i);
-    XmlOutputStream& operator<<(int64_t i);
-    XmlOutputStream& operator<<(float f);
-    XmlOutputStream& operator<<(double d);
-};
-
 } // xml
 
 // The XmlSerializable and XmlOutputStream is often used in header files
@@ -230,8 +33,8 @@ public:
 // vespalib namespace with all the other classes, use
 // "using namespace vespalib::xml" within your printXml functions
 
-typedef vespalib::xml::XmlOutputStream XmlOutputStream;
-typedef vespalib::xml::XmlSerializable XmlSerializable;
+using XmlSerializable = vespalib::xml::XmlSerializable;
+using XmlOutputStream = vespalib::xml::XmlOutputStream;
 
 } // vespalib
 
diff --git a/staging_vespalib/src/vespa/vespalib/util/xmlstream.cpp b/staging_vespalib/src/vespa/vespalib/util/xmlstream.cpp
new file mode 100644
index 00000000000..16fce61ddd1
--- /dev/null
+++ b/staging_vespalib/src/vespa/vespalib/util/xmlstream.cpp
@@ -0,0 +1,463 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "xmlstream.hpp"
+#include <vespa/vespalib/encoding/base64.h>
+#include <vespa/vespalib/stllike/asciistream.h>
+#include <vespa/vespalib/util/stringfmt.h>
+#include <cassert>
+
+namespace vespalib::xml {
+
+namespace {
+
+    std::vector<bool> getLegalIdentifierFirstCharacters() {
+        std::vector<bool> vec(256, false);
+        for (uint32_t i='a'; i<='z'; ++i) vec[i] = true;
+        for (uint32_t i='A'; i<='Z'; ++i) vec[i] = true;
+        vec[':'] = true;
+        vec['_'] = true;
+        return vec;
+    }
+
+    std::vector<bool> getLegalIdentifierCharacters() {
+        std::vector<bool> vec(getLegalIdentifierFirstCharacters());
+        vec['-'] = true;
+        vec['.'] = true;
+        for (uint32_t i='0'; i<='9'; ++i) {
+            vec[i] = true;
+        }
+        return vec;
+    }
+
+    std::vector<bool> getBinaryCharacters() {
+        std::vector<bool> vec(256, false);
+        for (uint32_t i=0; i<32; ++i) {
+            vec[i] = true;
+        }
+        vec['\t'] = false;
+        vec['\n'] = false;
+        vec['\r'] = false;
+        vec['\f'] = false;
+        return vec;
+    }
+
+    std::vector<bool> getEscapedXmlCharacters() {
+        std::vector<bool> vec(256, false);
+        for (uint32_t i=0; i<32; ++i) {
+            vec[i] = true;
+        }
+        vec['\n'] = false;
+        vec['<'] = true;
+        vec['>'] = true;
+        vec['&'] = true;
+        return vec;
+    }
+
+    std::vector<bool> legalIdentifierFirstChar(
+            getLegalIdentifierFirstCharacters());
+    std::vector<bool> legalIdentifierChars = getLegalIdentifierCharacters();
+    std::vector<bool> binaryChars = getBinaryCharacters();
+    std::vector<bool> escapedXmlChars = getEscapedXmlCharacters();
+
+    bool containsBinaryCharacters(const std::string& s) {
+        for (int i=0, n=s.size(); i<n; ++i) {
+            if (binaryChars[static_cast<uint8_t>(s[i])]) return true;
+        }
+        return false;
+    }
+
+    const std::string xmlAttributeEscape(const std::string& s) {
+        vespalib::asciistream ost;
+        for (uint32_t i=0, n=s.size(); i<n; ++i) {
+            if (s[i] == '"' || s[i] == '\n'
+                || escapedXmlChars[static_cast<uint8_t>(s[i])])
+            {
+                if      (s[i] == '<') ost << "&lt;";
+                else if (s[i] == '>') ost << "&gt;";
+                else if (s[i] == '&') ost << "&amp;";
+                else if (s[i] == '"') ost << "&quot;";
+                else {
+                    ost << "&#" << (int) s[i] << ";";
+                }
+            } else {
+                ost << s[i];
+            }
+        }
+        return ost.str();
+    }
+
+    void writeEscaped(std::ostream& out, const std::string& s) {
+        for (uint32_t i=0, n=s.size(); i<n; ++i) {
+            if (escapedXmlChars[static_cast<uint8_t>(s[i])]) {
+                if      (s[i] == '<') out << "&lt;";
+                else if (s[i] == '>') out << "&gt;";
+                else if (s[i] == '&') out << "&amp;";
+                else {
+                    out << "&#" << (int) s[i] << ";";
+                }
+            } else {
+                out << s[i];
+            }
+        }
+    }
+
+    void writeBase64Encoded(std::ostream& out, const std::string& s) {
+        out << vespalib::Base64::encode(&s[0], s.size());
+    }
+}
+
+bool isLegalName(const std::string& name) {
+    if (name.size() == 0) return false;
+    if (!legalIdentifierFirstChar[static_cast<uint8_t>(name[0])]) return false;
+    for (int i=1, n=name.size(); i<n; ++i) {
+        if (!legalIdentifierChars[static_cast<uint8_t>(name[i])]) return false;
+    }
+    return true;
+}
+
+void convertToLegalName(std::string& name) {
+    if (name.size() == 0) {
+        name == "__no_name__";
+    } else {
+        if (!legalIdentifierFirstChar[static_cast<uint8_t>(name[0])]) {
+            name[0] = '_';
+        }
+        for (int i=1, n=name.size(); i<n; ++i) {
+            if (!legalIdentifierChars[static_cast<uint8_t>(name[i])]) {
+                name[i] = '_';
+            }
+        }
+    }
+}
+
+XmlOutputStream::XmlOutputStream(std::ostream& ostream,
+                                 const std::string& indent)
+    : _indent(indent),
+      _wrappedStream(ostream),
+      _tagStack(),
+      _cachedTag(),
+      _cachedAttributes(),
+      _cachedContent()
+{
+}
+
+XmlAttribute::~XmlAttribute()
+{
+}
+
+XmlContent::~XmlContent()
+{
+}
+
+XmlOutputStream::~XmlOutputStream()
+{
+}
+
+XmlOutputStream&
+XmlOutputStream::operator<<(const XmlTag& tag)
+{
+    //std::cerr << "Trying to add tag " << tag.getName() << ". cached tag is "
+    //          << (void*) _cachedTag.get() << "\n";
+    if (_cachedTag.get() != 0) flush(false);
+    _cachedTag.reset(new XmlTag(tag));
+    _cachedContentType = XmlContent::AUTO;
+    //std::cerr << "Added tag " << _cachedTag->getName() << "\n";
+    return *this;
+}
+
+XmlOutputStream&
+XmlOutputStream::operator<<(const XmlAttribute& attribute)
+{
+    //std::cerr << "Adding attribute\n";
+    if (_cachedTag.get() == 0) {
+        throw vespalib::IllegalStateException("Cannot add attribute "
+                + attribute.getName() + ", as no tag is open");
+    }
+    _cachedAttributes.push_back(attribute);
+    return *this;
+}
+
+XmlOutputStream&
+XmlOutputStream::operator<<(const XmlEndTag&)
+{
+    //std::cerr << "Adding endtag\n";
+    if (_cachedTag.get()) {
+        flush(true);
+        _cachedContentType = XmlContent::ESCAPED;
+    } else if (_tagStack.empty()) {
+        throw vespalib::IllegalStateException("No open tags left to end");
+    } else {
+        for (uint32_t i=1; i<_tagStack.size(); ++i) {
+            _wrappedStream << _indent;
+        }
+        _wrappedStream << "</" << _tagStack.back() << ">";
+        _tagStack.pop_back();
+        if (!_tagStack.empty()) _wrappedStream << '\n';
+        _cachedContentType = XmlContent::ESCAPED;
+    }
+    return *this;
+}
+
+XmlOutputStream&
+XmlOutputStream::operator<<(const XmlContent& content)
+{
+    //std::cerr << "Adding content\n";
+    if (_cachedTag.get() == 0 && _tagStack.empty()) {
+        throw vespalib::IllegalStateException(
+                "No open tag to write content in");
+    }
+    if (_cachedTag.get() != 0) {
+        //std::cerr << "Content is '" << content.getContent() << "'\n";
+        if (content.getType() == XmlContent::AUTO) { // Do nothing.. Always ok
+        } else if (_cachedContentType == XmlContent::AUTO) {
+            _cachedContentType = content.getType();
+        } else if (_cachedContentType != content.getType()) {
+            throw vespalib::IllegalStateException(
+                "Have already added content of different type");
+        }
+        _cachedContent.push_back(content);
+    } else {
+        if (content.getType() == XmlContent::BASE64) {
+            throw vespalib::IllegalStateException(
+                "Cannot add Base64 encoded content after tag content");
+        }
+        for (uint32_t i=0; i<_tagStack.size(); ++i) {
+            _wrappedStream << _indent;
+        }
+        _wrappedStream << content.getContent() << '\n';
+    }
+    return *this;
+}
+
+XmlOutputStream&
+XmlOutputStream::operator<<(const XmlSerializable& serializable)
+{
+    //std::cerr << "Adding serializable\n";
+    serializable.printXml(*this);
+    return *this;
+}
+
+XmlOutputStream&
+XmlOutputStream::operator<<(const std::string& content)
+{
+    //std::cerr << "Adding content string\n";
+    return *this << XmlContent(content);
+}
+
+XmlOutputStream&
+XmlOutputStream::operator<<(char c)
+{
+    return *this << XmlContent(std::string(&c, 1));
+}
+
+XmlOutputStream&
+XmlOutputStream::operator<<(int32_t i)
+{
+    return *this << XmlContent(vespalib::make_string("%d", i));
+}
+
+XmlOutputStream&
+XmlOutputStream::operator<<(int64_t i)
+{
+    return *this << XmlContent(vespalib::make_string("%" PRId64, i));
+}
+
+XmlOutputStream&
+XmlOutputStream::operator<<(float f)
+{
+    return *this << XmlContent(vespalib::make_string("%g", f));
+}
+
+XmlOutputStream&
+XmlOutputStream::operator<<(double d)
+{
+    return *this << XmlContent(vespalib::make_string("%g", d));
+}
+
+void
+XmlOutputStream::flush(bool endTag)
+{
+    //std::cerr << "Flushing\n";
+    if (_cachedTag.get() == 0) {
+        throw vespalib::IllegalStateException("Cannot write non-existing tag");
+    }
+    for (uint32_t i=0; i<_tagStack.size(); ++i) {
+        _wrappedStream << _indent;
+    }
+    _wrappedStream << '<' << _cachedTag->getName();
+    for (std::list<XmlAttribute>::const_iterator it = _cachedAttributes.begin();
+         it != _cachedAttributes.end(); ++it)
+    {
+        _wrappedStream << ' ' << it->getName() << "=\""
+                       << xmlAttributeEscape(it->getValue()) << '"';
+    }
+    _cachedAttributes.clear();
+    if (_cachedContent.empty() && endTag) {
+        _wrappedStream << "/>\n";
+    } else if (_cachedContent.empty()) {
+        _wrappedStream << ">\n";
+        _tagStack.push_back(_cachedTag->getName());
+    } else {
+        if (_cachedContentType == XmlContent::AUTO) {
+            _cachedContentType = XmlContent::ESCAPED;
+            for (std::list<XmlContent>::const_iterator it
+                    = _cachedContent.begin(); it != _cachedContent.end(); ++it)
+            {
+                if (containsBinaryCharacters(it->getContent())) {
+                    _cachedContentType = XmlContent::BASE64;
+                    break;
+                }
+            }
+        }
+        if (_cachedContentType == XmlContent::BASE64) {
+            _wrappedStream << " binaryencoding=\"base64\"";
+        }
+        _wrappedStream << '>';
+        for (std::list<XmlContent>::const_iterator it = _cachedContent.begin();
+             it != _cachedContent.end(); ++it)
+        {
+            if (!endTag) {
+                _wrappedStream << '\n';
+                for (uint32_t i=0; i<=_tagStack.size(); ++i) {
+                    _wrappedStream << _indent;
+                }
+            }
+            switch (_cachedContentType) {
+                case XmlContent::ESCAPED: {
+                    writeEscaped(_wrappedStream, it->getContent());
+                    break;
+                }
+                case XmlContent::BASE64: {
+                    writeBase64Encoded(_wrappedStream, it->getContent());
+                    break;
+                }
+                default: assert(false);
+            }
+        }
+        _cachedContent.clear();
+        if (endTag) {
+            _wrappedStream << "</" << _cachedTag->getName() << ">\n";
+        } else {
+            _wrappedStream << '\n';
+            _tagStack.push_back(_cachedTag->getName());
+        }
+    }
+    _cachedTag.reset(0);
+}
+
+XmlTag::XmlTag(const XmlTag& tag)
+    : _name(tag._name),
+      _attributes(),
+      _content(),
+      _flags(tag._flags)
+{
+}
+
+XmlTag::~XmlTag() {}
+
+XmlTag::XmlTag(const std::string& name, XmlTagFlags flags)
+    : _name(name),
+      _attributes(),
+      _content(),
+      _flags(flags)
+{
+    if (_flags == CONVERT_ILLEGAL_CHARACTERS) {
+        convertToLegalName(_name);
+    }
+    if (!isLegalName(_name)) {
+        throw vespalib::IllegalArgumentException("Name '" + _name + "' contains "
+                "illegal XML characters and cannot be used as tag name");
+    }
+}
+
+XmlAttribute::XmlAttribute(const XmlAttribute& attribute)
+    : _name(attribute._name),
+      _value(attribute._value),
+      _next()
+{
+}
+
+XmlAttribute::XmlAttribute(const std::string& name, const char * value, uint32_t flags)
+    : _name(name),
+      _value(),
+      _next()
+{
+    vespalib::asciistream ost;
+    if (flags & HEX) ost << vespalib::hex << "0x";
+    ost << value;
+    _value = ost.str();
+    if (!isLegalName(name)) {
+        throw vespalib::IllegalArgumentException("Name '" + name + "' contains "
+                "illegal XML characters and cannot be used as attribute name");
+    }
+}
+
+XmlEndTag::XmlEndTag()
+{
+}
+
+XmlContent::XmlContent(Type type)
+    : _type(type),
+      _content(),
+      _nextContent(),
+      _nextTag()
+{
+}
+
+XmlContent::XmlContent()
+    : _type(AUTO),
+      _content(),
+      _nextContent(),
+      _nextTag()
+{
+}
+
+XmlContent::XmlContent(const XmlContent& content)
+    : _type(content._type),
+      _content(content._content),
+      _nextContent(),
+      _nextTag()
+{
+}
+
+XmlContent::XmlContent(const std::string& value)
+    : _type(AUTO),
+      _content(value),
+      _nextContent(),
+      _nextTag()
+{
+}
+
+XmlContentWrapper::XmlContentWrapper(const XmlContentWrapper& wrapper)
+    : XmlContent(wrapper)
+{
+}
+
+XmlContentWrapper::XmlContentWrapper(const char* value)
+    : XmlContent(std::string(value))
+{
+}
+
+XmlContentWrapper::XmlContentWrapper(const char* value, uint32_t size)
+    : XmlContent(std::string(value, size))
+{
+}
+
+using CharP = char *;
+using ConstCharP = const char *;
+
+template XmlAttribute::XmlAttribute(const std::string &, const std::string &, unsigned int);
+template XmlAttribute::XmlAttribute(const std::string &, const vespalib::string &, unsigned int);
+template XmlAttribute::XmlAttribute(const std::string &, const vespalib::stringref &, unsigned int);
+template XmlAttribute::XmlAttribute(const std::string &, const CharP &, unsigned int);
+template XmlAttribute::XmlAttribute(const std::string &, const ConstCharP &, unsigned int);
+template XmlAttribute::XmlAttribute(const std::string &, const bool &, unsigned int);
+template XmlAttribute::XmlAttribute(const std::string &, const int16_t &, unsigned int);
+template XmlAttribute::XmlAttribute(const std::string &, const int32_t &, unsigned int);
+template XmlAttribute::XmlAttribute(const std::string &, const int64_t &, unsigned int);
+template XmlAttribute::XmlAttribute(const std::string &, const uint16_t &, unsigned int);
+template XmlAttribute::XmlAttribute(const std::string &, const uint32_t &, unsigned int);
+template XmlAttribute::XmlAttribute(const std::string &, const uint64_t &, unsigned int);
+template XmlAttribute::XmlAttribute(const std::string &, const double &, unsigned int);
+
+}
diff --git a/staging_vespalib/src/vespa/vespalib/util/xmlstream.h b/staging_vespalib/src/vespa/vespalib/util/xmlstream.h
new file mode 100644
index 00000000000..8ed781e57c3
--- /dev/null
+++ b/staging_vespalib/src/vespa/vespalib/util/xmlstream.h
@@ -0,0 +1,210 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * @file xmlserializable.h
+ * @ingroup util
+ *
+ * @brief Interfaces to be used for XML serialization.
+ *
+ * This file contains XML utility classes, to make XML serialization simple.
+ * Rather than users writing their own XML, these tools let you define a tree
+ * structure, and this library builds the XML for you. This ensures that you
+ * write legal XML and that stuff that needs to be escaped is.
+ * <p>
+ * It defines a superclass for XML serializable classes, called XmlSerializable.
+ * This is what classes that should be XML serializable will inherit.
+ * <p>
+ * When implementing the printXml() function in XmlSerializable, one will
+ * use the various XML helper classes defined here to build a tree structure
+ * creating the XML. These are: XmlTag, XmlEndTag, XmlAttribute, and XmlContent.
+ * Some subclasses exist of XmlContent to facilitate various types of content.
+ * <p>
+ * The XmlOutputStream wraps a regular std::ostream. You write XML objects to it
+ * and it is responsible for writing all the XML code. This way, the XML
+ * serialization is done without interfering with regular output operators.
+ * <p>
+ * For example usage, refer to the unit test:
+ * vespalib/tests/xmlserializable/xmlserializabletest.cpp
+ *
+ */
+
+#pragma once
+
+#include "xmlserializable.h"
+#include <iosfwd>
+#include <list>
+#include <memory>
+
+namespace vespalib::xml {
+
+class XmlAttribute;
+class XmlContent;
+class XmlOutputStream;
+
+bool isLegalName(const std::string& name);
+
+enum XmlTagFlags { NONE = 0, CONVERT_ILLEGAL_CHARACTERS = 1 };
+
+/**
+ * @class document::XmlTag
+ *
+ * @brief Start a new tag with given name.
+ */
+class XmlTag {
+    std::string _name;
+    std::unique_ptr<XmlAttribute> _attributes;
+    std::unique_ptr<XmlContent> _content;
+    XmlTagFlags _flags;
+public:
+    XmlTag(const XmlTag&);
+    XmlTag(const std::string& name, XmlTagFlags = NONE);
+    ~XmlTag();
+
+    const std::string& getName() const { return _name; }
+};
+
+/**
+ * @class document::XmlEndTag
+ *
+ * @brief Indicates that current tag is closed.
+ */
+class XmlEndTag {
+public:
+    XmlEndTag();
+};
+
+/**
+ * @class document::XmlAttribute
+ *
+ * @brief Defined a single attribute within an XML tag.
+ *
+ * When adding an XML to an XML stream, the attribute will be added to the last
+ * tag added. This can not be called after the last tag opened in the stream is
+ * closed, so add all attributes before starting to add new XML child tags.
+ */
+class XmlAttribute {
+    std::string _name;
+    std::string _value;
+    std::unique_ptr<XmlAttribute> _next;
+public:
+    enum Flag { NONE = 0x0, HEX = 0x1 };
+    XmlAttribute(const XmlAttribute&);
+    /** Add any value that can be written to an ostringstream. */
+    template<typename T>
+    XmlAttribute(const std::string& name, const T& value, uint32_t flags = NONE);
+    XmlAttribute(const std::string& name, const char * value, uint32_t flags = NONE);
+    ~XmlAttribute();
+
+    const std::string& getName() const { return _name; }
+    const std::string& getValue() const { return _value; }
+};
+
+
+/**
+ * @class document::XmlContent
+ *
+ * XML content to be written to stream. By default it will autodetect whether to
+ * escape or base64 encode content. XmlOutputStream functions taking primitives
+ * will generate XmlContent instances.
+ */
+class XmlContent {
+public:
+    enum Type { AUTO, ESCAPED, BASE64 };
+protected:
+    XmlContent(Type type);
+private:
+    Type _type;
+    std::string _content;
+    std::unique_ptr<XmlContent> _nextContent;
+    std::unique_ptr<XmlTag> _nextTag;
+
+public:
+    XmlContent();
+    XmlContent(const XmlContent&);
+    XmlContent(const std::string& value);
+    ~XmlContent();
+
+    Type getType() const { return _type; }
+    const std::string& getContent() const { return _content; }
+};
+
+/**
+ * @class document::XmlEscapedContent
+ *
+ * Token used to tell that this content field should only be XML escaped.
+ */
+class XmlEscapedContent : public XmlContent {
+public:
+    XmlEscapedContent() : XmlContent(ESCAPED) {}
+};
+
+/**
+ * @class document::XmlBase64Content
+ *
+ * Token used to tell that this content field should always be base64 encoded.
+ */
+class XmlBase64Content : public XmlContent {
+public:
+    XmlBase64Content() : XmlContent(BASE64) {}
+};
+
+/**
+ * @class document::XmlContentWrapper
+ *
+ * A wrapper class for content that one doesn't want to copy or release
+ * ownership of. This wrapper merely takes pointer to data, and assumes it
+ * will stay alive as long as needed.
+ */
+class XmlContentWrapper : public XmlContent {
+public:
+    XmlContentWrapper(const XmlContentWrapper&);
+    XmlContentWrapper(const char* value);
+    XmlContentWrapper(const char* value, uint32_t size);
+};
+
+/**
+ * @class document::XmlOutputStream
+ *
+ * @brief std::ostream wrapper, only accepting data that will become XML.
+ *
+ * After XmlEndTag() has been sent to the stream, the tag is guarantueed to have
+ * been written. Call isFinalized() to ensure that you have closed all the tags
+ * that have been opened. Within a tag, the stream will cache some information,
+ * as more information might be required before knowing what to print.
+ */
+class XmlOutputStream {
+    const std::string _indent;
+    std::ostream& _wrappedStream;
+    std::list<std::string> _tagStack;
+    std::unique_ptr<XmlTag> _cachedTag;
+    std::list<XmlAttribute> _cachedAttributes;
+    std::list<XmlContent> _cachedContent;
+    XmlContent::Type _cachedContentType;
+
+    void flush(bool endTag);
+
+public:
+
+    XmlOutputStream(std::ostream& ostream, const std::string& indent = "");
+    ~XmlOutputStream();
+
+    bool isFinalized() const
+        { return (_tagStack.empty() && _cachedTag.get() == 0); }
+
+    std::ostream& getWrappedStream() { return _wrappedStream; }
+
+    XmlOutputStream& operator<<(const XmlTag& tag);
+    XmlOutputStream& operator<<(const XmlAttribute& attribute);
+    XmlOutputStream& operator<<(const XmlEndTag& endtag);
+    XmlOutputStream& operator<<(const XmlContent& content);
+    XmlOutputStream& operator<<(const XmlSerializable& serializable);
+
+    XmlOutputStream& operator<<(const std::string& content);
+    XmlOutputStream& operator<<(char c);
+    XmlOutputStream& operator<<(int32_t i);
+    XmlOutputStream& operator<<(int64_t i);
+    XmlOutputStream& operator<<(float f);
+    XmlOutputStream& operator<<(double d);
+};
+
+}
+
diff --git a/staging_vespalib/src/vespa/vespalib/util/xmlserializable.hpp b/staging_vespalib/src/vespa/vespalib/util/xmlstream.hpp
index 7fcc964d8f1..243efe9e94b 100644
--- a/staging_vespalib/src/vespa/vespalib/util/xmlserializable.hpp
+++ b/staging_vespalib/src/vespa/vespalib/util/xmlstream.hpp
@@ -2,12 +2,11 @@
 
 #pragma once
 
-#include "xmlserializable.h"
-#include <sstream>
+#include "xmlstream.h"
 #include <vespa/vespalib/util/exceptions.h>
+#include <sstream>
 
-namespace vespalib {
-namespace xml {
+namespace vespalib::xml {
 
 template<typename T>
 XmlAttribute::XmlAttribute(const std::string& name, const T& value, uint32_t flags)
@@ -20,10 +19,9 @@ XmlAttribute::XmlAttribute(const std::string& name, const T& value, uint32_t fla
     ost << value;
     _value = ost.str();
     if (!isLegalName(name)) {
-        throw vespalib::IllegalArgumentException("Name '" + name + "' contains "
+        throw IllegalArgumentException("Name '" + name + "' contains "
                 "illegal XML characters and cannot be used as attribute name");
     }
 }
 
 }
-}
diff --git a/storage/src/tests/bucketdb/bucketmanagertest.cpp b/storage/src/tests/bucketdb/bucketmanagertest.cpp
index f9ae4f57fa5..227dd40b34d 100644
--- a/storage/src/tests/bucketdb/bucketmanagertest.cpp
+++ b/storage/src/tests/bucketdb/bucketmanagertest.cpp
@@ -16,6 +16,7 @@
 #include <tests/common/testhelper.h>
 #include <vespa/vespalib/io/fileutil.h>
 #include <vespa/vespalib/testkit/testapp.h>
+#include <vespa/vespalib/util/stringfmt.h>
 #include <vespa/config/helper/configgetter.hpp>
 #include <future>
 
diff --git a/storage/src/tests/persistence/filestorage/filestormanagertest.cpp b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp
index e2f41aa5b3e..52b976586e8 100644
--- a/storage/src/tests/persistence/filestorage/filestormanagertest.cpp
+++ b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp
@@ -11,6 +11,7 @@
 #include <vespa/vdslib/container/mutabledocumentlist.h>
 #include <vespa/vdstestlib/cppunit/macros.h>
 #include <vespa/vespalib/io/fileutil.h>
+#include <vespa/vespalib/util/stringfmt.h>
 #include <vespa/storageapi/message/bucket.h>
 #include <vespa/storageapi/message/bucketsplitting.h>
 #include <vespa/storageapi/message/multioperation.h>
diff --git a/storage/src/tests/persistence/splitbitdetectortest.cpp b/storage/src/tests/persistence/splitbitdetectortest.cpp
index 3251dd9a9a8..47efaaa371e 100644
--- a/storage/src/tests/persistence/splitbitdetectortest.cpp
+++ b/storage/src/tests/persistence/splitbitdetectortest.cpp
@@ -1,7 +1,7 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include <vespa/vdstestlib/cppunit/macros.h>
-#include <fstream>
+#include <vespa/vespalib/util/stringfmt.h>
 #include <vespa/storage/persistence/splitbitdetector.h>
 #include <vespa/vespalib/io/fileutil.h>
 #include <vespa/persistence/dummyimpl/dummypersistence.h>
diff --git a/storage/src/tests/storageserver/documentapiconvertertest.cpp b/storage/src/tests/storageserver/documentapiconvertertest.cpp
index 1f1198efc3c..26317465b5a 100644
--- a/storage/src/tests/storageserver/documentapiconvertertest.cpp
+++ b/storage/src/tests/storageserver/documentapiconvertertest.cpp
@@ -2,14 +2,15 @@
 
 #include <vespa/document/base/testdocrepo.h>
 #include <cppunit/extensions/HelperMacros.h>
-#include <vespa/document/fieldvalue/document.h>
-#include <vespa/documentapi/documentapi.h>
-#include <vespa/messagebus/emptyreply.h>
 #include <vespa/storage/storageserver/documentapiconverter.h>
 #include <vespa/storageapi/message/batch.h>
 #include <vespa/storageapi/message/datagram.h>
 #include <vespa/storageapi/message/multioperation.h>
 #include <vespa/storageapi/message/persistence.h>
+#include <vespa/documentapi/documentapi.h>
+#include <vespa/messagebus/emptyreply.h>
+#include <vespa/document/datatype/documenttype.h>
+#include <vespa/document/bucket/bucketidfactory.h>
 #include <vespa/vespalib/testkit/test_kit.h>
 
 using document::DataType;
diff --git a/storage/src/vespa/storage/bucketdb/bucketmanager.cpp b/storage/src/vespa/storage/bucketdb/bucketmanager.cpp
index 0570d0a3e27..79150c160d5 100644
--- a/storage/src/vespa/storage/bucketdb/bucketmanager.cpp
+++ b/storage/src/vespa/storage/bucketdb/bucketmanager.cpp
@@ -15,11 +15,12 @@
 #include <vespa/storageapi/message/state.h>
 #include <vespa/storageapi/message/bucketsplitting.h>
 #include <vespa/storageapi/message/stat.h>
+#include <vespa/vespalib/util/stringfmt.h>
 #include <vespa/vespalib/stllike/hash_map.hpp>
 #include <vespa/config/config.h>
 #include <unordered_map>
-#include <vespa/log/log.h>
 
+#include <vespa/log/bufferedlogger.h>
 LOG_SETUP(".storage.bucketdb.manager");
 
 namespace storage {
diff --git a/storage/src/vespa/storage/bucketdb/bucketmanager.h b/storage/src/vespa/storage/bucketdb/bucketmanager.h
index d3a939d90c5..09cbc60b58e 100644
--- a/storage/src/vespa/storage/bucketdb/bucketmanager.h
+++ b/storage/src/vespa/storage/bucketdb/bucketmanager.h
@@ -25,6 +25,7 @@
 #include <vespa/storageapi/message/bucket.h>
 #include <vespa/config/subscription/configuri.h>
 #include <unordered_set>
+#include <list>
 
 namespace storage {
 
diff --git a/storage/src/vespa/storage/bucketdb/distribution_hash_normalizer.cpp b/storage/src/vespa/storage/bucketdb/distribution_hash_normalizer.cpp
index 80f82e65a9a..39e45e1b541 100644
--- a/storage/src/vespa/storage/bucketdb/distribution_hash_normalizer.cpp
+++ b/storage/src/vespa/storage/bucketdb/distribution_hash_normalizer.cpp
@@ -1,6 +1,5 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 #include "distribution_hash_normalizer.h"
-#include <vespa/log/log.h>
 #include <vespa/vespalib/stllike/asciistream.h>
 #include <boost/spirit/include/qi.hpp>
 #include <boost/spirit/include/phoenix_core.hpp>
@@ -13,6 +12,7 @@
 #include <iterator>
 #include <functional>
 
+#include <vespa/log/bufferedlogger.h>
 LOG_SETUP(".storage.bucketdb.distribution_hash_normalizer");
 
 // TODO
diff --git a/storage/src/vespa/storage/bucketdb/storagebucketdbinitializer.cpp b/storage/src/vespa/storage/bucketdb/storagebucketdbinitializer.cpp
index 178731469ba..49022db523b 100644
--- a/storage/src/vespa/storage/bucketdb/storagebucketdbinitializer.cpp
+++ b/storage/src/vespa/storage/bucketdb/storagebucketdbinitializer.cpp
@@ -13,7 +13,7 @@
 #include <vespa/config/helper/configgetter.hpp>
 #include <iomanip>
 
-#include <vespa/log/log.h>
+#include <vespa/log/bufferedlogger.h>
 LOG_SETUP(".storage.bucketdb.initializer");
 
 namespace storage {
diff --git a/storage/src/vespa/storage/bucketdb/storagebucketdbinitializer.h b/storage/src/vespa/storage/bucketdb/storagebucketdbinitializer.h
index f56b45c71b3..92c00c4420a 100644
--- a/storage/src/vespa/storage/bucketdb/storagebucketdbinitializer.h
+++ b/storage/src/vespa/storage/bucketdb/storagebucketdbinitializer.h
@@ -50,6 +50,7 @@
 #include <vespa/vespalib/util/sync.h>
 #include <vespa/vdslib/state/nodestate.h>
 #include <vespa/config/subscription/configuri.h>
+#include <list>
 
 namespace storage {
 
diff --git a/storage/src/vespa/storage/bucketmover/bucketmover.cpp b/storage/src/vespa/storage/bucketmover/bucketmover.cpp
index e4ddc3cb5a7..b38c061de44 100644
--- a/storage/src/vespa/storage/bucketmover/bucketmover.cpp
+++ b/storage/src/vespa/storage/bucketmover/bucketmover.cpp
@@ -6,11 +6,12 @@
 #include <vespa/storage/common/bucketmessages.h>
 #include <vespa/storage/common/nodestateupdater.h>
 #include <vespa/storage/storageutil/log.h>
+#include <vespa/vespalib/util/stringfmt.h>
 
+#include <vespa/log/bufferedlogger.h>
 LOG_SETUP(".bucketmover");
 
-namespace storage {
-namespace bucketmover {
+namespace storage::bucketmover {
 
 BucketMover::BucketMover(const config::ConfigUri & configUri,
                          ServiceLayerComponentRegister& reg)
@@ -525,5 +526,4 @@ BucketMover::printRunStatisticsHtml(std::ostream& out,
     rs.print(out, true, "");
 }
 
-} // bucketmover
-} // storage
+}
diff --git a/storage/src/vespa/storage/common/hostreporter/cpureporter.cpp b/storage/src/vespa/storage/common/hostreporter/cpureporter.cpp
index 66eb891840e..76a38e3e730 100644
--- a/storage/src/vespa/storage/common/hostreporter/cpureporter.cpp
+++ b/storage/src/vespa/storage/common/hostreporter/cpureporter.cpp
@@ -1,13 +1,10 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 #include "cpureporter.h"
 #include "kernelmetrictool.h"
-
 #include <vespa/vespalib/io/fileutil.h>
-#include <vespa/vespalib/stllike/string.h>
 #include <vespa/vespalib/text/stringtokenizer.h>
-#include <array>
 
-#include <vespa/log/log.h>
+#include <vespa/log/bufferedlogger.h>
 LOG_SETUP(".cpureporter");
 
 namespace storage {
diff --git a/storage/src/vespa/storage/common/statusmetricconsumer.cpp b/storage/src/vespa/storage/common/statusmetricconsumer.cpp
index 3111508fddc..8cb5c9018f4 100644
--- a/storage/src/vespa/storage/common/statusmetricconsumer.cpp
+++ b/storage/src/vespa/storage/common/statusmetricconsumer.cpp
@@ -1,20 +1,19 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "statusmetricconsumer.h"
-
+#include <vespa/storageframework/generic/memory/memorymanagerinterface.h>
+#include <vespa/storageframework/generic/status/htmlstatusreporter.h>
 #include <boost/assign.hpp>
 #include <boost/lexical_cast.hpp>
-#include <vespa/log/log.h>
 #include <vespa/metrics/printutils.h>
 #include <vespa/metrics/jsonwriter.h>
 #include <vespa/metrics/textwriter.h>
 #include <vespa/metrics/xmlwriter.h>
 #include <vespa/storageapi/messageapi/storagemessage.h>
 #include <vespa/vespalib/stllike/asciistream.h>
-#include <vespa/storageframework/generic/memory/memorymanagerinterface.h>
-#include <vespa/storageframework/generic/status/htmlstatusreporter.h>
-
+#include <vespa/vespalib/util/xmlstream.h>
 
+#include <vespa/log/log.h>
 LOG_SETUP(".status.metricreporter");
 
 namespace storage {
diff --git a/storage/src/vespa/storage/common/storagelink.cpp b/storage/src/vespa/storage/common/storagelink.cpp
index d3b6370347b..50097b132f6 100644
--- a/storage/src/vespa/storage/common/storagelink.cpp
+++ b/storage/src/vespa/storage/common/storagelink.cpp
@@ -5,7 +5,7 @@
 #include <vespa/vespalib/util/backtrace.h>
 #include <sstream>
 
-#include <vespa/log/log.h>
+#include <vespa/log/bufferedlogger.h>
 LOG_SETUP(".application.link");
 
 using std::shared_ptr;
diff --git a/storage/src/vespa/storage/config/distributorconfiguration.cpp b/storage/src/vespa/storage/config/distributorconfiguration.cpp
index 10d135ccf2d..57e8c55c23b 100644
--- a/storage/src/vespa/storage/config/distributorconfiguration.cpp
+++ b/storage/src/vespa/storage/config/distributorconfiguration.cpp
@@ -3,6 +3,7 @@
 #include <vespa/document/select/parser.h>
 #include <vespa/document/select/traversingvisitor.h>
 #include <vespa/vespalib/util/exceptions.h>
+#include <sstream>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".distributorconfiguration");
diff --git a/storage/src/vespa/storage/distributor/bucketdbupdater.cpp b/storage/src/vespa/storage/distributor/bucketdbupdater.cpp
index 068f42f8b31..9051bbac0f0 100644
--- a/storage/src/vespa/storage/distributor/bucketdbupdater.cpp
+++ b/storage/src/vespa/storage/distributor/bucketdbupdater.cpp
@@ -7,8 +7,9 @@
 #include <vespa/storageapi/message/persistence.h>
 #include <vespa/storageapi/message/removelocation.h>
 #include <vespa/storageapi/message/multioperation.h>
+#include <vespa/vespalib/util/xmlstream.h>
 
-#include <vespa/log/log.h>
+#include <vespa/log/bufferedlogger.h>
 LOG_SETUP(".distributor.bucketdb.updater");
 
 using storage::lib::Node;
diff --git a/storage/src/vespa/storage/distributor/bucketdbupdater.h b/storage/src/vespa/storage/distributor/bucketdbupdater.h
index 3f0989e9b4f..3bd86fc3db6 100644
--- a/storage/src/vespa/storage/distributor/bucketdbupdater.h
+++ b/storage/src/vespa/storage/distributor/bucketdbupdater.h
@@ -17,6 +17,7 @@
 #include <vespa/storageapi/messageapi/messagehandler.h>
 #include <set>
 #include <deque>
+#include <list>
 
 namespace storage::distributor {
 
diff --git a/storage/src/vespa/storage/distributor/messageguard.h b/storage/src/vespa/storage/distributor/messageguard.h
index 8d549965136..2944cc20c70 100644
--- a/storage/src/vespa/storage/distributor/messageguard.h
+++ b/storage/src/vespa/storage/distributor/messageguard.h
@@ -1,8 +1,9 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 #pragma once
 
+#include "pendingclusterstate.h"
 #include <vespa/storage/common/messagesender.h>
-#include <vespa/storage/distributor/pendingclusterstate.h>
+#include <vespa/vespalib/util/sync.h>
 
 namespace storage {
 
diff --git a/storage/src/vespa/storage/distributor/messagetracker.cpp b/storage/src/vespa/storage/distributor/messagetracker.cpp
index aeca85e821d..0b213a85dd3 100644
--- a/storage/src/vespa/storage/distributor/messagetracker.cpp
+++ b/storage/src/vespa/storage/distributor/messagetracker.cpp
@@ -33,7 +33,7 @@ MessageTracker::handleReply(api::BucketReply& reply)
 {
     std::map<uint64_t, uint16_t>::iterator found = _sentMessages.find(reply.getMsgId());
     if (found == _sentMessages.end()) {
-        LOG(warning, "Received reply %" PRIu64 " for callback which we have no recollection of", reply.getMsgId());
+        LOG(warning, "Received reply %zu for callback which we have no recollection of", reply.getMsgId());
         return (uint16_t)-1;
     } else {
         uint16_t node = found->second;
diff --git a/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp b/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp
index 5b907d21fcd..54ef503a772 100644
--- a/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp
+++ b/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp
@@ -5,6 +5,7 @@
 #include "putoperation.h"
 #include "updateoperation.h"
 #include <vespa/document/fieldvalue/document.h>
+#include <vespa/document/datatype/documenttype.h>
 #include <vespa/document/select/parser.h>
 #include <vespa/storageapi/message/persistence.h>
 #include <vespa/storageapi/message/batch.h>
diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/joinoperation.cpp b/storage/src/vespa/storage/distributor/operations/idealstate/joinoperation.cpp
index 29e65bfcc00..e2417ea1585 100644
--- a/storage/src/vespa/storage/distributor/operations/idealstate/joinoperation.cpp
+++ b/storage/src/vespa/storage/distributor/operations/idealstate/joinoperation.cpp
@@ -1,14 +1,9 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "joinoperation.h"
-#include <vespa/storage/distributor/idealstatemanager.h>
-#include <vespa/storage/distributor/pendingmessagetracker.h>
-#include <vespa/storageapi/messageapi/storagereply.h>
 #include <vespa/storageapi/message/bucketsplitting.h>
-#include <vespa/vdslib/state/clusterstate.h>
-
-#include <vespa/log/log.h>
 
+#include <vespa/log/bufferedlogger.h>
 LOG_SETUP(".distributor.operation.idealstate.join");
 
 using namespace storage::distributor;
diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.cpp b/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.cpp
index 0821408560b..80a79a8cde9 100644
--- a/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.cpp
+++ b/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.cpp
@@ -2,7 +2,7 @@
 #include "mergeoperation.h"
 #include <vespa/storage/distributor/idealstatemanager.h>
 
-#include <vespa/log/log.h>
+#include <vespa/log/bufferedlogger.h>
 LOG_SETUP(".distributor.operation.idealstate.merge");
 
 namespace storage::distributor {
diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/splitoperation.cpp b/storage/src/vespa/storage/distributor/operations/idealstate/splitoperation.cpp
index 7601d7faa6d..a6986c62715 100644
--- a/storage/src/vespa/storage/distributor/operations/idealstate/splitoperation.cpp
+++ b/storage/src/vespa/storage/distributor/operations/idealstate/splitoperation.cpp
@@ -5,7 +5,7 @@
 #include <vespa/storage/common/bucketoperationlogger.h>
 #include <vespa/storageapi/message/bucketsplitting.h>
 
-#include <vespa/log/log.h>
+#include <vespa/log/bufferedlogger.h>
 
 LOG_SETUP(".distributor.operation.idealstate.split");
 
diff --git a/storage/src/vespa/storage/distributor/operations/operation.cpp b/storage/src/vespa/storage/distributor/operations/operation.cpp
index 37e997d6080..8f6acc3258d 100644
--- a/storage/src/vespa/storage/distributor/operations/operation.cpp
+++ b/storage/src/vespa/storage/distributor/operations/operation.cpp
@@ -5,6 +5,7 @@
 #include <vespa/storageapi/messageapi/storagemessage.h>
 #include <vespa/storageapi/messageapi/storagecommand.h>
 #include <vespa/storageapi/messageapi/storagereply.h>
+#include <vespa/vespalib/util/stringfmt.h>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".distributor.callback");
diff --git a/storage/src/vespa/storage/distributor/operationtargetresolverimpl.cpp b/storage/src/vespa/storage/distributor/operationtargetresolverimpl.cpp
index fba6b131e09..6f331e23e2c 100644
--- a/storage/src/vespa/storage/distributor/operationtargetresolverimpl.cpp
+++ b/storage/src/vespa/storage/distributor/operationtargetresolverimpl.cpp
@@ -4,9 +4,9 @@
 #include <vespa/vespalib/util/exceptions.h>
 #include <vespa/vespalib/util/printable.hpp>
 #include <sstream>
+#include <cassert>
 
-namespace storage {
-namespace distributor {
+namespace storage::distributor {
 
 BucketInstance::BucketInstance(
         const document::BucketId& id, const api::BucketInfo& info,
@@ -197,5 +197,4 @@ OperationTargetResolverImpl::getAllInstances(OperationType type,
     return instances;
 }
 
-} // distributor
-} // storage
+}
diff --git a/storage/src/vespa/storage/distributor/pendingclusterstate.cpp b/storage/src/vespa/storage/distributor/pendingclusterstate.cpp
index 9497bf05f17..5a2f0d35e8c 100644
--- a/storage/src/vespa/storage/distributor/pendingclusterstate.cpp
+++ b/storage/src/vespa/storage/distributor/pendingclusterstate.cpp
@@ -4,7 +4,7 @@
 #include "bucketdbupdater.h"
 #include <vespa/storageframework/defaultimplementation/clock/realclock.h>
 #include <vespa/storage/common/bucketoperationlogger.h>
-#include <vespa/vespalib/util/xmlserializable.hpp>
+#include <vespa/vespalib/util/xmlstream.hpp>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".pendingclusterstate");
diff --git a/storage/src/vespa/storage/distributor/sentmessagemap.cpp b/storage/src/vespa/storage/distributor/sentmessagemap.cpp
index 7efbd3f55d2..02a6da98780 100644
--- a/storage/src/vespa/storage/distributor/sentmessagemap.cpp
+++ b/storage/src/vespa/storage/distributor/sentmessagemap.cpp
@@ -1,8 +1,8 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "sentmessagemap.h"
-
 #include <vespa/storage/distributor/operations/operation.h>
+#include <sstream>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".distributor.callback.map");
diff --git a/storage/src/vespa/storage/frameworkimpl/memory/memorystatusviewer.cpp b/storage/src/vespa/storage/frameworkimpl/memory/memorystatusviewer.cpp
index 883dc13cdb3..53967c1cae2 100644
--- a/storage/src/vespa/storage/frameworkimpl/memory/memorystatusviewer.cpp
+++ b/storage/src/vespa/storage/frameworkimpl/memory/memorystatusviewer.cpp
@@ -6,9 +6,9 @@
 #include <vespa/storage/storageutil/piechart.h>
 #include <vespa/metrics/metricmanager.h>
 #include <vespa/storageapi/messageapi/storagemessage.h>
+#include <sstream>
 
-#include <vespa/log/log.h>
-
+#include <vespa/log/bufferedlogger.h>
 LOG_SETUP(".memory.status.viewer");
 
 using storage::framework::defaultimplementation::MemoryState;
diff --git a/storage/src/vespa/storage/frameworkimpl/status/statuswebserver.cpp b/storage/src/vespa/storage/frameworkimpl/status/statuswebserver.cpp
index 00dce18e084..045a7514928 100644
--- a/storage/src/vespa/storage/frameworkimpl/status/statuswebserver.cpp
+++ b/storage/src/vespa/storage/frameworkimpl/status/statuswebserver.cpp
@@ -7,6 +7,7 @@
 #include <vespa/vespalib/util/host_name.h>
 #include <vespa/vespalib/util/exceptions.h>
 #include <vespa/vespalib/component/vtag.h>
+#include <sstream>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".status");
diff --git a/storage/src/vespa/storage/frameworkimpl/thread/deadlockdetector.cpp b/storage/src/vespa/storage/frameworkimpl/thread/deadlockdetector.cpp
index 2861ab904ff..3d9586ea398 100644
--- a/storage/src/vespa/storage/frameworkimpl/thread/deadlockdetector.cpp
+++ b/storage/src/vespa/storage/frameworkimpl/thread/deadlockdetector.cpp
@@ -5,7 +5,7 @@
 #include <vespa/storage/bucketmover/htmltable.h>
 #include <vespa/vespalib/stllike/asciistream.h>
 
-#include <vespa/log/log.h>
+#include <vespa/log/bufferedlogger.h>
 LOG_SETUP(".deadlock.detector");
 
 namespace storage {
diff --git a/storage/src/vespa/storage/persistence/bucketownershipnotifier.cpp b/storage/src/vespa/storage/persistence/bucketownershipnotifier.cpp
index b26c6ec893f..0a173b2945c 100644
--- a/storage/src/vespa/storage/persistence/bucketownershipnotifier.cpp
+++ b/storage/src/vespa/storage/persistence/bucketownershipnotifier.cpp
@@ -7,7 +7,7 @@
 #include <vespa/vdslib/distribution/distribution.h>
 #include <vespa/vespalib/util/backtrace.h>
 
-#include <vespa/log/log.h>
+#include <vespa/log/bufferedlogger.h>
 LOG_SETUP(".persistence.bucketownershipnotifier");
 
 namespace storage {
diff --git a/storage/src/vespa/storage/persistence/fieldvisitor.cpp b/storage/src/vespa/storage/persistence/fieldvisitor.cpp
index e67d15d4e91..6657a73543e 100644
--- a/storage/src/vespa/storage/persistence/fieldvisitor.cpp
+++ b/storage/src/vespa/storage/persistence/fieldvisitor.cpp
@@ -1,6 +1,8 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 // @author Vegard Sjonfjell
-#include <vespa/storage/persistence/fieldvisitor.h>
+
+#include "fieldvisitor.h"
+#include <vespa/document/select/valuenodes.h>
 
 namespace storage {
 
diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp
index a3c8fc3d671..128eff61436 100644
--- a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp
+++ b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp
@@ -17,7 +17,9 @@
 #include <vespa/storage/common/bucketoperationlogger.h>
 #include <vespa/storage/bucketdb/lockablemap.hpp>
 #include <vespa/vespalib/stllike/hash_map.hpp>
+#include <vespa/vespalib/util/stringfmt.h>
 
+#include <vespa/log/bufferedlogger.h>
 LOG_SETUP(".persistence.filestor.manager");
 
 using std::shared_ptr;
diff --git a/storage/src/vespa/storage/persistence/filestorage/modifiedbucketchecker.h b/storage/src/vespa/storage/persistence/filestorage/modifiedbucketchecker.h
index 5e45e846337..9938d6522e7 100644
--- a/storage/src/vespa/storage/persistence/filestorage/modifiedbucketchecker.h
+++ b/storage/src/vespa/storage/persistence/filestorage/modifiedbucketchecker.h
@@ -1,13 +1,13 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 #pragma once
 
-#include <memory>
 #include <vespa/storage/common/storagecomponent.h>
 #include <vespa/storage/common/servicelayercomponent.h>
 #include <vespa/storage/common/storagelink.h>
 #include <vespa/storage/config/config-stor-server.h>
 #include <vespa/storage/persistence/messages.h>
 #include <vespa/storage/persistence/types.h>
+#include <vespa/document/bucket/bucketidlist.h>
 #include <vespa/vespalib/util/sync.h>
 #include <vespa/config/config.h>
 
@@ -51,8 +51,7 @@ private:
     }
     bool requestModifiedBucketsFromProvider();
     void nextRecheckChunk(std::vector<RecheckBucketInfoCommand::SP>&);
-    void dispatchAllToPersistenceQueues(
-            const std::vector<RecheckBucketInfoCommand::SP>&);
+    void dispatchAllToPersistenceQueues(const std::vector<RecheckBucketInfoCommand::SP>&);
 
     spi::PersistenceProvider& _provider;
     ServiceLayerComponent::UP _component;
@@ -60,7 +59,7 @@ private:
     config::ConfigFetcher _configFetcher;
     vespalib::Monitor _monitor;
     vespalib::Lock _stateLock;
-    document::BucketId::List  _rechecksNotStarted;
+    document::bucket::BucketIdList  _rechecksNotStarted;
     size_t _pendingRequests;
     size_t _maxPendingChunkSize;
     bool _singleThreadMode; // For unit testing only
diff --git a/storage/src/vespa/storage/persistence/persistencethread.cpp b/storage/src/vespa/storage/persistence/persistencethread.cpp
index 10a5a7505dd..011c6ecc1b0 100644
--- a/storage/src/vespa/storage/persistence/persistencethread.cpp
+++ b/storage/src/vespa/storage/persistence/persistencethread.cpp
@@ -11,7 +11,8 @@
 #include <vespa/document/fieldset/fieldsetrepo.h>
 #include <vespa/vespalib/stllike/hash_map.hpp>
 #include <vespa/vespalib/util/exceptions.h>
-#include <vespa/log/log.h>
+
+#include <vespa/log/bufferedlogger.h>
 LOG_SETUP(".persistence.thread");
 
 namespace storage {
diff --git a/storage/src/vespa/storage/persistence/persistenceutil.cpp b/storage/src/vespa/storage/persistence/persistenceutil.cpp
index 2d8a8561ba6..202a7c5cac5 100644
--- a/storage/src/vespa/storage/persistence/persistenceutil.cpp
+++ b/storage/src/vespa/storage/persistence/persistenceutil.cpp
@@ -4,7 +4,7 @@
 #include <vespa/config/config.h>
 #include <vespa/config/helper/configgetter.hpp>
 
-#include <vespa/log/log.h>
+#include <vespa/log/bufferedlogger.h>
 LOG_SETUP(".persistence.util");
 
 namespace storage {
diff --git a/storage/src/vespa/storage/persistence/splitbitdetector.cpp b/storage/src/vespa/storage/persistence/splitbitdetector.cpp
index 62039f36133..cf3fde77b2b 100644
--- a/storage/src/vespa/storage/persistence/splitbitdetector.cpp
+++ b/storage/src/vespa/storage/persistence/splitbitdetector.cpp
@@ -1,11 +1,12 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
-#include <vespa/storage/persistence/splitbitdetector.h>
-#include <vespa/storage/persistence/bucketprocessor.h>
+#include "splitbitdetector.h"
+#include "bucketprocessor.h"
 #include <vespa/document/bucket/bucketidfactory.h>
 #include <vespa/document/base/documentid.h>
+#include <sstream>
 
-#include <vespa/log/log.h>
+#include <vespa/log/bufferedlogger.h>
 LOG_SETUP(".persistence.split.bitdetector");
 
 namespace storage {
diff --git a/storage/src/vespa/storage/storageserver/bouncer.cpp b/storage/src/vespa/storage/storageserver/bouncer.cpp
index 6ea898c344c..c285645309a 100644
--- a/storage/src/vespa/storage/storageserver/bouncer.cpp
+++ b/storage/src/vespa/storage/storageserver/bouncer.cpp
@@ -1,9 +1,9 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "bouncer.h"
-
 #include <vespa/storageapi/message/state.h>
 #include <vespa/storageapi/message/persistence.h>
+#include <sstream>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".bouncer");
diff --git a/storage/src/vespa/storage/storageserver/bucketintegritychecker.cpp b/storage/src/vespa/storage/storageserver/bucketintegritychecker.cpp
index 14b6adcb9a4..05b581d839a 100644
--- a/storage/src/vespa/storage/storageserver/bucketintegritychecker.cpp
+++ b/storage/src/vespa/storage/storageserver/bucketintegritychecker.cpp
@@ -7,10 +7,10 @@
 #include <vespa/storage/bucketdb/storbucketdb.h>
 #include <vespa/storageapi/message/state.h>
 #include <vespa/vdslib/distribution/distribution.h>
-#include <vespa/config/config.h>
 #include <vespa/storage/bucketdb/lockablemap.hpp>
 #include <vespa/vespalib/util/exceptions.h>
 
+#include <vespa/log/bufferedlogger.h>
 LOG_SETUP(".bucketintegritychecker");
 
 using std::shared_ptr;
diff --git a/storage/src/vespa/storage/storageserver/bucketintegritychecker.h b/storage/src/vespa/storage/storageserver/bucketintegritychecker.h
index 3add8c56e4b..f55cb18366c 100644
--- a/storage/src/vespa/storage/storageserver/bucketintegritychecker.h
+++ b/storage/src/vespa/storage/storageserver/bucketintegritychecker.h
@@ -16,6 +16,7 @@
 #include <vespa/storageapi/message/bucket.h>
 #include <vespa/storageframework/generic/status/htmlstatusreporter.h>
 #include <vespa/config/config.h>
+#include <list>
 
 namespace storage {
 
diff --git a/storage/src/vespa/storage/storageserver/changedbucketownershiphandler.cpp b/storage/src/vespa/storage/storageserver/changedbucketownershiphandler.cpp
index 27261424894..3390afcb9ec 100644
--- a/storage/src/vespa/storage/storageserver/changedbucketownershiphandler.cpp
+++ b/storage/src/vespa/storage/storageserver/changedbucketownershiphandler.cpp
@@ -7,7 +7,7 @@
 #include <vespa/storage/common/nodestateupdater.h>
 #include <vespa/vespalib/util/exceptions.h>
 
-#include <vespa/log/log.h>
+#include <vespa/log/bufferedlogger.h>
 LOG_SETUP(".bucketownershiphandler");
 
 namespace storage {
@@ -56,8 +56,7 @@ void
 ChangedBucketOwnershipHandler::reloadClusterState()
 {
     vespalib::LockGuard guard(_stateLock);
-    lib::ClusterState::CSP newState(_component.getStateUpdater()
-                                    .getSystemState());
+    lib::ClusterState::CSP newState(_component.getStateUpdater().getSystemState());
     setCurrentOwnershipWithStateNoLock(*newState);
 }
 
diff --git a/storage/src/vespa/storage/storageserver/communicationmanager.cpp b/storage/src/vespa/storage/storageserver/communicationmanager.cpp
index 1c93cefefbc..f32b1c242cf 100644
--- a/storage/src/vespa/storage/storageserver/communicationmanager.cpp
+++ b/storage/src/vespa/storage/storageserver/communicationmanager.cpp
@@ -10,8 +10,8 @@
 #include <vespa/storage/common/nodestateupdater.h>
 #include <vespa/vespalib/stllike/asciistream.h>
 #include <vespa/vespalib/stllike/hash_map.hpp>
-#include <vespa/log/log.h>
 
+#include <vespa/log/bufferedlogger.h>
 LOG_SETUP(".communication.manager");
 
 namespace storage {
diff --git a/storage/src/vespa/storage/storageserver/fnetlistener.cpp b/storage/src/vespa/storage/storageserver/fnetlistener.cpp
index 5bbeba4aa16..7daf2fb4777 100644
--- a/storage/src/vespa/storage/storageserver/fnetlistener.cpp
+++ b/storage/src/vespa/storage/storageserver/fnetlistener.cpp
@@ -5,6 +5,7 @@
 #include <vespa/storageapi/message/state.h>
 #include <vespa/vespalib/util/exceptions.h>
 #include <vespa/vespalib/util/host_name.h>
+#include <sstream>
 
 #include <vespa/log/log.h>
 
diff --git a/storage/src/vespa/storage/storageserver/mergethrottler.cpp b/storage/src/vespa/storage/storageserver/mergethrottler.cpp
index 577fedb58e1..ede7be3b9ad 100644
--- a/storage/src/vespa/storage/storageserver/mergethrottler.cpp
+++ b/storage/src/vespa/storage/storageserver/mergethrottler.cpp
@@ -2,9 +2,9 @@
 
 #include "mergethrottler.h"
 #include "storagemetricsset.h"
-#include <iostream>
 #include <sstream>
 #include <vespa/vespalib/stllike/asciistream.h>
+#include <vespa/vespalib/util/stringfmt.h>
 #include <vespa/storage/common/nodestateupdater.h>
 #include <vespa/storage/persistence/messages.h>
 #include <vespa/log/log.h>
diff --git a/storage/src/vespa/storage/storageserver/messagesink.cpp b/storage/src/vespa/storage/storageserver/messagesink.cpp
index 0df24ef3536..4960cd2085e 100644
--- a/storage/src/vespa/storage/storageserver/messagesink.cpp
+++ b/storage/src/vespa/storage/storageserver/messagesink.cpp
@@ -2,6 +2,7 @@
 
 #include "messagesink.h"
 #include <vespa/storageapi/message/persistence.h>
+#include <ostream>
 
 using std::shared_ptr;
 
diff --git a/storage/src/vespa/storage/storageserver/opslogger.cpp b/storage/src/vespa/storage/storageserver/opslogger.cpp
index 2a4465c62bc..f770fc0af51 100644
--- a/storage/src/vespa/storage/storageserver/opslogger.cpp
+++ b/storage/src/vespa/storage/storageserver/opslogger.cpp
@@ -2,6 +2,7 @@
 
 #include "opslogger.h"
 #include <vespa/storageapi/message/persistence.h>
+#include <sstream>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".operationslogger");
diff --git a/storage/src/vespa/storage/tools/.gitignore b/storage/src/vespa/storage/tools/.gitignore
index 6f27b7c441d..ba1c4352573 100644
--- a/storage/src/vespa/storage/tools/.gitignore
+++ b/storage/src/vespa/storage/tools/.gitignore
@@ -16,7 +16,7 @@ populatenode
 slotfilefeeder
 statfs
 stoccart
-storage-cmd
+vespa-storage-cmd
 throttlingsim
 vdsclient
 vdsdisktool
diff --git a/storage/src/vespa/storage/tools/CMakeLists.txt b/storage/src/vespa/storage/tools/CMakeLists.txt
index 3f5febcd28a..f2bbb266ef9 100644
--- a/storage/src/vespa/storage/tools/CMakeLists.txt
+++ b/storage/src/vespa/storage/tools/CMakeLists.txt
@@ -2,7 +2,6 @@
 vespa_add_executable(storage_getidealstate_app
     SOURCES
     getidealstate.cpp
-    INSTALL bin
     DEPENDS
     storage
     AFTER
@@ -11,7 +10,6 @@ vespa_add_executable(storage_getidealstate_app
 vespa_add_executable(storage_generatedistributionbits_app
     SOURCES
     generatedistributionbits.cpp
-    INSTALL bin
     DEPENDS
     storage
     AFTER
@@ -20,7 +18,6 @@ vespa_add_executable(storage_generatedistributionbits_app
 vespa_add_executable(storage_analyzedistribution_app
     SOURCES
     analyzedistribution.cpp
-    INSTALL bin
     DEPENDS
     storage
     AFTER
@@ -29,8 +26,7 @@ vespa_add_executable(storage_analyzedistribution_app
 vespa_add_executable(storage_storage-cmd_app
     SOURCES
     storage-cmd.cpp
-    OUTPUT_NAME storage-cmd
-    INSTALL bin
+    OUTPUT_NAME vespa-storage-cmd
     DEPENDS
     AFTER
     storage_storageconfig
@@ -38,7 +34,6 @@ vespa_add_executable(storage_storage-cmd_app
 vespa_add_executable(storage_throttlingsim_app
     SOURCES
     throttlingsim.cpp
-    INSTALL bin
     DEPENDS
     AFTER
     storage_storageconfig
@@ -46,7 +41,6 @@ vespa_add_executable(storage_throttlingsim_app
 vespa_add_executable(storage_statfs_app
     SOURCES
     statfs.cpp
-    INSTALL bin
     DEPENDS
     AFTER
     storage_storageconfig
diff --git a/storage/src/vespa/storage/tools/generatedistributionbits.cpp b/storage/src/vespa/storage/tools/generatedistributionbits.cpp
index 53c7c7cf1a1..3f5b3f4ae3d 100644
--- a/storage/src/vespa/storage/tools/generatedistributionbits.cpp
+++ b/storage/src/vespa/storage/tools/generatedistributionbits.cpp
@@ -1,9 +1,11 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
 #include <vespa/document/bucket/bucketidfactory.h>
 #include <vespa/vespalib/util/programoptions.h>
 #include <vespa/vdslib/distribution/distribution.h>
 #include <vespa/vdslib/state/clusterstate.h>
 #include <vespa/storage/bucketdb/judyarray.h>
+#include <vespa/vespalib/util/stringfmt.h>
 #include <iomanip>
 #include <iostream>
 #include <vespa/config-stor-distribution.h>
diff --git a/storage/src/vespa/storage/tools/storage-cmd.cpp b/storage/src/vespa/storage/tools/storage-cmd.cpp
index e73376fcbe5..b9ead2be834 100644
--- a/storage/src/vespa/storage/tools/storage-cmd.cpp
+++ b/storage/src/vespa/storage/tools/storage-cmd.cpp
@@ -4,7 +4,7 @@
 #include <vespa/fastos/app.h>
 
 #include <vespa/log/log.h>
-LOG_SETUP("storage-cmd");
+LOG_SETUP("vespa-storage-cmd");
 
 class RPCClient : public FastOS_Application
 {
@@ -46,7 +46,7 @@ private:
 public:
     int Main() override {
         if (_argc < 3) {
-            fprintf(stderr, "usage: storage-cmd <connectspec> <method> [args]\n");
+            fprintf(stderr, "usage: vespa-storage-cmd <connectspec> <method> [args]\n");
             fprintf(stderr, "Calls RPC method on a storage/distributor process\n");
             fprintf(stderr, "Call frt.rpc.getMethodList to get available RPC methods\n");
             fprintf(stderr, "    each arg must be on the form <type>:<value>\n");
diff --git a/storage/src/vespa/storage/visiting/countvisitor.cpp b/storage/src/vespa/storage/visiting/countvisitor.cpp
index 282e2f32873..ca6678c1b3d 100644
--- a/storage/src/vespa/storage/visiting/countvisitor.cpp
+++ b/storage/src/vespa/storage/visiting/countvisitor.cpp
@@ -3,6 +3,7 @@
 #include "countvisitor.h"
 #include <vespa/document/fieldvalue/document.h>
 #include <vespa/documentapi/messagebus/messages/visitor.h>
+#include <vespa/vespalib/util/stringfmt.h>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".visitor.instance.countvisitor");
diff --git a/storage/src/vespa/storage/visiting/visitor.cpp b/storage/src/vespa/storage/visiting/visitor.cpp
index 704b523a3b6..aff3cb0725a 100644
--- a/storage/src/vespa/storage/visiting/visitor.cpp
+++ b/storage/src/vespa/storage/visiting/visitor.cpp
@@ -10,7 +10,10 @@
 #include <vespa/document/select/node.h>
 #include <vespa/vespalib/stllike/hash_map.hpp>
 #include <vespa/vespalib/stllike/asciistream.h>
+#include <vespa/vespalib/util/stringfmt.h>
 #include <unordered_map>
+#include <sstream>
+
 #include <vespa/log/log.h>
 LOG_SETUP(".visitor.instance");
 
diff --git a/storage/src/vespa/storage/visiting/visitormanager.cpp b/storage/src/vespa/storage/visiting/visitormanager.cpp
index 7b8ce974ca9..5e1f8b4df79 100644
--- a/storage/src/vespa/storage/visiting/visitormanager.cpp
+++ b/storage/src/vespa/storage/visiting/visitormanager.cpp
@@ -8,10 +8,10 @@
 #include "testvisitor.h"
 #include "recoveryvisitor.h"
 #include <vespa/storageframework/generic/memory/memorymanagerinterface.h>
-#include <vespa/storageapi/message/visitor.h>
 #include <vespa/storage/common/statusmessages.h>
-#include <vespa/storage/storageserver/storagemetricsset.h>
 #include <vespa/documentapi/loadtypes/loadtypeset.h>
+#include <vespa/vespalib/util/stringfmt.h>
+#include <sstream>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".visitor.manager");
diff --git a/storage/src/vespa/storage/visiting/visitorthread.cpp b/storage/src/vespa/storage/visiting/visitorthread.cpp
index e57bcebf372..aeea3c7ba8e 100644
--- a/storage/src/vespa/storage/visiting/visitorthread.cpp
+++ b/storage/src/vespa/storage/visiting/visitorthread.cpp
@@ -14,10 +14,11 @@
 #include <vespa/vespalib/stllike/asciistream.h>
 #include <vespa/vespalib/util/exceptions.h>
 #include <vespa/document/base/exceptions.h>
-#include <locale>
 #include <vespa/vespalib/stllike/hash_map.hpp>
-#include <vespa/log/log.h>
+#include <locale>
+#include <sstream>
 
+#include <vespa/log/log.h>
 LOG_SETUP(".visitor.thread");
 
 using storage::api::ReturnCode;
diff --git a/storageapi/src/vespa/storageapi/buckets/bucketinfo.cpp b/storageapi/src/vespa/storageapi/buckets/bucketinfo.cpp
index 07ba2773e91..f9924c953c2 100644
--- a/storageapi/src/vespa/storageapi/buckets/bucketinfo.cpp
+++ b/storageapi/src/vespa/storageapi/buckets/bucketinfo.cpp
@@ -1,9 +1,9 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 #include "bucketinfo.h"
 #include <vespa/vespalib/stllike/asciistream.h>
+#include <vespa/vespalib/util/xmlstream.h>
 
-namespace storage {
-namespace api {
+namespace storage::api {
 
 BucketInfo::BucketInfo()
     : _lastModified(0),
@@ -124,5 +124,4 @@ BucketInfo::printXml(vespalib::XmlOutputStream& xos) const
         << XmlAttribute("lastmodified", _lastModified);
 }
 
-} // api
-} // storage
+}
diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.cpp b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.cpp
index 1ad07387ba4..9d4d25d9d8e 100644
--- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.cpp
+++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.cpp
@@ -13,14 +13,13 @@
 #include <vespa/storageapi/message/batch.h>
 #include <vespa/storageapi/message/removelocation.h>
 #include <vespa/vespalib/util/exceptions.h>
-
 #include <vespa/vespalib/util/growablebytebuffer.h>
+#include <sstream>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".storage.api.mbusprot.serialization.base");
 
-namespace storage {
-namespace mbusprot {
+namespace storage::mbusprot {
 
 ProtocolSerialization::ProtocolSerialization(
         const document::DocumentTypeRepo::SP& repo)
@@ -299,5 +298,4 @@ ProtocolSerialization::decodeReply(mbus::BlobRef data,
     return StorageReply::UP(new StorageReply(SRep::SP(reply.release())));
 }
 
-} // mbusprot
-} // storage
+}
diff --git a/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.cpp b/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.cpp
index 62a85e0f45e..87dcacb0fb9 100644
--- a/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.cpp
+++ b/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.cpp
@@ -5,12 +5,12 @@
 #include "storagereply.h"
 #include <vespa/vespalib/util/exceptions.h>
 #include <vespa/document/util/stringutil.h>
+#include <sstream>
 
-#include <vespa/log/log.h>
+#include <vespa/log/bufferedlogger.h>
 LOG_SETUP(".storage.api.mbusprot.protocol");
 
-namespace storage {
-namespace mbusprot {
+namespace storage::mbusprot {
 
 mbus::string StorageProtocol::NAME = "StorageProtocol";
 
@@ -169,5 +169,4 @@ StorageProtocol::decode(const vespalib::Version & version,
     return mbus::Routable::UP();
 }
 
-} // mbusprot
-} // storage
+}
diff --git a/storageapi/src/vespa/storageapi/message/batch.cpp b/storageapi/src/vespa/storageapi/message/batch.cpp
index 405d5b4c9d2..ee7bbbcbdf6 100644
--- a/storageapi/src/vespa/storageapi/message/batch.cpp
+++ b/storageapi/src/vespa/storageapi/message/batch.cpp
@@ -2,7 +2,7 @@
 //
 #include "batch.h"
 #include <vespa/document/bucket/bucketidfactory.h>
-#include <algorithm>
+#include <ostream>
 
 using namespace storage::api;
 
diff --git a/storageapi/src/vespa/storageapi/message/bucket.cpp b/storageapi/src/vespa/storageapi/message/bucket.cpp
index 3400e0d6399..e4e6f796ac2 100644
--- a/storageapi/src/vespa/storageapi/message/bucket.cpp
+++ b/storageapi/src/vespa/storageapi/message/bucket.cpp
@@ -4,9 +4,10 @@
 #include <vespa/document/fieldvalue/document.h>
 #include <vespa/vespalib/stllike/asciistream.h>
 #include <vespa/vespalib/util/array.hpp>
+#include <ostream>
+#include <iterator>
 
-namespace storage {
-namespace api {
+namespace storage::api {
 
 IMPLEMENT_COMMAND(CreateBucketCommand, CreateBucketReply)
 IMPLEMENT_REPLY(CreateBucketReply)
@@ -625,7 +626,6 @@ SetBucketStateReply::print(std::ostream& out,
     }
 }
 
-} // api
-} // storage
+}
 
 template class vespalib::Array<storage::api::RequestBucketInfoReply::Entry>;
diff --git a/storageapi/src/vespa/storageapi/message/persistence.cpp b/storageapi/src/vespa/storageapi/message/persistence.cpp
index 6de53d9db82..ce1adcfa2ca 100644
--- a/storageapi/src/vespa/storageapi/message/persistence.cpp
+++ b/storageapi/src/vespa/storageapi/message/persistence.cpp
@@ -3,6 +3,7 @@
 #include "persistence.h"
 #include <vespa/vespalib/util/exceptions.h>
 #include <vespa/vespalib/stllike/asciistream.h>
+#include <ostream>
 
 namespace storage {
 namespace api {
diff --git a/storageapi/src/vespa/storageapi/messageapi/storagemessage.cpp b/storageapi/src/vespa/storageapi/messageapi/storagemessage.cpp
index 536135c48e6..399a30543da 100644
--- a/storageapi/src/vespa/storageapi/messageapi/storagemessage.cpp
+++ b/storageapi/src/vespa/storageapi/messageapi/storagemessage.cpp
@@ -2,13 +2,29 @@
 
 #include "storagemessage.h"
 
-#include <vespa/vespalib/util/exceptions.h>
 #include <vespa/messagebus/routing/verbatimdirective.h>
+#include <vespa/vespalib/util/exceptions.h>
+#include <vespa/vespalib/util/sync.h>
 #include <vespa/vespalib/stllike/asciistream.h>
 #include <sstream>
+#include <cassert>
+
+namespace storage::api {
+
+namespace {
 
-namespace storage {
-namespace api {
+/**
+ * TODO
+ * From @vekterli
+ * I have no idea why the _lastMsgId update code masks away the 8 MSB, but if we assume it's probably for no
+ * overwhelmingly good reason we could replace this mutex with just a std::atomic<uint64_t> and do a relaxed
+ * fetch_add (shouldn't be any need for any barriers; ID increments have no other memory dependencies). U64 overflows
+ * here come under the category "never gonna happen in the real world".
+ * @balder agree - @vekterli fix in separate pull request :)
+ */
+vespalib::Lock _G_msgIdLock;
+
+}
 
 static const vespalib::string STORAGEADDRESS_PREFIX = "storage/cluster.";
 
@@ -177,6 +193,20 @@ MessageType::MessageType::get(Id id)
     }
     return *it->second;
 }
+MessageType::MessageType(const vespalib::stringref & name, Id id,
+            const MessageType* replyOf)
+        : _name(name), _id(id), _reply(NULL), _replyOf(replyOf)
+{
+    _codes[id] = this;
+    if (_replyOf != 0) {
+        assert(_replyOf->_reply == 0);
+        // Ugly cast to let initialization work
+        MessageType& type = const_cast<MessageType&>(*_replyOf);
+        type._reply = this;
+    }
+}
+
+MessageType::~MessageType() {}
 
 void
 MessageType::print(std::ostream& out, bool verbose, const std::string& indent) const
@@ -305,17 +335,14 @@ StorageMessageAddress::print(vespalib::asciistream & out) const
     }
 }
 
-TransportContext::~TransportContext()
-{
-}
+TransportContext::~TransportContext() { }
 
-vespalib::Lock StorageMessage::_msgIdLock;
 StorageMessage::Id StorageMessage::_lastMsgId = 1000;
 
 StorageMessage::Id
 StorageMessage::generateMsgId()
 {
-    vespalib::LockGuard sync(_msgIdLock);
+    vespalib::LockGuard sync(_G_msgIdLock);
     Id msgId = _lastMsgId++;
     _lastMsgId &= ((Id(-1) << 8) >> 8);
     return msgId;
@@ -343,7 +370,7 @@ StorageMessage::~StorageMessage() { }
 
 void StorageMessage::setNewMsgId()
 {
-    vespalib::LockGuard sync(_msgIdLock);
+    vespalib::LockGuard sync(_G_msgIdLock);
     _msgId = _lastMsgId++;
     _lastMsgId &= ((Id(-1) << 8) >> 8);
 }
@@ -353,5 +380,5 @@ StorageMessage::getSummary() const {
     return toString();
 }
 
-} // api
-} // storage
+}
+
diff --git a/storageapi/src/vespa/storageapi/messageapi/storagemessage.h b/storageapi/src/vespa/storageapi/messageapi/storagemessage.h
index 59ae462432b..442ee0c9899 100644
--- a/storageapi/src/vespa/storageapi/messageapi/storagemessage.h
+++ b/storageapi/src/vespa/storageapi/messageapi/storagemessage.h
@@ -19,7 +19,6 @@
 #include <vespa/vdslib/state/nodetype.h>
 #include <vespa/document/bucket/bucketid.h>
 #include <vespa/vespalib/util/printable.h>
-#include <vespa/vespalib/util/sync.h>
 #include <map>
 
 namespace vespalib {
@@ -82,10 +81,6 @@ namespace api {
  * This is used to be able to deserialize messages of various classes.
  */
 class MessageType : public vespalib::Printable {
-private:
-    MessageType(const MessageType &);
-    MessageType& operator=(const MessageType &);
-
 public:
     enum Id {
         GET_ID = 4,
@@ -172,18 +167,7 @@ private:
     MessageType *_reply;
     const MessageType *_replyOf;
 
-    MessageType(const vespalib::stringref & name, Id id,
-                const MessageType* replyOf = 0)
-        : _name(name), _id(id), _reply(NULL), _replyOf(replyOf)
-    {
-        _codes[id] = this;
-        if (_replyOf != 0) {
-            assert(_replyOf->_reply == 0);
-                // Ugly cast to let initialization work
-            MessageType& type = const_cast<MessageType&>(*_replyOf);
-            type._reply = this;
-        }
-    }
+    MessageType(const vespalib::stringref & name, Id id, const MessageType* replyOf = 0);
 public:
     static const MessageType DOCBLOCK;
     static const MessageType DOCBLOCK_REPLY;
@@ -264,16 +248,17 @@ public:
 
     static const MessageType& get(Id id);
 
+    MessageType(const MessageType &) = delete;
+    MessageType& operator=(const MessageType &) = delete;
+    ~MessageType();
     Id getId() const { return _id; }
     static Id getMaxId() { return MESSAGETYPE_MAX_ID; }
     const vespalib::string& getName() const { return _name; }
     bool isReply() const { return (_replyOf != 0); }
-        /** Only valid to call on replies. */
-    const MessageType& getCommandType() const
-        { assert(_replyOf); return *_replyOf; }
-        /** Only valid to call on commands. */
-    const MessageType& getReplyType() const
-        { assert(_reply); return *_reply; }
+    /** Only valid to call on replies. */
+    const MessageType& getCommandType() const { return *_replyOf; }
+    /** Only valid to call on commands. */
+    const MessageType& getReplyType() const { return *_reply; }
     bool operator==(const MessageType& type) const { return (_id == type._id); }
     bool operator!=(const MessageType& type) const { return (_id != type._id); }
 
@@ -294,7 +279,7 @@ private:
     mbus::Route _route;
     bool        _retryEnabled;
     Protocol    _protocol;
-        // Used for internal VDS addresses only
+    // Used for internal VDS addresses only
     vespalib::string     _cluster;
     const lib::NodeType* _type;
     uint16_t             _index;
@@ -346,7 +331,6 @@ public:
     static const char* getPriorityString(Priority);
 
 private:
-    static vespalib::Lock _msgIdLock;
     static Id _lastMsgId;
 
     StorageMessage& operator=(const StorageMessage&);
diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/memory/memorystate.cpp b/storageframework/src/vespa/storageframework/defaultimplementation/memory/memorystate.cpp
index 125e01db179..a19851ae01f 100644
--- a/storageframework/src/vespa/storageframework/defaultimplementation/memory/memorystate.cpp
+++ b/storageframework/src/vespa/storageframework/defaultimplementation/memory/memorystate.cpp
@@ -1,13 +1,12 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "memorystate.h"
+#include <sstream>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".memory.state");
 
-namespace storage {
-namespace framework {
-namespace defaultimplementation {
+namespace storage::framework::defaultimplementation {
 
 MemoryState::Entry::Entry()
     : _currentUsedSize(0),
@@ -223,6 +222,4 @@ MemoryState::print(std::ostream& out, bool verbose,
     out << "\n" << indent << "}";
 }
 
-} // defaultimplementation
-} // framework
-} // storage
+}
diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/memory/simplememorylogic.cpp b/storageframework/src/vespa/storageframework/defaultimplementation/memory/simplememorylogic.cpp
index 55723eec558..5d5d0b97541 100644
--- a/storageframework/src/vespa/storageframework/defaultimplementation/memory/simplememorylogic.cpp
+++ b/storageframework/src/vespa/storageframework/defaultimplementation/memory/simplememorylogic.cpp
@@ -1,15 +1,12 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "simplememorylogic.h"
-
 #include <vespa/vespalib/util/exceptions.h>
 
-#include <vespa/log/log.h>
+#include <vespa/log/bufferedlogger.h>
 LOG_SETUP(".memory.logic.simple");
 
-namespace storage {
-namespace framework {
-namespace defaultimplementation {
+namespace storage::framework::defaultimplementation {
 
 SimpleMemoryLogic::SimpleMemoryLogic(Clock& c, uint64_t maxMemory)
     : _cacheThreshold(0.98),
@@ -232,6 +229,4 @@ SimpleMemoryLogic::print(std::ostream& out, bool verbose,
     _state.print(out, verbose, indent + "  ");
 }
 
-} // defaultimplementation
-} // framework
-} // storage
+}
diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/thread/threadimpl.cpp b/storageframework/src/vespa/storageframework/defaultimplementation/thread/threadimpl.cpp
index 3e3785f67b1..eaa66fedc88 100644
--- a/storageframework/src/vespa/storageframework/defaultimplementation/thread/threadimpl.cpp
+++ b/storageframework/src/vespa/storageframework/defaultimplementation/thread/threadimpl.cpp
@@ -4,7 +4,7 @@
 #include "threadpoolimpl.h"
 #include <vespa/storageframework/generic/clock/clock.h>
 
-#include <vespa/log/log.h>
+#include <vespa/log/bufferedlogger.h>
 LOG_SETUP(".framework.thread.impl");
 
 namespace storage::framework::defaultimplementation {
diff --git a/storageframework/src/vespa/storageframework/generic/status/httpurlpath.h b/storageframework/src/vespa/storageframework/generic/status/httpurlpath.h
index f7229a1d297..2704f90d5f1 100644
--- a/storageframework/src/vespa/storageframework/generic/status/httpurlpath.h
+++ b/storageframework/src/vespa/storageframework/generic/status/httpurlpath.h
@@ -9,6 +9,7 @@
 #include <vespa/vespalib/util/printable.h>
 #include <vespa/vespalib/stllike/string.h>
 #include <map>
+#include <sstream>
 
 namespace storage::framework {
 
diff --git a/storageframework/src/vespa/storageframework/generic/status/xmlstatusreporter.h b/storageframework/src/vespa/storageframework/generic/status/xmlstatusreporter.h
index 522a6480a75..2f38b89fa15 100644
--- a/storageframework/src/vespa/storageframework/generic/status/xmlstatusreporter.h
+++ b/storageframework/src/vespa/storageframework/generic/status/xmlstatusreporter.h
@@ -18,7 +18,7 @@
 #pragma once
 
 #include "statusreporter.h"
-#include <vespa/vespalib/util/xmlserializable.h>
+#include <vespa/vespalib/util/xmlstream.h>
 
 namespace storage::framework {
 
diff --git a/storageserver/src/apps/storaged/.gitignore b/storageserver/src/apps/storaged/.gitignore
index 67fbcdcf4db..282d97fe962 100644
--- a/storageserver/src/apps/storaged/.gitignore
+++ b/storageserver/src/apps/storaged/.gitignore
@@ -1,3 +1,3 @@
 /Makefile
 /storaged
-storaged-bin
+vespa-storaged-bin
diff --git a/storageserver/src/apps/storaged/CMakeLists.txt b/storageserver/src/apps/storaged/CMakeLists.txt
index 8a488b69d77..039058caaff 100644
--- a/storageserver/src/apps/storaged/CMakeLists.txt
+++ b/storageserver/src/apps/storaged/CMakeLists.txt
@@ -3,7 +3,7 @@ vespa_add_executable(storageserver_storaged_app
     SOURCES
     storage.cpp
     forcelink.cpp
-    OUTPUT_NAME storaged-bin
+    OUTPUT_NAME vespa-storaged-bin
     INSTALL sbin
     DEPENDS
     storageserver_storageapp
diff --git a/storageserver/src/vespa/storageserver/app/distributorprocess.h b/storageserver/src/vespa/storageserver/app/distributorprocess.h
index 8911372b05e..476dd054cd0 100644
--- a/storageserver/src/vespa/storageserver/app/distributorprocess.h
+++ b/storageserver/src/vespa/storageserver/app/distributorprocess.h
@@ -8,6 +8,7 @@
 #pragma once
 
 #include "process.h"
+#include <vespa/storage/storageserver/distributornode.h>
 
 namespace storage {
 
diff --git a/storageserver/src/vespa/storageserver/app/process.cpp b/storageserver/src/vespa/storageserver/app/process.cpp
index 352254fba14..c873cb15bb1 100644
--- a/storageserver/src/vespa/storageserver/app/process.cpp
+++ b/storageserver/src/vespa/storageserver/app/process.cpp
@@ -1,6 +1,7 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "process.h"
+#include <vespa/storage/storageserver/storagenode.h>
 #include <vespa/vespalib/util/exceptions.h>
 
 #include <vespa/log/log.h>
diff --git a/storageserver/src/vespa/storageserver/app/process.h b/storageserver/src/vespa/storageserver/app/process.h
index 06e76425fdc..50ceeb43c14 100644
--- a/storageserver/src/vespa/storageserver/app/process.h
+++ b/storageserver/src/vespa/storageserver/app/process.h
@@ -14,23 +14,29 @@
 
 #pragma once
 
-#include <vespa/document/datatype/documenttype.h>
+#include <vespa/document/config/config-documenttypes.h>
 #include <vespa/storage/storageserver/applicationgenerationfetcher.h>
-#include <vespa/storage/storageserver/distributornode.h>
-#include <vespa/storage/storageserver/servicelayernode.h>
-#include <vespa/config/config.h>
+#include <vespa/config/subscription/configuri.h>
+#include <vespa/config/subscription/configsubscriber.h>
+
+
+namespace document { class DocumentTypeRepo; }
 
 namespace storage {
 
+class StorageNode;
+class StorageNodeContext;
+
 class Process : public ApplicationGenerationFetcher {
 protected:
+    using DocumentTypeRepoSP = std::shared_ptr<document::DocumentTypeRepo>;
     config::ConfigUri _configUri;
-    document::DocumentTypeRepo::SP getTypeRepo() { return _repos.back(); }
+    DocumentTypeRepoSP getTypeRepo() { return _repos.back(); }
     config::ConfigSubscriber _configSubscriber;
 
 private:
     config::ConfigHandle<document::DocumenttypesConfig>::UP _documentHandler;
-    std::vector<document::DocumentTypeRepo::SP> _repos;
+    std::vector<DocumentTypeRepoSP> _repos;
 
 public:
     typedef std::unique_ptr<Process> UP;
diff --git a/storageserver/src/vespa/storageserver/app/rpcservicelayerprocess.cpp b/storageserver/src/vespa/storageserver/app/rpcservicelayerprocess.cpp
index c8b40be2404..fa37ed10190 100644
--- a/storageserver/src/vespa/storageserver/app/rpcservicelayerprocess.cpp
+++ b/storageserver/src/vespa/storageserver/app/rpcservicelayerprocess.cpp
@@ -1,6 +1,7 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "rpcservicelayerprocess.h"
+#include <vespa/storage/config/config-stor-server.h>
 #include <vespa/config/helper/configgetter.hpp>
 
 #include <vespa/log/log.h>
diff --git a/storageserver/src/vespa/storageserver/app/servicelayerprocess.cpp b/storageserver/src/vespa/storageserver/app/servicelayerprocess.cpp
index 696e79503d7..30b71560287 100644
--- a/storageserver/src/vespa/storageserver/app/servicelayerprocess.cpp
+++ b/storageserver/src/vespa/storageserver/app/servicelayerprocess.cpp
@@ -1,6 +1,7 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "servicelayerprocess.h"
+#include <vespa/storage/storageserver/servicelayernode.h>
 #include <vespa/searchvisitor/searchvisitor.h>
 
 namespace storage {
@@ -28,4 +29,19 @@ ServiceLayerProcess::createNode()
     _node->init();
 }
 
+StorageNode&
+ServiceLayerProcess::getNode() {
+    return *_node;
+}
+
+StorageNodeContext&
+ServiceLayerProcess::getContext() {
+    return _context;
+}
+
+std::string
+ServiceLayerProcess::getComponentName() const {
+    return "servicelayer";
+}
+
 } // storage
diff --git a/storageserver/src/vespa/storageserver/app/servicelayerprocess.h b/storageserver/src/vespa/storageserver/app/servicelayerprocess.h
index 4140e072a50..dbef3930c20 100644
--- a/storageserver/src/vespa/storageserver/app/servicelayerprocess.h
+++ b/storageserver/src/vespa/storageserver/app/servicelayerprocess.h
@@ -17,15 +17,21 @@
 #pragma once
 
 #include "process.h"
+#include <vespa/storage/storageserver/servicelayernodecontext.h>
+#include <vespa/storage/common/visitorfactory.h>
 #include <vespa/config/config.h>
 #include <vespa/config/helper/configfetcher.h>
 #include <vespa/config-persistence.h>
 
 namespace storage {
 
+namespace spi { class PersistenceProvider; }
+
+class ServiceLayerNode;
+
 class ServiceLayerProcess : public Process {
     VisitorFactory::Map _externalVisitors;
-    ServiceLayerNode::UP _node;
+    std::unique_ptr<ServiceLayerNode> _node;
 
 protected:
     ServiceLayerNodeContext _context;
@@ -40,9 +46,9 @@ public:
     virtual spi::PersistenceProvider& getProvider() = 0;
 
     void createNode() override;
-    StorageNode& getNode() override { return *_node; }
-    StorageNodeContext& getContext() override { return _context; }
-    std::string getComponentName() const override { return "servicelayer"; }
+    StorageNode& getNode() override;
+    StorageNodeContext& getContext() override;
+    std::string getComponentName() const override;
 };
 
 } // storage
diff --git a/streamingvisitors/src/tests/hitcollector/hitcollector.cpp b/streamingvisitors/src/tests/hitcollector/hitcollector.cpp
index dc10ea168c0..4e1ac516a7b 100644
--- a/streamingvisitors/src/tests/hitcollector/hitcollector.cpp
+++ b/streamingvisitors/src/tests/hitcollector/hitcollector.cpp
@@ -2,6 +2,7 @@
 #include <vespa/vespalib/testkit/testapp.h>
 
 #include <vespa/document/fieldvalue/fieldvalues.h>
+#include <vespa/document/datatype/documenttype.h>
 #include <vespa/searchlib/fef/matchdata.h>
 #include <vespa/searchlib/fef/feature_resolver.h>
 #include <vespa/searchvisitor/hitcollector.h>
diff --git a/streamingvisitors/src/vespa/searchvisitor/hitcollector.cpp b/streamingvisitors/src/vespa/searchvisitor/hitcollector.cpp
index 5ca14fd617d..074070d2ccd 100644
--- a/streamingvisitors/src/vespa/searchvisitor/hitcollector.cpp
+++ b/streamingvisitors/src/vespa/searchvisitor/hitcollector.cpp
@@ -2,6 +2,7 @@
 
 #include "hitcollector.h"
 #include <vespa/searchlib/fef/feature_resolver.h>
+#include <vespa/vespalib/util/stringfmt.h>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".searchvisitor.hitcollector");
diff --git a/streamingvisitors/src/vespa/searchvisitor/rankmanager.cpp b/streamingvisitors/src/vespa/searchvisitor/rankmanager.cpp
index d25154aba7f..4befe163377 100644
--- a/streamingvisitors/src/vespa/searchvisitor/rankmanager.cpp
+++ b/streamingvisitors/src/vespa/searchvisitor/rankmanager.cpp
@@ -3,7 +3,7 @@
 #include "rankmanager.h"
 #include <vespa/searchlib/features/setup.h>
 #include <vespa/searchlib/fef/functiontablefactory.h>
-#include <vespa/vespalib/util/vstringfmt.h>
+#include <vespa/vespalib/util/stringfmt.h>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".searchvisitor.rankmanager");
@@ -106,7 +106,7 @@ RankManager::Snapshot::initRankSetup(const BlueprintFactory & factory)
     LOG(debug, "Number of index environments and rank setups: %u", (uint32_t)_indexEnv.size());
     LOG_ASSERT(_properties.size() == _rankSetup.size());
     for (uint32_t i = 0; i < _properties.size(); ++i) {
-        vespalib::string number = vespalib::make_vespa_string("%u", i);
+        vespalib::string number = vespalib::make_string("%u", i);
         _rpmap[number] = i;
     }
     for (uint32_t i = 0; i < _properties.size(); ++i) {
diff --git a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp
index 616da034c54..7b5e5f17671 100644
--- a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp
+++ b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp
@@ -4,6 +4,7 @@
 #include "searchenvironment.h"
 #include "searchvisitor.h"
 #include <vespa/document/datatype/positiondatatype.h>
+#include <vespa/document/datatype/documenttype.h>
 #include <vespa/searchlib/aggregation/modifiers.h>
 #include <vespa/searchlib/common/packets.h>
 #include <vespa/searchlib/uca/ucaconverter.h>
@@ -367,7 +368,7 @@ SearchVisitorFactory::makeVisitor(StorageComponent& component,
 }
 
 void
-SearchVisitor::AttributeInserter::onPrimitive(uint32_t, const IteratorContent & c)
+SearchVisitor::AttributeInserter::onPrimitive(uint32_t, const Content & c)
 {
     const document::FieldValue & value = c.getValue();
     LOG(debug, "AttributeInserter: Adding value '%s'(%d) to attribute '%s' for docid '%d'",
@@ -401,13 +402,13 @@ SearchVisitor::PositionInserter::PositionInserter(search::AttributeVector & attr
 SearchVisitor::PositionInserter::~PositionInserter() {}
 
 void
-SearchVisitor::PositionInserter::onPrimitive(uint32_t, const IteratorContent & c)
+SearchVisitor::PositionInserter::onPrimitive(uint32_t, const Content & c)
 {
     (void) c;
 }
 
 void
-SearchVisitor::PositionInserter::onStructStart(const IteratorContent & c)
+SearchVisitor::PositionInserter::onStructStart(const Content & c)
 {
     const document::StructuredFieldValue & value = static_cast<const document::StructuredFieldValue &>(c.getValue());
     LOG(debug, "PositionInserter: Adding value '%s'(%d) to attribute '%s' for docid '%d'",
diff --git a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.h b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.h
index 7767a601ea4..47d6157525c 100644
--- a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.h
+++ b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.h
@@ -27,6 +27,7 @@
 #include <vespa/storage/visiting/visitor.h>
 #include <vespa/document/fieldvalue/fieldvalues.h>
 #include <vespa/documentapi/messagebus/messages/queryresultmessage.h>
+#include <vespa/document/fieldvalue/iteratorhandler.h>
 
 using namespace search::aggregation;
 
@@ -86,13 +87,12 @@ private:
      * This class gets callbacks when iterating through a field value and
      * inserts the values into a given attribute vector.
      **/
-    class AttributeInserter : public document::FieldValue::IteratorHandler {
+    class AttributeInserter : public document::fieldvalue::IteratorHandler {
     protected:
-        typedef document::FieldValue::IteratorHandler::Content IteratorContent;
         search::AttributeVector &      _attribute;
         search::AttributeVector::DocId _docId;
 
-        void onPrimitive(uint32_t fid, const IteratorContent & c) override;
+        void onPrimitive(uint32_t fid, const Content & c) override;
 
     public:
         AttributeInserter(search::AttributeVector & attribute, search::AttributeVector::DocId docId);
@@ -103,7 +103,7 @@ private:
         PositionInserter(search::AttributeVector & attribute, search::AttributeVector::DocId docId);
         ~PositionInserter();
     private:
-        void onPrimitive(uint32_t fid, const IteratorContent & c) override;
+        void onPrimitive(uint32_t fid, const Content & c) override;
         void onStructStart(const Content & fv) override;
         document::Field _fieldX;
         document::Field _fieldY;
@@ -466,4 +466,3 @@ public:
 };
 
 }
-
diff --git a/vdslib/src/tests/container/documentlisttest.cpp b/vdslib/src/tests/container/documentlisttest.cpp
index 5120c4efa05..02e2d0fd935 100644
--- a/vdslib/src/tests/container/documentlisttest.cpp
+++ b/vdslib/src/tests/container/documentlisttest.cpp
@@ -6,6 +6,7 @@
 #include <vespa/document/update/documentupdate.h>
 #include <vespa/vespalib/objects/nbostream.h>
 #include <vespa/vespalib/util/random.h>
+#include <vespa/vespalib/util/stringfmt.h>
 #include <vespa/vespalib/testkit/testapp.h>
 #include <cppunit/extensions/HelperMacros.h>
 
diff --git a/vdslib/src/vespa/vdslib/container/documentlist.cpp b/vdslib/src/vespa/vdslib/container/documentlist.cpp
index 2b105a77408..71bee5cf624 100644
--- a/vdslib/src/vespa/vdslib/container/documentlist.cpp
+++ b/vdslib/src/vespa/vdslib/container/documentlist.cpp
@@ -5,8 +5,9 @@
 #include <vespa/document/util/stringutil.h>
 #include <vespa/document/util/serializableexceptions.h>
 #include <vespa/document/update/documentupdate.h>
-#include <vespa/log/log.h>
+#include <vespa/document/datatype/documenttype.h>
 
+#include <vespa/log/log.h>
 LOG_SETUP(".vdslib.container.documentlist");
 
 using document::ByteBuffer;
diff --git a/vdslib/src/vespa/vdslib/container/mutabledocumentlist.cpp b/vdslib/src/vespa/vdslib/container/mutabledocumentlist.cpp
index 1c80e3ba8ee..f265a6c947f 100644
--- a/vdslib/src/vespa/vdslib/container/mutabledocumentlist.cpp
+++ b/vdslib/src/vespa/vdslib/container/mutabledocumentlist.cpp
@@ -2,6 +2,7 @@
 
 #include "mutabledocumentlist.h"
 #include <vespa/document/update/documentupdate.h>
+#include <vespa/document/datatype/documenttype.h>
 #include <vespa/vespalib/objects/nbostream.h>
 
 using vespalib::nbostream;
diff --git a/vdslib/src/vespa/vdslib/container/operationlist.cpp b/vdslib/src/vespa/vdslib/container/operationlist.cpp
index 6af338a4c0e..8c4515f15ff 100644
--- a/vdslib/src/vespa/vdslib/container/operationlist.cpp
+++ b/vdslib/src/vespa/vdslib/container/operationlist.cpp
@@ -3,6 +3,7 @@
 #include "operationlist.h"
 #include "documentlist.h"
 #include <vespa/document/update/documentupdate.h>
+#include <vespa/document/datatype/documenttype.h>
 #include <vespa/vespalib/objects/nbostream.h>
 
 namespace vdslib {
@@ -22,8 +23,7 @@ int OperationList::getRequiredBufferSize() const {
         switch(_operations[i].opt) {
         case OperationList::Operation::REMOVE:
         {
-            document::Document doc(*document::DataType::DOCUMENT,
-                                   _operations[i].docId);
+            document::Document doc(*document::DataType::DOCUMENT, _operations[i].docId);
             doc.serializeHeader(stream);
             break;
         }
diff --git a/vdslib/src/vespa/vdslib/container/parameters.cpp b/vdslib/src/vespa/vdslib/container/parameters.cpp
index d82f89da29d..c685a9b884f 100644
--- a/vdslib/src/vespa/vdslib/container/parameters.cpp
+++ b/vdslib/src/vespa/vdslib/container/parameters.cpp
@@ -4,6 +4,7 @@
 #include <vespa/vespalib/objects/nbostream.h>
 #include <vespa/vespalib/objects/hexdump.h>
 #include <vespa/vespalib/stllike/hash_map.hpp>
+#include <vespa/vespalib/util/xmlstream.h>
 
 using namespace vdslib;
 
@@ -60,7 +61,7 @@ void Parameters::onDeserialize(const document::DocumentTypeRepo &repo, document:
 void
 Parameters::printXml(document::XmlOutputStream& xos) const
 {
-    using namespace document;
+    using namespace vespalib::xml;
     xos << XmlTag("parameters");
     for (const auto & entry : _parameters) {
         xos << XmlTag("item")
diff --git a/vdslib/src/vespa/vdslib/container/visitorordering.h b/vdslib/src/vespa/vdslib/container/visitorordering.h
index 3d31a6ac078..e5a61d6332d 100644
--- a/vdslib/src/vespa/vdslib/container/visitorordering.h
+++ b/vdslib/src/vespa/vdslib/container/visitorordering.h
@@ -1,8 +1,8 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 #pragma once
 
-#include <vespa/fastos/types.h>
 #include <string>
+#include <cstdint>
 
 namespace vdslib {
 
diff --git a/vdslib/src/vespa/vdslib/distribution/distribution.cpp b/vdslib/src/vespa/vdslib/distribution/distribution.cpp
index a75ba591af5..cc099164306 100644
--- a/vdslib/src/vespa/vdslib/distribution/distribution.cpp
+++ b/vdslib/src/vespa/vdslib/distribution/distribution.cpp
@@ -13,11 +13,10 @@
 #include <vespa/config-stor-distribution.h>
 #include <list>
 
-#include <vespa/log/log.h>
+#include <vespa/log/bufferedlogger.h>
 LOG_SETUP(".vdslib.distribution");
 
-namespace storage {
-namespace lib {
+namespace storage::lib {
 
 namespace {
     std::vector<uint32_t> getDistributionBitMasks() {
@@ -711,5 +710,4 @@ Distribution::splitNodesIntoLeafGroups(IndexList nodeList) const
     return result;
 }
 
-} // lib
-} // storage
+}
diff --git a/vdslib/src/vespa/vdslib/distribution/group.cpp b/vdslib/src/vespa/vdslib/distribution/group.cpp
index 8ad3dcb2f80..2fce519298f 100644
--- a/vdslib/src/vespa/vdslib/distribution/group.cpp
+++ b/vdslib/src/vespa/vdslib/distribution/group.cpp
@@ -6,9 +6,9 @@
 #include <vespa/vespalib/util/exceptions.h>
 #include <vespa/vespalib/stllike/asciistream.h>
 #include <ostream>
+#include <cassert>
 
-namespace storage {
-namespace lib {
+namespace storage::lib {
 
 Group::Group(uint16_t index, vespalib::stringref name)
     : _name(name),
@@ -204,4 +204,3 @@ Group::getDistributionConfigHash() const {
 }
 
 }
-}
diff --git a/vdslib/src/vespa/vdslib/distribution/idealnodecalculatorimpl.cpp b/vdslib/src/vespa/vdslib/distribution/idealnodecalculatorimpl.cpp
index ca7ab742278..0ab4bc6b21a 100644
--- a/vdslib/src/vespa/vdslib/distribution/idealnodecalculatorimpl.cpp
+++ b/vdslib/src/vespa/vdslib/distribution/idealnodecalculatorimpl.cpp
@@ -3,9 +3,9 @@
 #include "idealnodecalculatorimpl.h"
 #include <vespa/vespalib/util/exceptions.h>
 #include <ostream>
+#include <cassert>
 
-namespace storage {
-namespace lib {
+namespace storage::lib {
 
 IdealNodeList::IdealNodeList() :
     _idealNodes()
@@ -71,5 +71,4 @@ IdealNodeCalculatorImpl::initUpStateMapping() {
     }
 }
 
-} // lib
-} // storage
+}
diff --git a/vdstestlib/src/vespa/vdstestlib/cppunit/dirconfig.cpp b/vdstestlib/src/vespa/vdstestlib/cppunit/dirconfig.cpp
index 64ff1377128..5e1c7c6b1b4 100644
--- a/vdstestlib/src/vespa/vdstestlib/cppunit/dirconfig.cpp
+++ b/vdstestlib/src/vespa/vdstestlib/cppunit/dirconfig.cpp
@@ -4,6 +4,7 @@
 
 #include <vespa/vespalib/io/fileutil.h>
 #include <vespa/vespalib/util/exceptions.h>
+#include <vespa/vespalib/util/stringfmt.h>
 #include <fstream>
 #include <atomic>
 
diff --git a/vespabase/CMakeLists.txt b/vespabase/CMakeLists.txt
index b4ed5c69044..4d21e535a42 100644
--- a/vespabase/CMakeLists.txt
+++ b/vespabase/CMakeLists.txt
@@ -6,7 +6,7 @@ vespa_install_script(src/start-cbinaries.sh vdsgetnodestate bin)
 vespa_install_script(src/start-cbinaries.sh vdsgetsystemstate bin)
 vespa_install_script(src/start-cbinaries.sh vdssetnodestate bin)
 vespa_install_script(src/start-cbinaries.sh vdsstat bin)
-vespa_install_script(src/start-cbinaries.sh verify_ranksetup bin)
+vespa_install_script(src/start-cbinaries.sh vespa-verify-ranksetup bin)
 vespa_install_script(src/start-cbinaries.sh vespa-config-status bin)
 vespa_install_script(src/start-cbinaries.sh vespadoclocator bin)
 vespa_install_script(src/start-cbinaries.sh vespaget bin)
@@ -17,13 +17,13 @@ vespa_install_script(src/start-cbinaries.sh vesparoute bin)
 vespa_install_script(src/start-cbinaries.sh vespa-transactionlog-inspect bin)
 vespa_install_script(src/start-cbinaries.sh vespavisit bin)
 vespa_install_script(src/start-cbinaries.sh vespavisittarget bin)
-vespa_install_script(src/start-cbinaries.sh distributord sbin)
-vespa_install_script(src/start-cbinaries.sh fdispatch sbin)
+vespa_install_script(src/start-cbinaries.sh vespa-vds-disktool bin)
+vespa_install_script(src/start-cbinaries.sh vespa-distributord sbin)
+vespa_install_script(src/start-cbinaries.sh vespa-fdispatch sbin)
 vespa_install_script(src/start-cbinaries.sh filedistributor sbin)
-vespa_install_script(src/start-cbinaries.sh proton sbin)
-vespa_install_script(src/start-cbinaries.sh storaged sbin)
+vespa_install_script(src/start-cbinaries.sh vespa-proton sbin)
+vespa_install_script(src/start-cbinaries.sh vespa-storaged sbin)
 
-vespa_install_script(src/vespa-allow-downgrade-from-6-to-5.sh vespa-allow-downgrade-from-6-to-5 bin)
 vespa_install_script(src/start-tool.sh vespa-start-tool.sh bin)
 
 vespa_install_script(src/rhel-prestart.sh vespa-prestart.sh bin)
diff --git a/vespabase/src/vespa-allow-downgrade-from-6-to-5.sh b/vespabase/src/vespa-allow-downgrade-from-6-to-5.sh
deleted file mode 100755
index c01333bdd79..00000000000
--- a/vespabase/src/vespa-allow-downgrade-from-6-to-5.sh
+++ /dev/null
@@ -1,8 +0,0 @@
-#!/bin/sh
-# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-for f in /home/y/var/db/vespa/search/*/*/documents/*/config/config-*
-do
-    echo $f
-    touch $f/smartsummary.cfg
-done
diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/RestUri.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/RestUri.java
index 7b6bc4e87c5..26c66085437 100644
--- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/RestUri.java
+++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/RestUri.java
@@ -105,13 +105,18 @@ public class RestUri {
             if (readPos >= rawParts.size()) {
                 throwUsage(originalPath);
             }
-            return rawParts.get(readPos++);
+            String nextToken = rawParts.get(readPos++);
+            return urlDecodeOrException(nextToken);
         }
 
         String restOfPath() throws RestApiException {
             String rawId = Joiner.on("/").join(rawParts.listIterator(readPos));
+            return urlDecodeOrException(rawId);
+        }
+
+        String urlDecodeOrException(String url) throws RestApiException {
             try {
-                return URLDecoder.decode(rawId, StandardCharsets.UTF_8.name());
+                return URLDecoder.decode(url, StandardCharsets.UTF_8.name());
             } catch (UnsupportedEncodingException e) {
                 throw new RestApiException(Response.createErrorResponse(BAD_REQUEST,"Problems decoding the URI: " + e.getMessage(), apiErrorCodes.ERROR_ID_DECODING_PATH));
             }
@@ -132,7 +137,8 @@ public class RestUri {
             case "number":
                 group = Optional.of(new Group(NUMBER_STREAMING, pathParser.nextTokenOrException()));
                 break;
-            case "docid": group = Optional.empty();
+            case "docid":
+                group = Optional.empty();
                 break;
             case "group":
                 group = Optional.of(new Group(GROUP_STREAMING, pathParser.nextTokenOrException()));
diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/RestUriTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/RestUriTest.java
index b2a8200b4e1..5dc0cef9e1a 100644
--- a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/RestUriTest.java
+++ b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/RestUriTest.java
@@ -94,6 +94,16 @@ public class RestUriTest {
     }
 
     @Test
+    public void testGroupUrlDecode() throws Exception {
+        RestUri restUri = new RestUri(createUri("/document/v1/namespace/doctype/group/group#123/myid", ""));
+        assertThat(restUri.getDocId(), is("myid"));
+        assertThat(restUri.getDocumentType(), is("doctype"));
+        assertThat(restUri.getGroup().get().name, is('g'));
+        assertThat(restUri.getGroup().get().value, is("group#123"));
+        assertThat(restUri.generateFullId(), is("id:namespace:doctype:g=group#123:myid"));
+    }
+
+    @Test
 	public void testGroupN() throws Exception {
         RestUri restUri = new RestUri(createUri("/document/v1/namespace/doctype/number/group/myid", ""));
         assertThat(restUri.getGroup().get().name, is('n'));
diff --git a/vespaclient-java/OWNERS b/vespaclient-java/OWNERS
new file mode 100644
index 00000000000..569bf1cc3a1
--- /dev/null
+++ b/vespaclient-java/OWNERS
@@ -0,0 +1 @@
+bjorncs
diff --git a/vespaclient-java/pom.xml b/vespaclient-java/pom.xml
new file mode 100644
index 00000000000..fb82220cb16
--- /dev/null
+++ b/vespaclient-java/pom.xml
@@ -0,0 +1,82 @@
+<?xml version="1.0"?>
+<!-- Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<!-- TODO: Remove this module on Vespa 7 -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>com.yahoo.vespa</groupId>
+        <artifactId>parent</artifactId>
+        <version>6-SNAPSHOT</version>
+    </parent>
+    <artifactId>vespaclient-java</artifactId>
+    <version>6-SNAPSHOT</version>
+    <dependencies>
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.mockito</groupId>
+            <artifactId>mockito-core</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.yahoo.vespa</groupId>
+            <artifactId>vespaclient-core</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.yahoo.vespa</groupId>
+            <artifactId>defaults</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.yahoo.vespa</groupId>
+            <artifactId>documentapi</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.yahoo.vespa</groupId>
+            <artifactId>container-dev</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>commons-cli</groupId>
+            <artifactId>commons-cli</artifactId>
+        </dependency>
+    </dependencies>
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <configuration>
+                    <compilerArgs>
+                        <arg>-Xlint:all</arg>
+                        <arg>-Xlint:-serial</arg>
+                        <arg>-Werror</arg>
+                    </compilerArgs>
+                </configuration>
+            </plugin>
+            <plugin>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <configuration>
+                    <descriptorRefs>
+                        <descriptorRef>jar-with-dependencies</descriptorRef>
+                    </descriptorRefs>
+                </configuration>
+                <executions>
+                    <execution>
+                        <id>make-assembly</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>single</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+</project>
diff --git a/vespaclient-java/src/main/java/com/yahoo/dummyreceiver/DummyReceiver.java b/vespaclient-java/src/main/java/com/yahoo/dummyreceiver/DummyReceiver.java
new file mode 100755
index 00000000000..7b104455f15
--- /dev/null
+++ b/vespaclient-java/src/main/java/com/yahoo/dummyreceiver/DummyReceiver.java
@@ -0,0 +1,196 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.dummyreceiver;
+
+import com.yahoo.concurrent.DaemonThreadFactory;
+import com.yahoo.documentapi.ThroughputLimitQueue;
+import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess;
+import com.yahoo.documentapi.messagebus.MessageBusParams;
+import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet;
+import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage;
+import com.yahoo.documentapi.messagebus.protocol.RemoveDocumentMessage;
+import com.yahoo.documentapi.messagebus.protocol.UpdateDocumentMessage;
+import com.yahoo.log.LogSetup;
+import com.yahoo.messagebus.DestinationSession;
+import com.yahoo.messagebus.EmptyReply;
+import com.yahoo.messagebus.Error;
+import com.yahoo.messagebus.ErrorCode;
+import com.yahoo.messagebus.Message;
+import com.yahoo.messagebus.MessageHandler;
+import com.yahoo.messagebus.Reply;
+import com.yahoo.messagebus.network.Identity;
+import com.yahoo.messagebus.network.rpc.RPCNetworkParams;
+
+import java.util.LinkedList;
+import java.util.List;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingDeque;
+import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+import static java.lang.System.out;
+
+public class DummyReceiver implements MessageHandler {
+    String name = null;
+    DestinationSession session;
+    MessageBusDocumentAccess da;
+    long sleepTime = 0;
+    long messageCount = 0;
+    int maxPendingCount = 0;
+    long silentNum = 0;
+    boolean instant = false;
+    ThreadPoolExecutor executor = null;
+    int threads = 10;
+    long maxQueueTime = -1;
+    BlockingQueue<Runnable> queue;
+    boolean verbose = false;
+
+    DummyReceiver() {
+    }
+
+    public class Task implements Runnable {
+        Reply reply;
+
+        public Task(Reply reply) {
+            this.reply = reply;
+        }
+
+        public void run() {
+            if (sleepTime > 0) {
+                try {
+                    Thread.sleep(sleepTime);
+                } catch (InterruptedException e) {
+                    e.printStackTrace();
+                }
+            }
+            session.reply(reply);
+        }
+    }
+
+    public void init() {
+        MessageBusParams params = new MessageBusParams(new LoadTypeSet());
+        params.setRPCNetworkParams(new RPCNetworkParams().setIdentity(new Identity(name)));
+        params.setDocumentManagerConfigId("client");
+        params.getMessageBusParams().setMaxPendingCount(maxPendingCount);
+        params.getMessageBusParams().setMaxPendingSize(0);
+        da = new MessageBusDocumentAccess(params);
+        queue = (maxQueueTime < 0) ? new LinkedBlockingDeque<>() : new ThroughputLimitQueue<>(maxQueueTime);
+        session = da.getMessageBus().createDestinationSession("default", true, this);
+        executor = new ThreadPoolExecutor(threads, threads, 5, TimeUnit.SECONDS, queue, new DaemonThreadFactory());
+        System.out.println("Registered listener at " + name + "/default with " + maxPendingCount + " max pending and sleep time of " + sleepTime);
+    }
+
+    public void handleMessage(Message message) {
+        messageCount++;
+        if ( silentNum == 0 ) {
+            System.out.println("Received message " + message + ". Received " + messageCount + " messages so far. In queue size " + queue.size());
+
+            if (verbose) {
+                if (message instanceof PutDocumentMessage) {
+                    System.out.println("  Document:\n" + ((PutDocumentMessage) message).getDocumentPut().getDocument().toXML("  "));
+                } else if (message instanceof RemoveDocumentMessage) {
+                    System.out.println("  Document id: " + ((RemoveDocumentMessage) message).getDocumentId());
+                } else if (message instanceof UpdateDocumentMessage) {
+                    System.out.println("  Update:\n  " + ((UpdateDocumentMessage) message).getDocumentUpdate().toString());
+                }
+            }
+        } else {
+            if ((messageCount % silentNum) == 0) {
+                System.out.println("Received " + messageCount + " messages so far. In queue size " + queue.size());
+            }
+        }
+
+        EmptyReply reply = new EmptyReply();
+        message.swapState(reply);
+
+        if ( ! instant ) {
+            try {
+                executor.execute(new Task(reply));
+            } catch (RejectedExecutionException e) {
+                reply.addError(new Error(ErrorCode.SESSION_BUSY, "Session " + name + "/default is busy"));
+                session.reply(reply);
+            }
+        } else {
+            session.reply(reply);
+        }
+    }
+
+    String getParam(List<String> args, String arg) throws IllegalArgumentException {
+        try {
+            return args.remove(0);
+        } catch (Exception e) {
+            System.err.println("--" + arg + " requires an argument");
+            throw new IllegalArgumentException(arg);
+        }
+    }
+
+    public void help() {
+        out.println("Simple receiver for messagebus messages. Prints the messages received to stdout.\n" +
+                    "\n" +
+                    "The options are:\n" +
+                    "  --instant          Reply in message thread." +
+                    "  --name arg         Slobrok name to register\n" +
+                    "  --maxqueuetime arg Adjust the in queue size to have a maximum queue wait period of this many ms (default -1 = unlimited)\n" +
+                    "  --silent #nummsg   Do not dump anything, but progress every #nummsg\n" +
+                    "  --sleeptime arg    The number of milliseconds to sleep per message, to simulate processing time\n" +
+                    "  --threads arg      The number of threads to process the incoming data\n" +
+                    "  --verbose          If set, dump the contents of certain messages to stdout");
+    }
+
+    boolean parseArgs(List<String> args) {
+        try {
+            while (!args.isEmpty()) {
+                String arg = args.remove(0);
+
+                if (arg.equals("-h") || arg.equals("--help")) {
+                    help();
+                    return false;
+                } else if ("--name".equals(arg)) {
+                    name = getParam(args, arg);
+                } else if ("--sleeptime".equals(arg)) {
+                    sleepTime = Long.parseLong(getParam(args, arg));
+                } else if ("--instant".equals(arg)) {
+                    instant = true;
+                } else if ("--silent".equals(arg)) {
+                    silentNum = Long.parseLong(getParam(args, arg));
+                } else if ("--maxqueuetime".equals(arg)) {
+                    maxQueueTime = Long.parseLong(getParam(args, arg));
+                } else if ("--threads".equals(arg)) {
+                    threads = Integer.parseInt(getParam(args, arg));
+                } else if ("--verbose".equals(arg)) {
+                    verbose = true;
+                } else {
+                    help();
+                    return false;
+                }
+            }
+
+            return true;
+        } catch (IllegalArgumentException e) {
+            return false;
+        }
+    }
+
+
+    public static void main(String[] args) {
+        LogSetup.initVespaLogging("dummyreceiver");
+        DummyReceiver rcv = new DummyReceiver();
+
+        List<String> l = new LinkedList<>();
+        for (String arg : args) {
+            l.add(arg);
+        }
+        if (!rcv.parseArgs(l)) {
+            System.exit(1);
+        }
+
+        rcv.init();
+        while (true) {
+            try {
+                Thread.sleep(100);
+            } catch (InterruptedException e) {
+                e.printStackTrace();
+            }
+        }
+    }
+}
diff --git a/vespaclient-java/src/main/java/com/yahoo/vespafeeder/Arguments.java b/vespaclient-java/src/main/java/com/yahoo/vespafeeder/Arguments.java
new file mode 100644
index 00000000000..249ed1fd70c
--- /dev/null
+++ b/vespaclient-java/src/main/java/com/yahoo/vespafeeder/Arguments.java
@@ -0,0 +1,191 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespafeeder;
+
+import com.yahoo.vespa.config.content.LoadTypeConfig;
+import com.yahoo.feedapi.DummySessionFactory;
+import com.yahoo.feedapi.MessageBusSessionFactory;
+import com.yahoo.feedapi.MessagePropertyProcessor;
+import com.yahoo.feedapi.SessionFactory;
+import com.yahoo.vespaclient.config.FeederConfig;
+
+import java.io.BufferedOutputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.LinkedList;
+import java.util.List;
+
+import static java.lang.System.out;
+
+/**
+ * Argument parsing class for the vespa feeder.
+ */
+public class Arguments {
+    public FeederConfig getFeederConfig() {
+        return new FeederConfig(feederConfigBuilder);
+    }
+
+    public List<String> getFiles() {
+        return files;
+    }
+
+    public String getMode() {
+        return mode;
+    }
+
+    public boolean isVerbose() {
+        return verbose;
+    }
+
+    private FeederConfig.Builder feederConfigBuilder = new FeederConfig.Builder();
+    private List<String> files = new ArrayList<String>();
+    private String dumpDocumentsFile = null;
+    private String mode = "standard";
+    private boolean validateOnly = false;
+    private boolean verbose = false;
+    SessionFactory sessionFactory = null;
+    MessagePropertyProcessor propertyProcessor = null;
+    private String priority = null;
+
+    public MessagePropertyProcessor getPropertyProcessor() {
+        return propertyProcessor;
+    }
+
+    public void help() {
+        out.println("This is a tool for feeding xml (deprecated) or json data to a Vespa application.\n" +
+                "\n" +
+                "The options are:\n" +
+                "  --abortondataerror arg (true) Whether or not to abort if the xml input has \n" +
+                "                                errors (true|false).\n" +
+                "  --abortonsenderror arg (true) Whether or not to abort if an error occured while\n" +
+                "                                sending operations to Vespa (true|false).\n" +
+                "  --file arg                    The name of the input files to read. These can \n" +
+                "                                also be passed as arguments without the option \n" +
+                "                                prefix. If none is given, this tool parses \n" +
+                "                                identifiers from standard in.\n" +
+                "  -h [ --help ]                 Shows this help page.\n" +
+                "  --maxpending arg              The maximum number of operations that are allowed\n" +
+                "                                to be pending at any given time. NOTE: This disables dynamic throttling. Use with care.\n" +
+                "  --maxpendingsize arg          The maximum size (in bytes) of operations that \n" +
+                "                                are allowed to be pending at any given time. \n" +
+                "  --maxfeedrate arg             Limits the feed rate to the given number (operations/second). You may still want to increase\n" +
+                "                                the max pending size if your feed rate doesn't reach the desired number.\n" +
+                "  --mode arg (=standard)        The mode to run vespafeeder in (standard | benchmark).\n" +
+                "  --noretry                     Turns off retries of recoverable failures.\n" +
+                "  --retrydelay arg (=1)         The time (in seconds) to wait between retries of \n" +
+                "                                a failed operation.\n" +
+                "  --route arg (=default)        The route to send the data to.\n" +
+                "  --timeout arg (=180)          The time (in seconds) allowed for sending \n" +
+                "                                operations.\n" +
+                "  --trace arg (=0)              The trace level of network traffic.\n" +
+                "  --validate                    Run validation tool on input files instead of \n" +
+                "                                feeding them.\n" +
+                "  --dumpDocuments <filename>    Specify a file where documents in the put are serialized.\n" +
+                "  --priority arg                Specify priority of sent messages (see documentation for priority values)\n" +
+                "  --create-if-non-existent      Enable setting of create-if-non-existent to true on all document updates in the given xml feed.\n" +
+                "  -v [ --verbose ]              Enable verbose output of progress.\n");
+    }
+
+    public class HelpShownException extends Exception {
+
+    }
+
+    public Arguments(String[] argList, SessionFactory factory) throws HelpShownException, FileNotFoundException {
+        parse(argList);
+
+        if (factory != null) {
+            sessionFactory = factory;
+        } else if (validateOnly) {
+            if (dumpDocumentsFile != null) {
+                BufferedOutputStream out = new BufferedOutputStream(new FileOutputStream(dumpDocumentsFile));
+                sessionFactory = new DummySessionFactory(null, out);
+            } else {
+                sessionFactory = new DummySessionFactory(null, null);
+            }
+        } else {
+            sessionFactory = new MessageBusSessionFactory(propertyProcessor);
+        }
+    }
+
+    void parse(String[] argList) throws HelpShownException {
+        List<String> args = new LinkedList<String>();
+        args.addAll(Arrays.asList(argList));
+
+        while (!args.isEmpty()) {
+            String arg = args.remove(0);
+
+            if (arg.equals("-h") || arg.equals("--help")) {
+                help();
+                throw new HelpShownException();
+            } else if ("--abortondataerror".equals(arg)) {
+                feederConfigBuilder.abortondocumenterror(getBoolean(getParam(args, arg)));
+            } else if ("--abortonsenderror".equals(arg)) {
+                feederConfigBuilder.abortonsenderror(getBoolean(getParam(args, arg)));
+            } else if ("--file".equals(arg)) {
+                files.add(getParam(args, arg));
+            } else if ("--maxpending".equals(arg)) {
+                feederConfigBuilder.maxpendingdocs(Integer.parseInt(getParam(args, arg)));
+            } else if ("--maxpendingsize".equals(arg)) {
+                feederConfigBuilder.maxpendingbytes(Integer.parseInt(getParam(args, arg)));
+            } else if ("--mode".equals(arg)) {
+                mode = getParam(args, arg);
+            } else if ("--noretry".equals(arg)) {
+                feederConfigBuilder.retryenabled(false);
+            } else if ("--retrydelay".equals(arg)) {
+                feederConfigBuilder.retrydelay(Integer.parseInt(getParam(args, arg)));
+            } else if ("--route".equals(arg)) {
+                feederConfigBuilder.route(getParam(args, arg));
+            } else if ("--timeout".equals(arg)) {
+                feederConfigBuilder.timeout(Double.parseDouble(getParam(args, arg)));
+            } else if ("--trace".equals(arg)) {
+                feederConfigBuilder.tracelevel(Integer.parseInt(getParam(args, arg)));
+            } else if ("--validate".equals(arg)) {
+                validateOnly = true;
+            } else if ("--dumpDocuments".equals(arg)) {
+                dumpDocumentsFile = getParam(args, arg);
+            } else if ("--maxfeedrate".equals(arg)) {
+                feederConfigBuilder.maxfeedrate(Double.parseDouble(getParam(args, arg)));
+            } else if ("--create-if-non-existent".equals(arg)) {
+                feederConfigBuilder.createifnonexistent(true);
+            } else if ("-v".equals(arg) || "--verbose".equals(arg)) {
+                verbose = true;
+            } else if ("--priority".equals(arg)) {
+                priority = getParam(args, arg);
+            } else {
+                files.add(arg);
+            }
+        }
+
+        propertyProcessor = new MessagePropertyProcessor(getFeederConfig(), new LoadTypeConfig(new LoadTypeConfig.Builder()));
+    }
+
+    private String getParam(List<String> args, String arg) throws IllegalArgumentException {
+        try {
+            return args.remove(0);
+        } catch (Exception e) {
+            System.err.println("--" + arg + " requires an argument");
+            throw new IllegalArgumentException(arg);
+        }
+    }
+
+    private Boolean getBoolean(String arg) {
+        if (arg.equalsIgnoreCase("yes")) {
+            return true;
+        } else if (arg.equalsIgnoreCase("no")) {
+            return false;
+        } else {
+            return Boolean.parseBoolean(arg);
+        }
+    }
+
+    public String getPriority() {
+        return priority;
+    }
+
+    public SessionFactory getSessionFactory() {
+        return sessionFactory;
+    }
+
+
+}
diff --git a/vespaclient-java/src/main/java/com/yahoo/vespafeeder/BenchmarkProgressPrinter.java b/vespaclient-java/src/main/java/com/yahoo/vespafeeder/BenchmarkProgressPrinter.java
new file mode 100644
index 00000000000..cc0d8f8c780
--- /dev/null
+++ b/vespaclient-java/src/main/java/com/yahoo/vespafeeder/BenchmarkProgressPrinter.java
@@ -0,0 +1,76 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespafeeder;
+
+import com.yahoo.clientmetrics.MessageTypeMetricSet;
+import com.yahoo.clientmetrics.RouteMetricSet;
+import com.yahoo.concurrent.Timer;
+import com.yahoo.metrics.Metric;
+import com.yahoo.metrics.MetricSet;
+import com.yahoo.metrics.MetricVisitor;
+
+import java.io.PrintStream;
+
+/**
+ * Class that takes progress from the feed and prints to a stream.
+ */
+public class BenchmarkProgressPrinter implements RouteMetricSet.ProgressCallback {
+    private final long startTime;
+    private final Timer timer;
+    private final PrintStream output;
+
+    public BenchmarkProgressPrinter(Timer timer, PrintStream output) {
+        this.timer = timer;
+        this.output = output;
+        this.startTime = timer.milliTime();
+    }
+
+    class PrintVisitor extends MetricVisitor {
+        private final PrintStream out;
+
+        PrintVisitor(PrintStream out) {
+            this.out = out;
+        }
+
+        @Override
+        public boolean visitMetricSet(MetricSet set, boolean autoGenerated) {
+            if (set instanceof MessageTypeMetricSet && set.getName().equals("total")) {
+                Metric m = set.getMetric("latency");
+                Metric count = set.getMetric("count");
+                Metric err = set.getMetric("errors.total");
+
+                long okCount = 0, errCount = 0, minLatency = 0, maxLatency = 0, avgLatency = 0;
+
+                if (m != null) {
+                    minLatency = m.getLongValue("min");
+                    maxLatency = m.getLongValue("max");
+                    avgLatency = m.getLongValue("average");
+                }
+                if (count != null) {
+                    okCount = count.getLongValue("count");
+                }
+
+                if (err != null) {
+                    errCount = err.getLongValue("count");
+                }
+                long timeUsed = timer.milliTime() - startTime;
+                out.println(timeUsed + ", " + okCount + ", " + errCount + ", " + minLatency + ", " + maxLatency + ", " + avgLatency);
+            }
+            return true;
+        }
+    }
+
+    @Override
+    public void onProgress(RouteMetricSet metrics) {
+	//metrics.visit(new PrintVisitor(output), false);
+    }
+
+    @Override
+    public void done(RouteMetricSet metrics) {
+        try {
+		output.println("# Time used, num ok, num error, min latency, max latency, average latency");
+            metrics.visit(new PrintVisitor(output), false);
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+}
diff --git a/vespaclient-java/src/main/java/com/yahoo/vespafeeder/FileRequest.java b/vespaclient-java/src/main/java/com/yahoo/vespafeeder/FileRequest.java
new file mode 100755
index 00000000000..3479221257d
--- /dev/null
+++ b/vespaclient-java/src/main/java/com/yahoo/vespafeeder/FileRequest.java
@@ -0,0 +1,14 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespafeeder;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+
+public class FileRequest extends InputStreamRequest {
+
+    FileRequest(File f) throws FileNotFoundException {
+        super(new FileInputStream(f));
+    }
+
+}
diff --git a/vespaclient-java/src/main/java/com/yahoo/vespafeeder/InputStreamRequest.java b/vespaclient-java/src/main/java/com/yahoo/vespafeeder/InputStreamRequest.java
new file mode 100644
index 00000000000..e69eb6727b0
--- /dev/null
+++ b/vespaclient-java/src/main/java/com/yahoo/vespafeeder/InputStreamRequest.java
@@ -0,0 +1,38 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespafeeder;
+
+import com.yahoo.container.jdisc.HttpRequest;
+
+import java.io.InputStream;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * This is needed because whoever wrote this library moronically decided to pass in-process communication through
+ * the HTTP layer. As the feeded is being phased out in favor of the standalone HTTP client we don't bother to clean
+ * it up properly.
+ *
+ * @author bratseth
+ */
+public class InputStreamRequest {
+
+    private InputStream input;
+    private Map<String, String> properties = new HashMap<>();
+
+    protected InputStreamRequest(InputStream input) {
+        this.input = input;
+    }
+
+    public void setProperty(String key, String value) {
+        properties.put(key, value);
+    }
+
+    public String getProperty(String key) {
+        return properties.get(key);
+    }
+
+    public HttpRequest toRequest() {
+        return HttpRequest.createTestRequest("", com.yahoo.jdisc.http.HttpRequest.Method.POST, input, properties);
+    }
+
+}
diff --git a/vespaclient-java/src/main/java/com/yahoo/vespafeeder/ProgressPrinter.java b/vespaclient-java/src/main/java/com/yahoo/vespafeeder/ProgressPrinter.java
new file mode 100644
index 00000000000..52087d33a47
--- /dev/null
+++ b/vespaclient-java/src/main/java/com/yahoo/vespafeeder/ProgressPrinter.java
@@ -0,0 +1,149 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespafeeder;
+
+import com.yahoo.clientmetrics.MessageTypeMetricSet;
+import com.yahoo.clientmetrics.RouteMetricSet;
+import com.yahoo.concurrent.Timer;
+import com.yahoo.metrics.Metric;
+import com.yahoo.metrics.MetricSet;
+import com.yahoo.metrics.MetricVisitor;
+import com.yahoo.metrics.SumMetric;
+
+import java.io.IOException;
+import java.io.PrintStream;
+import java.math.RoundingMode;
+import java.text.NumberFormat;
+import java.util.Locale;
+
+/**
+ * Class that takes progress from the feed and prints to a stream.
+ */
+public class ProgressPrinter implements RouteMetricSet.ProgressCallback {
+    private long startTime = 0;
+    private long lastProgressTime = 0;
+    private long lastVerboseProgress = 0;
+    final Timer timer;
+    final PrintStream output;
+
+    public ProgressPrinter(Timer timer, PrintStream output) {
+        this.timer = timer;
+        this.output = output;
+
+        startTime = timer.milliTime();
+        lastProgressTime = startTime;
+        lastVerboseProgress = startTime;
+    }
+
+    class PrintVisitor extends MetricVisitor {
+        final PrintStream out;
+        final NumberFormat format;
+
+        PrintVisitor(PrintStream out) {
+            this.out = out;
+            format = NumberFormat.getNumberInstance(Locale.US);
+            format.setMaximumFractionDigits(2);
+            format.setMinimumFractionDigits(2);
+            format.setMinimumIntegerDigits(1);
+            format.setParseIntegerOnly(false);
+            format.setRoundingMode(RoundingMode.HALF_UP);
+            format.setGroupingUsed(false);
+        }
+
+        @Override
+        public boolean visitMetricSet(MetricSet set, boolean autoGenerated) {
+            if (set instanceof MessageTypeMetricSet && !set.getName().equals("total")) {
+                Metric m = set.getMetric("latency");
+                Metric count = set.getMetric("count");
+                Metric err = set.getMetric("errors.total");
+
+                long okCount = 0, errCount = 0, ignored = 0;
+                long minLatency = 0, maxLatency = 0, avgLatency = 0;
+
+                if (m != null) {
+                    minLatency = m.getLongValue("min");
+                    maxLatency = m.getLongValue("max");
+                    avgLatency = m.getLongValue("average");
+                }
+                if (count != null) {
+                    okCount = count.getLongValue("count");
+                }
+                Metric ignoredMetric = set.getMetric("ignored");
+                if (ignoredMetric != null) {
+                    ignored = ignoredMetric.getLongValue("count");
+                }
+
+                if (err != null) {
+                    errCount = err.getLongValue("count");
+                }
+
+                long timeSinceStart = timer.milliTime() - startTime;
+
+                out.println(((MessageTypeMetricSet)set).getMessageName() + ":\t" +
+                        "ok: " + okCount +
+                        " msgs/sec: " + format.format((double)okCount * 1000 / timeSinceStart) +
+                        " failed: " + errCount +
+                        " ignored: " + ignored +
+                        " latency(min, max, avg): " + minLatency + ", " + maxLatency + ", " + avgLatency);
+            }
+            return true;
+        }
+    }
+
+    public static String getDashes(int count) {
+        String dashes = "";
+        for (int i = 0; i < count; i++) {
+            dashes += "-";
+        }
+
+        return dashes;
+    }
+
+    public synchronized void renderStatusText(RouteMetricSet metrics, PrintStream stream) throws IOException {
+        String headline = "Messages sent to vespa (route " + metrics.getName() + ") :";
+        stream.println(headline);
+        stream.println(getDashes(headline.length()));
+        metrics.visit(new PrintVisitor(stream), false);
+    }
+
+    public long getOkMessageCount(RouteMetricSet metrics) {
+        SumMetric sum = (SumMetric)metrics.getMetric("total");
+
+        MetricSet ms = (MetricSet)sum.generateSum();
+        if (ms != null) {
+            Metric latency = ms.getMetric("latency");
+            if (latency != null) {
+                return latency.getLongValue("count");
+            }
+        }
+
+        return 0;
+    }
+
+    @Override
+    public void onProgress(RouteMetricSet metrics) {
+        try {
+            long timeNow = timer.milliTime();
+
+            if (timeNow - lastVerboseProgress > 30000) {
+                output.println("\n");
+                renderStatusText(metrics, output);
+                lastVerboseProgress = timeNow;
+            } else if (timeNow - lastProgressTime > 1000) {
+                output.print("\rSuccessfully sent " + getOkMessageCount(metrics) + " messages so far");
+                lastProgressTime = timeNow;
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+
+    @Override
+    public void done(RouteMetricSet metrics) {
+        try {
+            output.println("\n");
+            renderStatusText(metrics, output);
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+}
diff --git a/vespaclient-java/src/main/java/com/yahoo/vespafeeder/VespaFeeder.java b/vespaclient-java/src/main/java/com/yahoo/vespafeeder/VespaFeeder.java
new file mode 100755
index 00000000000..a6ede66c43d
--- /dev/null
+++ b/vespaclient-java/src/main/java/com/yahoo/vespafeeder/VespaFeeder.java
@@ -0,0 +1,171 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespafeeder;
+
+import com.yahoo.clientmetrics.RouteMetricSet;
+import com.yahoo.concurrent.ThreadFactoryFactory;
+import com.yahoo.document.DocumentTypeManager;
+import com.yahoo.document.DocumentTypeManagerConfigurer;
+import com.yahoo.feedapi.FeedContext;
+import com.yahoo.feedhandler.FeedResponse;
+import com.yahoo.feedhandler.NullFeedMetric;
+import com.yahoo.feedhandler.VespaFeedHandler;
+import com.yahoo.log.LogSetup;
+import com.yahoo.concurrent.SystemTimer;
+import com.yahoo.vespaclient.ClusterList;
+
+import java.io.*;
+import java.util.*;
+import java.util.concurrent.Executor;
+import java.util.concurrent.Executors;
+
+public class VespaFeeder {
+
+    Arguments args;
+    DocumentTypeManager manager;
+    Executor threadPool = Executors.newCachedThreadPool(ThreadFactoryFactory.getThreadFactory("vespafeeder"));
+
+    public VespaFeeder(Arguments args, DocumentTypeManager manager) {
+        this.args = args;
+        this.manager = manager;
+    }
+
+    public static class FeedErrorException extends Exception {
+        String message;
+
+        public FeedErrorException(String message) {
+            this.message = message;
+        }
+
+        @Override
+        public String getMessage() {
+            return message;
+        }
+
+    }
+
+    static FeedErrorException renderErrors(List<String> errors) {
+        StringBuilder buffer = new StringBuilder();
+
+        if (!errors.isEmpty()) {
+            String headline = (errors.size() > 10) ? "First 10 errors (of " + errors.size() + "):" : "Errors:";
+            buffer.append(headline).append("\n");
+            for (int i = 0; i < headline.length(); ++i) {
+                buffer.append("-");
+            }
+            buffer.append("\n");
+            for (int i = 0; i < errors.size() && i < 10; ++i) {
+                buffer.append("    ").append(errors.get(i)).append("\n");
+            }
+        }
+
+        return new FeedErrorException(buffer.toString());
+    }
+
+    public RouteMetricSet.ProgressCallback createProgressCallback(PrintStream output) {
+        if ("benchmark".equals(args.getMode())) {
+            return new BenchmarkProgressPrinter(SystemTimer.INSTANCE, output);
+        } else {
+            return new ProgressPrinter(SystemTimer.INSTANCE, output);
+        }
+    }
+
+    void parseFiles(InputStream stdin, PrintStream output) throws Exception {
+        FeedContext context = new FeedContext(
+                args.getPropertyProcessor(),
+                args.getSessionFactory(),
+                manager,
+                new ClusterList(), new NullFeedMetric());
+
+        final BufferedInputStream input = new BufferedInputStream(stdin);
+        VespaFeedHandler handler = VespaFeedHandler.createFromContext(context, threadPool);
+
+        if (args.getFiles().isEmpty()) {
+            InputStreamRequest req = new InputStreamRequest(input);
+            setProperties(req, input);
+            FeedResponse response = (FeedResponse)handler.handle(req.toRequest(), createProgressCallback(output));
+            if ( ! response.isSuccess()) {
+                throw renderErrors(response.getErrorList());
+            }
+        } else {
+            if (args.isVerbose()) {
+                for (String fileName : args.getFiles()) {
+                    long thisSize = new File(fileName).length();
+                    output.println("Size of file '" + fileName + "' is " + thisSize + " B.");
+                }
+            }
+
+            for (String fileName : args.getFiles()) {
+                File f = new File(fileName);
+                FileRequest req = new FileRequest(f);
+                final BufferedInputStream inputSnooper = new BufferedInputStream(new FileInputStream(fileName));
+                setProperties(req, inputSnooper);
+                inputSnooper.close();
+                FeedResponse response = (FeedResponse)handler.handle(req.toRequest(), createProgressCallback(output));
+                if (!response.isSuccess()) {
+                    throw renderErrors(response.getErrorList());
+                }
+            }
+        }
+    }
+
+    // use BufferedInputStream to enforce the input.markSupported() == true
+    private void setProperties(InputStreamRequest req, BufferedInputStream input) throws IOException {
+        setPriority(req);
+        setCreateIfNonExistent(req);
+        setJsonInput(req, input);
+    }
+
+    private void setPriority(InputStreamRequest req) {
+        if (args.getPriority() != null) {
+            req.setProperty("priority", args.getPriority());
+        }
+    }
+
+    private void setCreateIfNonExistent(InputStreamRequest req) {
+        if (args.getFeederConfig().createifnonexistent()) {
+            req.setProperty("createifnonexistent", "true");
+        }
+    }
+
+    // package access for easy testing
+    static void setJsonInput(InputStreamRequest req, BufferedInputStream input) throws IOException {
+        input.mark(4);
+        int b = input.read();
+        input.reset();
+        // A valid JSON feed will always start with '['
+        if (b == '[') {
+            req.setProperty(VespaFeedHandler.JSON_INPUT, Boolean.TRUE.toString());
+        } else {
+            req.setProperty(VespaFeedHandler.JSON_INPUT, Boolean.FALSE.toString());
+        }
+    }
+
+    public static void main(String[] args) {
+        LogSetup.initVespaLogging("vespafeeder");
+
+        try {
+            Arguments arguments = new Arguments(args, null);
+
+            DocumentTypeManager manager = new DocumentTypeManager();
+            DocumentTypeManagerConfigurer.configure(manager, "client").close();
+
+            VespaFeeder feeder = new VespaFeeder(arguments, manager);
+            feeder.parseFiles(System.in, System.out);
+            System.exit(0);
+        } catch (Arguments.HelpShownException e) {
+            System.exit(0);
+        } catch (IllegalArgumentException e) {
+            System.exit(1);
+        } catch (FileNotFoundException e) {
+            System.err.println("Could not open file " + e.getMessage());
+            System.exit(1);
+        } catch (FeedErrorException e) {
+            System.err.println("\n" + e.getMessage());
+            System.exit(1);
+        } catch (Exception e) {
+            System.err.println("Got exception " + e.getMessage() + ", aborting feed.");
+            System.exit(1);
+        }
+    }
+
+}
diff --git a/vespaclient-java/src/main/java/com/yahoo/vespaget/ClientParameters.java b/vespaclient-java/src/main/java/com/yahoo/vespaget/ClientParameters.java
new file mode 100644
index 00000000000..b57a9f7bf85
--- /dev/null
+++ b/vespaclient-java/src/main/java/com/yahoo/vespaget/ClientParameters.java
@@ -0,0 +1,160 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespaget;
+
+import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol;
+
+import java.util.Iterator;
+/**
+ * This class contains the the program parameters.
+ *
+ * @author bjorncs
+ */
+public class ClientParameters {
+
+    // Determines if the help page should be presented
+    public final boolean help;
+    // Contains the document ids. Is backed by either a list iterator if the ids were given as CLI arguments or Scanner(System.in) if ids are provided by standard input.
+    public final Iterator<String> documentIds;
+    // Print only the id for retrieved documents
+    public final boolean printIdsOnly;
+    // Determines which document fields to retrieve. Default is '[all]'.
+    public final String fieldSet;
+    // The Vespa route
+    public final String route;
+    // Alternative way to specify the route using cluster name.
+    public final String cluster;
+    // The configuration id for message bus. Default "client".
+    public final String configId;
+    // Determines if the serialized document size should be printed
+    public final boolean showDocSize;
+    // Document request timeout
+    public final double timeout;
+    // Determines whether or not the document request can be resent
+    public final boolean noRetry;
+    // Vespa trace level
+    public final int traceLevel;
+    // Document request priority
+    public final DocumentProtocol.Priority priority;
+    // Determines the Vespa load type
+    public final String loadTypeName;
+    // If full documents are printed, they will be printed as JSON (instead of XML)
+    public final boolean jsonOutput;
+
+
+    private ClientParameters(
+            boolean help, Iterator<String> documentIds, boolean printIdsOnly,
+            String fieldSet, String route, String cluster, String configId,
+            boolean showDocSize, double timeout, boolean noRetry, int traceLevel,
+            DocumentProtocol.Priority priority, String loadTypeName, boolean jsonOutput) {
+
+        this.help = help;
+        this.documentIds = documentIds;
+        this.printIdsOnly = printIdsOnly;
+        this.fieldSet = fieldSet;
+        this.route = route;
+        this.cluster = cluster;
+        this.configId = configId;
+        this.showDocSize = showDocSize;
+        this.timeout = timeout;
+        this.noRetry = noRetry;
+        this.traceLevel = traceLevel;
+        this.priority = priority;
+        this.loadTypeName = loadTypeName;
+        this.jsonOutput = jsonOutput;
+    }
+
+    public static class Builder {
+        private boolean help;
+        private Iterator<String> documentIds;
+        private boolean printIdsOnly;
+        private String fieldSet;
+        private String route;
+        private String cluster;
+        private String configId;
+        private boolean showDocSize;
+        private double timeout;
+        private boolean noRetry;
+        private int traceLevel;
+        private DocumentProtocol.Priority priority;
+        private String loadTypeName;
+        private boolean jsonOutput;
+
+        public Builder setHelp(boolean help) {
+            this.help = help;
+            return this;
+        }
+
+        public Builder setDocumentIds(Iterator<String> documentIds) {
+            this.documentIds = documentIds;
+            return this;
+        }
+
+        public Builder setPrintIdsOnly(boolean printIdsOnly) {
+            this.printIdsOnly = printIdsOnly;
+            return this;
+        }
+
+        public Builder setFieldSet(String fieldSet) {
+            this.fieldSet = fieldSet;
+            return this;
+        }
+
+        public Builder setRoute(String route) {
+            this.route = route;
+            return this;
+        }
+
+        public Builder setCluster(String cluster) {
+            this.cluster = cluster;
+            return this;
+        }
+
+        public Builder setConfigId(String configId) {
+            this.configId = configId;
+            return this;
+        }
+
+        public Builder setShowDocSize(boolean showDocSize) {
+            this.showDocSize = showDocSize;
+            return this;
+        }
+
+        public Builder setTimeout(double timeout) {
+            this.timeout = timeout;
+            return this;
+        }
+
+        public Builder setNoRetry(boolean noRetry) {
+            this.noRetry = noRetry;
+            return this;
+        }
+
+        public Builder setTraceLevel(int traceLevel) {
+            this.traceLevel = traceLevel;
+            return this;
+        }
+
+        public Builder setPriority(DocumentProtocol.Priority priority) {
+            this.priority = priority;
+            return this;
+        }
+
+        public Builder setLoadTypeName(String loadTypeName) {
+            this.loadTypeName = loadTypeName;
+            return this;
+        }
+
+        public Builder setJsonOutput(boolean jsonOutput) {
+            this.jsonOutput = jsonOutput;
+            return this;
+        }
+
+        public ClientParameters build() {
+            return new ClientParameters(
+                    help, documentIds, printIdsOnly, fieldSet, route, cluster, configId,
+                    showDocSize, timeout, noRetry, traceLevel, priority, loadTypeName, jsonOutput);
+        }
+    }
+
+
+}
diff --git a/vespaclient-java/src/main/java/com/yahoo/vespaget/CommandLineOptions.java b/vespaclient-java/src/main/java/com/yahoo/vespaget/CommandLineOptions.java
new file mode 100644
index 00000000000..cbaef17a70a
--- /dev/null
+++ b/vespaclient-java/src/main/java/com/yahoo/vespaget/CommandLineOptions.java
@@ -0,0 +1,263 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespaget;
+
+import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.DefaultParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+
+import java.io.InputStream;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Scanner;
+
+/**
+ * This class is responsible for parsing the command line arguments and print the help page.
+ *
+ * @author bjorncs
+ */
+public class CommandLineOptions {
+
+    public static final String HELP_OPTION = "help";
+    public static final String PRINTIDS_OPTION = "printids";
+    public static final String HEADERSONLY_OPTION = "headersonly";
+    public static final String FIELDSET_OPTION = "fieldset";
+    public static final String CLUSTER_OPTION = "cluster";
+    public static final String ROUTE_OPTION = "route";
+    public static final String CONFIGID_OPTION = "configid";
+    public static final String SHOWDOCSIZE_OPTION = "showdocsize";
+    public static final String TIMEOUT_OPTION = "timeout";
+    public static final String NORETRY_OPTION = "noretry";
+    public static final String TRACE_OPTION = "trace";
+    public static final String PRIORITY_OPTION = "priority";
+    public static final String LOADTYPE_OPTION = "loadtype";
+    public static final String JSONOUTPUT_OPTION = "jsonoutput";
+
+    private final Options options = createOptions();
+    private final InputStream stdIn;
+
+    public CommandLineOptions(InputStream stdIn) {
+        this.stdIn = stdIn;
+    }
+
+    public CommandLineOptions() {
+        this(System.in);
+    }
+
+    @SuppressWarnings("AccessStaticViaInstance")
+    private static Options createOptions() {
+        Options options = new Options();
+
+        options.addOption(Option.builder("h")
+                .hasArg(false)
+                .desc("Show this syntax page.")
+                .longOpt(HELP_OPTION)
+                .build());
+
+        options.addOption(Option.builder("i")
+                .hasArg(false)
+                .desc("Show only identifiers of retrieved documents.")
+                .longOpt(PRINTIDS_OPTION)
+                .build());
+
+        options.addOption(Option.builder("e")
+                .hasArg(false)
+                .desc("Retrieve header fields only. [Deprecated].")
+                .longOpt(HEADERSONLY_OPTION).build());
+
+        options.addOption(Option.builder("f")
+                .hasArg(true)
+                .desc("Retrieve the specified fields only (see http://vespa.corp.yahoo.com/5/documentation/reference/fieldsets.html) (default '[all]')")
+                .longOpt(FIELDSET_OPTION)
+                .argName("fieldset").build());
+
+        options.addOption(Option.builder("u")
+                .hasArg(true)
+                .desc("Send request to the given content cluster.")
+                .longOpt(CLUSTER_OPTION)
+                .argName("cluster").build());
+
+        options.addOption(Option.builder("r")
+                .hasArg(true)
+                .desc("Send request to the given messagebus route.")
+                .longOpt(ROUTE_OPTION)
+                .argName("route").build());
+
+        options.addOption(Option.builder("c")
+                .hasArg(true)
+                .desc("Use the specified config id for messagebus configuration.")
+                .longOpt(CONFIGID_OPTION)
+                .argName("configid").build());
+
+        options.addOption(Option.builder("s")
+                .hasArg(false)
+                .desc("Show binary size of document.")
+                .longOpt(SHOWDOCSIZE_OPTION).build());
+
+        options.addOption(Option.builder("t")
+                .hasArg(true)
+                .desc("Set timeout for the request in seconds (default 0).")
+                .longOpt(TIMEOUT_OPTION)
+                .argName("timeout")
+                .type(Number.class).build());
+
+        options.addOption(Option.builder("n")
+                .hasArg(false)
+                .desc("Do not retry operation on transient errors, as is default.")
+                .longOpt(NORETRY_OPTION).build());
+
+        options.addOption(Option.builder("a")
+                .hasArg(true)
+                .desc("Trace level to use (default 0).")
+                .longOpt(TRACE_OPTION)
+                .argName("trace")
+                .type(Number.class).build());
+
+        options.addOption(Option.builder("p")
+                .hasArg(true)
+                .desc("Priority (default 6).")
+                .longOpt(PRIORITY_OPTION)
+                .argName("priority")
+                .type(Number.class).build());
+
+        options.addOption(Option.builder("l")
+                .hasArg(true)
+                .desc("Load type (default \"\").")
+                .longOpt(LOADTYPE_OPTION)
+                .argName("loadtype").build());
+
+        options.addOption(Option.builder("j")
+                .hasArg(false)
+                .desc("JSON output")
+                .longOpt(JSONOUTPUT_OPTION).build());
+
+        return options;
+    }
+
+    public void printHelp() {
+        HelpFormatter formatter = new HelpFormatter();
+
+        formatter.printHelp(
+                "vespaget <options> [documentid...]", "Fetch a document from a Vespa Content cluster.", options,
+                "If one or more document identifier are specified, these documents will be " +
+                        "retrieved. Otherwise, document identifiers (separated with line break) will be read from standard in.\n",
+                false);
+    }
+
+    public ClientParameters parseCommandLineArguments(String[] args) throws IllegalArgumentException {
+        try {
+            CommandLineParser clp = new DefaultParser();
+            CommandLine cl = clp.parse(options, args);
+
+            boolean printIdsOnly = cl.hasOption(PRINTIDS_OPTION);
+            boolean headersOnly = cl.hasOption(HEADERSONLY_OPTION);
+            String fieldSet = cl.getOptionValue(FIELDSET_OPTION, "");
+            String cluster = cl.getOptionValue(CLUSTER_OPTION, "");
+            String route = cl.getOptionValue(ROUTE_OPTION, "");
+            String configId = cl.getOptionValue(CONFIGID_OPTION, "");
+            boolean help = cl.hasOption(HELP_OPTION);
+            String loadtype = cl.getOptionValue(LOADTYPE_OPTION, "");
+            boolean noRetry = cl.hasOption(NORETRY_OPTION);
+            boolean showDocSize = cl.hasOption(SHOWDOCSIZE_OPTION);
+            boolean jsonOutput = cl.hasOption(JSONOUTPUT_OPTION);
+            int trace = getTrace(cl);
+            DocumentProtocol.Priority priority = getPriority(cl);
+            double timeout = getTimeout(cl);
+            Iterator<String> documentIds = getDocumentIds(cl);
+
+            if (printIdsOnly && headersOnly) {
+                throw new IllegalArgumentException("Print ids and headers only options are mutually exclusive.");
+            }
+            if ((printIdsOnly || headersOnly) && !fieldSet.isEmpty()) {
+                throw new IllegalArgumentException("Field set option can not be used in combination with print ids or headers only options.");
+            }
+
+            if (printIdsOnly) {
+                fieldSet = "[id]";
+            } else if (headersOnly) {
+                fieldSet = "[header]";
+            } else if (fieldSet.isEmpty()) {
+                fieldSet = "[all]";
+            }
+
+            if (!cluster.isEmpty() && !route.isEmpty()) {
+                throw new IllegalArgumentException("Cluster and route options are mutually exclusive.");
+            }
+
+            if (route.isEmpty() && cluster.isEmpty()) {
+                route = "default";
+            }
+
+            if (trace < 0 || trace > 9) {
+                throw new IllegalArgumentException("Invalid tracelevel: " + trace);
+            }
+
+            if (configId.isEmpty()) {
+                configId = "client";
+            }
+
+            ClientParameters.Builder paramsBuilder = new ClientParameters.Builder();
+            return paramsBuilder
+                    .setDocumentIds(documentIds)
+                    .setConfigId(configId)
+                    .setFieldSet(fieldSet)
+                    .setHelp(help)
+                    .setPrintIdsOnly(printIdsOnly)
+                    .setLoadTypeName(loadtype)
+                    .setNoRetry(noRetry)
+                    .setCluster(cluster)
+                    .setRoute(route)
+                    .setShowDocSize(showDocSize)
+                    .setTraceLevel(trace)
+                    .setPriority(priority)
+                    .setTimeout(timeout)
+                    .setJsonOutput(jsonOutput)
+                    .build();
+        } catch (ParseException pe) {
+            throw new IllegalArgumentException(pe.getMessage());
+        }
+    }
+
+    private Iterator<String> getDocumentIds(CommandLine cl) {
+        // Fetch document ids from stdin if no ids are passed in as command line arguments
+        List<String> documentIds = Arrays.asList(cl.getArgs());
+        // WARNING: CommandLine.getArgs may return a single empty string as the only element
+        if (documentIds.isEmpty() ||
+                documentIds.size() == 1 && documentIds.get(0).isEmpty()) {
+            return new Scanner(stdIn);
+        } else {
+            return documentIds.iterator();
+        }
+    }
+
+    private static double getTimeout(CommandLine cl) throws ParseException {
+        Number timeoutObj = (Number) cl.getParsedOptionValue(TIMEOUT_OPTION);
+        return timeoutObj != null ? timeoutObj.doubleValue() : 0;
+    }
+
+    private static int getTrace(CommandLine cl) throws ParseException {
+        Number traceObj = (Number) cl.getParsedOptionValue(TRACE_OPTION);
+        return traceObj != null ? traceObj.intValue() : 0;
+    }
+
+    private static DocumentProtocol.Priority getPriority(CommandLine cl) throws ParseException {
+        Number priorityObj = (Number) cl.getParsedOptionValue(PRIORITY_OPTION);
+        int priorityNumber = priorityObj != null ? priorityObj.intValue() : DocumentProtocol.Priority.NORMAL_2.getValue();
+        return parsePriority(priorityNumber);
+    }
+
+    private static DocumentProtocol.Priority parsePriority(int n) {
+        for (DocumentProtocol.Priority priority : DocumentProtocol.Priority.values()) {
+            if (priority.getValue() == n) {
+                return priority;
+            }
+        }
+        throw new IllegalArgumentException("Invalid priority: " + n);
+    }
+
+}
diff --git a/vespaclient-java/src/main/java/com/yahoo/vespaget/DocumentAccessFactory.java b/vespaclient-java/src/main/java/com/yahoo/vespaget/DocumentAccessFactory.java
new file mode 100644
index 00000000000..6836f033c11
--- /dev/null
+++ b/vespaclient-java/src/main/java/com/yahoo/vespaget/DocumentAccessFactory.java
@@ -0,0 +1,17 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespaget;
+
+import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess;
+import com.yahoo.documentapi.messagebus.MessageBusParams;
+
+/**
+ * Factory class for {@link com.yahoo.documentapi.messagebus.MessageBusDocumentAccess}.
+ *
+ * @author bjorncs
+ */
+public class DocumentAccessFactory {
+
+    public MessageBusDocumentAccess createDocumentAccess(MessageBusParams messageBusParams) {
+        return new MessageBusDocumentAccess(messageBusParams);
+    }
+}
diff --git a/vespaclient-java/src/main/java/com/yahoo/vespaget/DocumentRetriever.java b/vespaclient-java/src/main/java/com/yahoo/vespaget/DocumentRetriever.java
new file mode 100644
index 00000000000..6e52e89c580
--- /dev/null
+++ b/vespaclient-java/src/main/java/com/yahoo/vespaget/DocumentRetriever.java
@@ -0,0 +1,207 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespaget;
+
+import com.yahoo.document.Document;
+import com.yahoo.document.DocumentId;
+import com.yahoo.document.json.JsonWriter;
+import com.yahoo.documentapi.SyncParameters;
+import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess;
+import com.yahoo.documentapi.messagebus.MessageBusParams;
+import com.yahoo.documentapi.messagebus.MessageBusSyncSession;
+import com.yahoo.documentapi.messagebus.loadtypes.LoadType;
+import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet;
+import com.yahoo.documentapi.messagebus.protocol.GetDocumentMessage;
+import com.yahoo.documentapi.messagebus.protocol.GetDocumentReply;
+import com.yahoo.messagebus.Message;
+import com.yahoo.messagebus.Reply;
+import com.yahoo.messagebus.Trace;
+import com.yahoo.text.Utf8;
+import com.yahoo.vespaclient.ClusterDef;
+import com.yahoo.vespaclient.ClusterList;
+
+import java.util.Iterator;
+import java.util.Map;
+
+/**
+ * The document retriever is responsible for retrieving documents using the Document API and printing the result to standard out.
+ *
+ * @author bjorncs
+ */
+public class DocumentRetriever {
+
+    private final ClusterList clusterList;
+    private final DocumentAccessFactory documentAccessFactory;
+    private final ClientParameters params;
+    private final LoadTypeSet loadTypeSet;
+
+    private MessageBusSyncSession session;
+    private MessageBusDocumentAccess documentAccess;
+
+    public DocumentRetriever(ClusterList clusterList,
+                             DocumentAccessFactory documentAccessFactory,
+                             LoadTypeSet loadTypeSet,
+                             ClientParameters params) {
+        this.clusterList = clusterList;
+        this.documentAccessFactory = documentAccessFactory;
+        this.loadTypeSet = loadTypeSet;
+        this.params = params;
+    }
+
+    public void shutdown() {
+        try {
+            if (session != null) {
+                session.destroy();
+            }
+        } catch (IllegalStateException e) {
+            // Ignore exception on shutdown
+        }
+        try {
+            if (documentAccess != null) {
+                documentAccess.shutdown();
+            }
+        } catch (IllegalStateException e) {
+            // Ignore exception on shutdown
+        }
+    }
+
+    public void retrieveDocuments() throws DocumentRetrieverException {
+        boolean first = true;
+        String route = params.cluster.isEmpty() ? params.route : resolveClusterRoute(params.cluster);
+        LoadType loadType = params.loadTypeName.isEmpty() ? null : resolveLoadType(params.loadTypeName);
+
+        MessageBusParams messageBusParams = createMessageBusParams(params.configId, params.timeout, route);
+        documentAccess = documentAccessFactory.createDocumentAccess(messageBusParams);
+        session = documentAccess.createSyncSession(new SyncParameters());
+        int trace = params.traceLevel;
+        if (trace > 0) {
+            session.setTraceLevel(trace);
+        }
+
+        Iterator<String> iter = params.documentIds;
+        if (params.jsonOutput && !params.printIdsOnly) {
+            System.out.println('[');
+        }
+        while (iter.hasNext()) {
+            if (params.jsonOutput && !params.printIdsOnly) {
+                if (!first) {
+                    System.out.println(',');
+                } else {
+                    first = false;
+                }
+            }
+            String docid = iter.next();
+            Message msg = createDocumentRequest(docid, loadType);
+            Reply reply = session.syncSend(msg);
+            printReply(reply);
+        }
+        if (params.jsonOutput && !params.printIdsOnly) {
+            System.out.println(']');
+        }
+    }
+
+    private String resolveClusterRoute(String clusterName) throws DocumentRetrieverException {
+        if (clusterList.getStorageClusters().isEmpty()) {
+            throw new DocumentRetrieverException("The Vespa cluster does not have any content clusters declared.");
+        }
+
+        ClusterDef clusterDef = null;
+        for (ClusterDef c : clusterList.getStorageClusters()) {
+            if (c.getName().equals(clusterName)) {
+                clusterDef = c;
+            }
+        }
+        if (clusterDef == null) {
+            String names = createClusterNamesString();
+            throw new DocumentRetrieverException(String.format(
+                    "The Vespa cluster contains the content clusters %s, not %s. Please select a valid vespa cluster.",
+                    names, clusterName));
+        }
+        return String.format("[Storage:cluster=%s;clusterconfigid=%s]", clusterDef.getName(), clusterDef.getConfigId());
+    }
+
+    private LoadType resolveLoadType(String loadTypeName) throws DocumentRetrieverException {
+        Map<String, LoadType> loadTypesNameMap = loadTypeSet.getNameMap();
+        if (!loadTypesNameMap.containsKey(loadTypeName)) {
+            throw new DocumentRetrieverException(String.format("Loadtype with name '%s' does not exist.\n", loadTypeName));
+        } else {
+            return loadTypesNameMap.get(loadTypeName);
+        }
+    }
+
+    private MessageBusParams createMessageBusParams(String configId, double timeout, String route) {
+        MessageBusParams messageBusParams = new MessageBusParams(loadTypeSet);
+        messageBusParams.setRoute(route);
+        messageBusParams.setProtocolConfigId(configId);
+        messageBusParams.setRoutingConfigId(configId);
+        messageBusParams.setDocumentManagerConfigId(configId);
+
+        if (timeout > 0) {
+            messageBusParams.getSourceSessionParams().setTimeout(timeout);
+        }
+        return messageBusParams;
+    }
+
+    private Message createDocumentRequest(String docid, LoadType loadType) {
+        GetDocumentMessage msg = new GetDocumentMessage(new DocumentId(docid), params.fieldSet);
+        msg.setPriority(params.priority);
+        msg.setRetryEnabled(!params.noRetry);
+
+        if (loadType != null) {
+            msg.setLoadType(loadType);
+        }
+        return msg;
+    }
+
+    private void printReply(Reply reply) {
+        Trace trace = reply.getTrace();
+        if (!trace.getRoot().isEmpty()) {
+            System.out.println(trace);
+        }
+
+        if (reply.hasErrors()) {
+            System.err.print("Request failed: ");
+            for (int i = 0; i < reply.getNumErrors(); i++) {
+                System.err.printf("\n  %s", reply.getError(i));
+            }
+            System.err.println();
+            return;
+        }
+
+        if (!(reply instanceof GetDocumentReply)) {
+            System.err.printf("Unexpected reply %s: '%s'\n", reply.getType(), reply.toString());
+            return;
+        }
+
+        GetDocumentReply documentReply = (GetDocumentReply) reply;
+        Document document = documentReply.getDocument();
+
+        if (document == null) {
+            System.out.println("Document not found.");
+            return;
+        }
+
+        if (params.showDocSize) {
+            System.out.printf("Document size: %d bytes.\n", document.getSerializedSize());
+        }
+        if (params.printIdsOnly) {
+            System.out.println(document.getId());
+        } else {
+            if (params.jsonOutput) {
+                System.out.print(Utf8.toString(JsonWriter.toByteArray(document)));
+            } else {
+                System.out.print(document.toXML("  "));
+            }
+        }
+    }
+
+    private String createClusterNamesString() {
+        StringBuilder names = new StringBuilder();
+        for (ClusterDef c : clusterList.getStorageClusters()) {
+            if (names.length() > 0) {
+                names.append(", ");
+            }
+            names.append(c.getName());
+        }
+        return names.toString();
+    }
+}
diff --git a/vespaclient-java/src/main/java/com/yahoo/vespaget/DocumentRetrieverException.java b/vespaclient-java/src/main/java/com/yahoo/vespaget/DocumentRetrieverException.java
new file mode 100644
index 00000000000..4cf0e9885a3
--- /dev/null
+++ b/vespaclient-java/src/main/java/com/yahoo/vespaget/DocumentRetrieverException.java
@@ -0,0 +1,14 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespaget;
+
+/**
+ * Exception thrown by {@link DocumentRetriever}.
+ *
+ * @author bjorncs
+ */
+public class DocumentRetrieverException extends Exception {
+
+    public DocumentRetrieverException(String message) {
+        super(message);
+    }
+}
diff --git a/vespaclient-java/src/main/java/com/yahoo/vespaget/Main.java b/vespaclient-java/src/main/java/com/yahoo/vespaget/Main.java
new file mode 100644
index 00000000000..324107d8909
--- /dev/null
+++ b/vespaclient-java/src/main/java/com/yahoo/vespaget/Main.java
@@ -0,0 +1,46 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespaget;
+
+
+import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet;
+import com.yahoo.vespaclient.ClusterList;
+
+/**
+ * The vespaget tool retrieves documents from a Vespa Document Storage cluster, and prints them to stdout as XML.
+ *
+ * @author bjorncs
+ */
+public class Main {
+
+    public static void main(String[] args) {
+        try {
+            CommandLineOptions options = new CommandLineOptions();
+            ClientParameters params = options.parseCommandLineArguments(args);
+
+            if (params.help) {
+                options.printHelp();
+            } else {
+                DocumentRetriever documentRetriever = createDocumentRetriever(params);
+                addShutdownHook(documentRetriever);
+                documentRetriever.retrieveDocuments();
+            }
+        } catch (IllegalArgumentException e) {
+            System.err.printf("Failed to parse command line arguments: %s.\n", e.getMessage());
+        } catch (DocumentRetrieverException e) {
+            System.err.printf("Failed to retrieve documents: %s\n", e.getMessage());
+        }
+    }
+
+    private static void addShutdownHook(DocumentRetriever documentRetriever) {
+        Runtime.getRuntime().addShutdownHook(new Thread(documentRetriever::shutdown));
+    }
+
+    private static DocumentRetriever createDocumentRetriever(ClientParameters params) {
+        return new DocumentRetriever(
+                new ClusterList("client"),
+                new DocumentAccessFactory(),
+                new LoadTypeSet(params.configId),
+                params
+        );
+    }
+}
diff --git a/vespaclient-java/src/main/java/com/yahoo/vespastat/BucketStatsException.java b/vespaclient-java/src/main/java/com/yahoo/vespastat/BucketStatsException.java
new file mode 100644
index 00000000000..a6f471e7b5e
--- /dev/null
+++ b/vespaclient-java/src/main/java/com/yahoo/vespastat/BucketStatsException.java
@@ -0,0 +1,18 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespastat;
+
+/**
+ * Exception class used by {@link com.yahoo.vespastat.BucketStatsRetriever}.
+ *
+ * @author bjorncs
+ */
+public class BucketStatsException extends Exception {
+    public BucketStatsException(String message) {
+        super(message);
+    }
+
+    public BucketStatsException(String message, Throwable cause) {
+        super(message, cause);
+    }
+
+}
diff --git a/vespaclient-java/src/main/java/com/yahoo/vespastat/BucketStatsPrinter.java b/vespaclient-java/src/main/java/com/yahoo/vespastat/BucketStatsPrinter.java
new file mode 100644
index 00000000000..a4a263188b5
--- /dev/null
+++ b/vespaclient-java/src/main/java/com/yahoo/vespastat/BucketStatsPrinter.java
@@ -0,0 +1,59 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespastat;
+
+import com.yahoo.document.BucketId;
+import com.yahoo.documentapi.messagebus.protocol.GetBucketListReply;
+
+import java.io.PrintStream;
+import java.util.List;
+
+/**
+ * The class is responsible for printing bucket information to a printstream.
+ *
+ * @author bjorncs
+ */
+public class BucketStatsPrinter {
+    private final BucketStatsRetriever retriever;
+    private final PrintStream out;
+
+    public BucketStatsPrinter(
+            BucketStatsRetriever retriever,
+            PrintStream out) {
+        this.retriever = retriever;
+        this.out = out;
+    }
+
+    public void retrieveAndPrintBucketStats(ClientParameters.SelectionType type, String id, boolean dumpData) throws BucketStatsException {
+        BucketId bucketId = retriever.getBucketIdForType(type, id);
+        if (type == ClientParameters.SelectionType.GROUP || type == ClientParameters.SelectionType.USER) {
+            out.printf("Generated 32-bit bucket id: %s\n", bucketId);
+        }
+
+        List<GetBucketListReply.BucketInfo> bucketList = retriever.retrieveBucketList(bucketId);
+        printBucketList(bucketList);
+
+        if (dumpData) {
+            for (GetBucketListReply.BucketInfo bucketInfo : bucketList) {
+                BucketId bucket = bucketInfo.getBucketId();
+                String bucketStats = retriever.retrieveBucketStats(type, id, bucket);
+                printBucketStats(bucket, bucketStats);
+            }
+        }
+    }
+
+    private void printBucketList(List<GetBucketListReply.BucketInfo> bucketList) {
+        if (bucketList.isEmpty()) {
+            out.println("No actual files were stored for this bucket.");
+        } else {
+            out.println("Bucket maps to the following actual files:");
+            for (GetBucketListReply.BucketInfo bucketInfo : bucketList) {
+                out.printf("\t%s\n", bucketInfo);
+            }
+        }
+    }
+
+    private void printBucketStats(BucketId bucket, String stats) {
+        out.printf("\nDetails for %s:\n%s", bucket, stats);
+    }
+
+}
diff --git a/vespaclient-java/src/main/java/com/yahoo/vespastat/BucketStatsRetriever.java b/vespaclient-java/src/main/java/com/yahoo/vespastat/BucketStatsRetriever.java
new file mode 100644
index 00000000000..84e89349f9f
--- /dev/null
+++ b/vespaclient-java/src/main/java/com/yahoo/vespastat/BucketStatsRetriever.java
@@ -0,0 +1,176 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespastat;
+
+import com.yahoo.document.BucketId;
+import com.yahoo.document.BucketIdFactory;
+import com.yahoo.document.DocumentId;
+import com.yahoo.document.GlobalId;
+import com.yahoo.document.select.BucketSelector;
+import com.yahoo.document.select.BucketSet;
+import com.yahoo.document.select.parser.ParseException;
+import com.yahoo.documentapi.SyncParameters;
+import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess;
+import com.yahoo.documentapi.messagebus.MessageBusSyncSession;
+import com.yahoo.documentapi.messagebus.protocol.DocumentMessage;
+import com.yahoo.documentapi.messagebus.protocol.GetBucketListMessage;
+import com.yahoo.documentapi.messagebus.protocol.GetBucketListReply;
+import com.yahoo.documentapi.messagebus.protocol.StatBucketMessage;
+import com.yahoo.documentapi.messagebus.protocol.StatBucketReply;
+import com.yahoo.messagebus.Reply;
+import com.yahoo.messagebus.routing.Route;
+
+import java.util.List;
+
+/**
+ * This class fetches bucket information from Vespa
+ *
+ * @author bjorncs
+ */
+public class BucketStatsRetriever {
+
+    private final BucketIdFactory bucketIdFactory = new BucketIdFactory();
+    private final BucketSelector selector = new BucketSelector(bucketIdFactory);
+
+    private final MessageBusSyncSession session;
+    private final MessageBusDocumentAccess documentAccess;
+    private final String route;
+
+    public BucketStatsRetriever(
+            DocumentAccessFactory documentAccessFactory,
+            String route,
+            ShutdownHookRegistrar registrar) {
+        registerShutdownHook(registrar);
+        this.documentAccess = documentAccessFactory.createDocumentAccess();
+        this.session = documentAccess.createSyncSession(new SyncParameters());
+        this.route = route;
+    }
+
+    private void registerShutdownHook(ShutdownHookRegistrar registrar) {
+        registrar.registerShutdownHook(() -> {
+            try {
+                session.destroy();
+            } catch (Exception e) {
+                // Ignore exception on shutdown
+            }
+            try {
+                documentAccess.shutdown();
+            } catch (Exception e) {
+                // Ignore exception on shutdown
+            }
+        });
+    }
+
+    public BucketId getBucketIdForType(ClientParameters.SelectionType type, String id) throws BucketStatsException {
+        switch (type) {
+            case DOCUMENT:
+                return bucketIdFactory.getBucketId(new DocumentId(id));
+            case BUCKET:
+                // The internal parser of BucketID is used since the Java Long.decode cannot handle unsigned longs.
+                return new BucketId(String.format("BucketId(%s)", id));
+            case GID:
+                return convertGidToBucketId(id);
+            case USER:
+            case GROUP:
+                try {
+                    BucketSet bucketList = selector.getBucketList(createDocumentSelection(type, id));
+                    if (bucketList.size() != 1) {
+                        String message = String.format("Document selection must map to only one location. " +
+                                "Specified selection matches %d locations.", bucketList.size());
+                        throw new BucketStatsException(message);
+                    }
+                    return bucketList.iterator().next();
+                } catch (ParseException e) {
+                    throw new BucketStatsException(String.format("Invalid id: %s (%s).", id, e.getMessage()), e);
+                }
+            default:
+                throw new RuntimeException("Unreachable code");
+        }
+    }
+
+    public String retrieveBucketStats(ClientParameters.SelectionType type, String id, BucketId bucketId) throws BucketStatsException {
+        String documentSelection = createDocumentSelection(type, id);
+        StatBucketMessage msg = new StatBucketMessage(bucketId, documentSelection);
+        StatBucketReply statBucketReply = sendMessage(msg, StatBucketReply.class);
+        return statBucketReply.getResults();
+    }
+
+    public List<GetBucketListReply.BucketInfo> retrieveBucketList(BucketId bucketId) throws BucketStatsException {
+        GetBucketListMessage msg = new GetBucketListMessage(bucketId);
+        GetBucketListReply bucketListReply = sendMessage(msg, GetBucketListReply.class);
+        return bucketListReply.getBuckets();
+    }
+
+
+    private <T extends Reply> T sendMessage(DocumentMessage msg, Class<T> expectedReply) throws BucketStatsException {
+        setRoute(msg, route);
+        Reply reply = session.syncSend(msg);
+        return validateReply(reply, expectedReply);
+    }
+
+    private static void setRoute(DocumentMessage msg, String route) throws BucketStatsException {
+        try {
+            msg.setRoute(Route.parse(route));
+        } catch (Exception e) {
+            throw new BucketStatsException(String.format("Invalid route: '%s'.", route));
+        }
+    }
+
+    private static <T extends Reply> T validateReply(Reply reply, Class<T> type) throws BucketStatsException {
+        if (reply.hasErrors()) {
+            throw new BucketStatsException(makeErrorMessage(reply));
+        }
+        if (!type.isInstance(reply)) {
+            throw new BucketStatsException(String.format("Unexpected reply %s: '%s'", reply.getType(), reply.toString()));
+        }
+        return type.cast(reply);
+    }
+
+    private static String makeErrorMessage(Reply reply) {
+        StringBuilder b = new StringBuilder();
+        b.append("Request failed: \n");
+        for (int i = 0; i < reply.getNumErrors(); i++) {
+            b.append(String.format("\t %s\n", reply.getError(i)));
+        }
+        return b.toString();
+    }
+
+    private static String createDocumentSelection(ClientParameters.SelectionType type, String id) {
+        switch (type) {
+            case BUCKET:
+                return "true";
+            case DOCUMENT:
+                return String.format("id=\"%s\"", id);
+            case GID:
+                return String.format("id.gid=\"gid(%s)\"", id);
+            case USER:
+                return String.format("id.user=%s", id);
+            case GROUP:
+                return String.format("id.group=\"%s\"", id);
+            default:
+                throw new RuntimeException("Unreachable code");
+        }
+    }
+
+    private static BucketId convertGidToBucketId(String id) throws BucketStatsException {
+        if (!id.matches("0x\\p{XDigit}{24}")) {
+            throw new BucketStatsException("Invalid gid: " + id);
+        }
+        String hexWithoutPrefix = id.substring(2);
+        return new GlobalId(convertHexStringToByteArray(hexWithoutPrefix)).toBucketId();
+    }
+
+    private static byte[] convertHexStringToByteArray(String s) throws BucketStatsException {
+        int len = s.length();
+        byte[] data = new byte[len / 2];
+        for (int i = 0; i < len; i += 2) {
+            int digit1 = Character.digit(s.charAt(i), 16);
+            int digit2 = Character.digit(s.charAt(i + 1), 16);
+            data[i / 2] = (byte) ((digit1 << 4) + digit2);
+        }
+        return data;
+    }
+
+    public interface ShutdownHookRegistrar {
+        void registerShutdownHook(Runnable runnable);
+    }
+}
diff --git a/vespaclient-java/src/main/java/com/yahoo/vespastat/ClientParameters.java b/vespaclient-java/src/main/java/com/yahoo/vespastat/ClientParameters.java
new file mode 100644
index 00000000000..bba0d9803ed
--- /dev/null
+++ b/vespaclient-java/src/main/java/com/yahoo/vespastat/ClientParameters.java
@@ -0,0 +1,73 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespastat;
+
+/**
+ * This class contains the program parameters.
+ *
+ * @author bjorncs
+ */
+public class ClientParameters {
+    // Show help page if true
+    public final boolean help;
+    // Dump list of documents for all buckets matching the selection if true
+    public final boolean dumpData;
+    // The message bus route
+    public final String route;
+    // The selection type
+    public final SelectionType selectionType;
+    // The selection id
+    public final String id;
+
+    public ClientParameters(
+            boolean help,
+            boolean dumpData,
+            String route,
+            SelectionType selectionType,
+            String id) {
+        this.help = help;
+        this.dumpData = dumpData;
+        this.route = route;
+        this.selectionType = selectionType;
+        this.id = id;
+    }
+
+    public enum SelectionType {USER, GROUP, BUCKET, GID, DOCUMENT}
+
+    public static class Builder {
+        private boolean help;
+        private boolean dumpData;
+        private String route;
+        private SelectionType selectionType;
+        private String id;
+
+        public Builder setHelp(boolean help) {
+            this.help = help;
+            return this;
+        }
+
+        public Builder setDumpData(boolean dumpData) {
+            this.dumpData = dumpData;
+            return this;
+        }
+
+        public Builder setRoute(String route) {
+            this.route = route;
+            return this;
+        }
+
+        public Builder setSelectionType(SelectionType selectionType) {
+            this.selectionType = selectionType;
+            return this;
+        }
+
+        public Builder setId(String id) {
+            this.id = id;
+            return this;
+        }
+
+        public ClientParameters build() {
+            return new ClientParameters(help, dumpData, route, selectionType, id);
+        }
+    }
+
+}
diff --git a/vespaclient-java/src/main/java/com/yahoo/vespastat/CommandLineOptions.java b/vespaclient-java/src/main/java/com/yahoo/vespastat/CommandLineOptions.java
new file mode 100644
index 00000000000..b0b6246a262
--- /dev/null
+++ b/vespaclient-java/src/main/java/com/yahoo/vespastat/CommandLineOptions.java
@@ -0,0 +1,139 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespastat;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.DefaultParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionGroup;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+
+/**
+ * Responsible for parsing the command line arguments and presenting the help page
+ *
+ * @author bjorncs
+ */
+public class CommandLineOptions {
+
+    private static final String HELP_OPTION = "help";
+    private static final String DUMP_OPTION = "dump";
+    private static final String ROUTE_OPTION = "route";
+    private static final String USER_OPTION = "user";
+    private static final String GROUP_OPTION = "group";
+    private static final String BUCKET_OPTION = "bucket";
+    private static final String GID_OPTION = "gid";
+    private static final String DOCUMENT_OPTION = "document";
+
+    private final Options options = createOptions();
+
+    @SuppressWarnings("AccessStaticViaInstance")
+    private static Options createOptions() {
+        Options options = new Options();
+
+        options.addOption(Option.builder("h")
+                .hasArg(false)
+                .desc("Show this syntax page.")
+                .longOpt(HELP_OPTION)
+                .build());
+
+        options.addOption(Option.builder("d")
+                .hasArg(false)
+                .desc("Dump list of documents for all buckets matching the selection command.")
+                .longOpt(DUMP_OPTION)
+                .build());
+
+        options.addOption(Option.builder("r")
+                .hasArg(true)
+                .desc("Route to send the messages to, usually the name of the storage cluster.")
+                .argName("route")
+                .longOpt(ROUTE_OPTION)
+                .build());
+
+        // A group of mutually exclusive options for user, group, bucket, gid and document.
+        OptionGroup optionGroup = new OptionGroup();
+        optionGroup.setRequired(false);
+
+        optionGroup.addOption(Option.builder("u")
+                .hasArg(true)
+                .desc("Dump list of buckets that can contain the given user.")
+                .argName("userid")
+                .longOpt(USER_OPTION)
+                .build());
+
+        optionGroup.addOption(Option.builder("g")
+                .hasArg(true)
+                .desc("Dump list of buckets that can contain the given group.")
+                .argName("groupid")
+                .longOpt(GROUP_OPTION)
+                .build());
+
+        optionGroup.addOption(Option.builder("b")
+                .hasArg(true)
+                .desc("Dump list of buckets that are contained in the given bucket, or that contain it.")
+                .argName("bucketid")
+                .longOpt(BUCKET_OPTION)
+                .build());
+
+        optionGroup.addOption(Option.builder("l")
+                .hasArg(true)
+                .desc("Dump information about one specific document, as given by the GID (implies --dump).")
+                .argName("globalid")
+                .longOpt(GID_OPTION)
+                .build());
+
+        optionGroup.addOption(Option.builder("o")
+                .hasArg(true)
+                .desc("Dump information about one specific document (implies --dump).")
+                .argName("docid")
+                .longOpt(DOCUMENT_OPTION)
+                .build());
+
+        options.addOptionGroup(optionGroup);
+        return options;
+    }
+
+    public void printHelp() {
+        HelpFormatter formatter = new HelpFormatter();
+        formatter.printHelp("vdsstat [options]",
+                "Fetch statistics about a specific user, group, bucket, gid or document.", options, "", false);
+    }
+
+    public ClientParameters parseCommandLineArguments(String[] args) {
+        try {
+            CommandLineParser clp = new DefaultParser();
+            CommandLine cl = clp.parse(options, args);
+            ClientParameters.Builder builder = new ClientParameters.Builder();
+
+            builder.setHelp(cl.hasOption(HELP_OPTION));
+            builder.setDumpData(cl.hasOption(DUMP_OPTION));
+            builder.setRoute(cl.getOptionValue(ROUTE_OPTION, "default"));
+
+            if (cl.hasOption(USER_OPTION)) {
+                builder.setSelectionType(ClientParameters.SelectionType.USER);
+                builder.setId(cl.getOptionValue(USER_OPTION));
+            } else if (cl.hasOption(GROUP_OPTION)) {
+                builder.setSelectionType(ClientParameters.SelectionType.GROUP);
+                builder.setId(cl.getOptionValue(GROUP_OPTION));
+            } else if (cl.hasOption(BUCKET_OPTION)) {
+                builder.setSelectionType(ClientParameters.SelectionType.BUCKET);
+                builder.setId(cl.getOptionValue(BUCKET_OPTION));
+            } else if (cl.hasOption(GID_OPTION)) {
+                builder.setSelectionType(ClientParameters.SelectionType.GID);
+                builder.setId(cl.getOptionValue(GID_OPTION));
+                builder.setDumpData(true);
+            } else if (cl.hasOption(DOCUMENT_OPTION)) {
+                builder.setSelectionType(ClientParameters.SelectionType.DOCUMENT);
+                builder.setId(cl.getOptionValue(DOCUMENT_OPTION));
+                builder.setDumpData(true);
+            } else if (!cl.hasOption(HELP_OPTION)) {
+                throw new IllegalArgumentException("Must specify one of 'user', 'group', 'bucket', 'document' or 'gid'.");
+            }
+
+            return builder.build();
+        } catch (ParseException e) {
+            throw new IllegalArgumentException(e.getMessage(), e);
+        }
+    }
+}
diff --git a/vespaclient-java/src/main/java/com/yahoo/vespastat/DocumentAccessFactory.java b/vespaclient-java/src/main/java/com/yahoo/vespastat/DocumentAccessFactory.java
new file mode 100644
index 00000000000..55a31f30a2b
--- /dev/null
+++ b/vespaclient-java/src/main/java/com/yahoo/vespastat/DocumentAccessFactory.java
@@ -0,0 +1,15 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespastat;
+
+import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess;
+
+/**
+ * Factory class for {@link com.yahoo.documentapi.messagebus.MessageBusDocumentAccess}.
+ *
+ * @author bjorncs
+ */
+public class DocumentAccessFactory {
+    public MessageBusDocumentAccess createDocumentAccess() {
+        return new MessageBusDocumentAccess();
+    }
+}
diff --git a/vespaclient-java/src/main/java/com/yahoo/vespastat/Main.java b/vespaclient-java/src/main/java/com/yahoo/vespastat/Main.java
new file mode 100644
index 00000000000..9d87a6f68f4
--- /dev/null
+++ b/vespaclient-java/src/main/java/com/yahoo/vespastat/Main.java
@@ -0,0 +1,38 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespastat;
+
+/**
+ * Main application class
+ *
+ * @author bjorncs
+ */
+public class Main {
+
+    private Main() {
+    }
+
+    public static void main(String[] args) {
+        CommandLineOptions options = new CommandLineOptions();
+        try {
+            ClientParameters params = options.parseCommandLineArguments(args);
+            if (params.help) {
+                options.printHelp();
+                return;
+            }
+            BucketStatsRetriever retriever = new BucketStatsRetriever(
+                    new DocumentAccessFactory(),
+                    params.route,
+                    createShutdownHookRegistrar());
+            BucketStatsPrinter printer = new BucketStatsPrinter(retriever, System.out);
+            printer.retrieveAndPrintBucketStats(params.selectionType, params.id, params.dumpData);
+        } catch (IllegalArgumentException e) {
+            System.err.printf("Failed to parse command line arguments: %s.\n", e.getMessage());
+        } catch (BucketStatsException e) {
+            System.err.println(e.getMessage());
+        }
+    }
+
+    private static BucketStatsRetriever.ShutdownHookRegistrar createShutdownHookRegistrar() {
+        return runnable -> Runtime.getRuntime().addShutdownHook(new Thread(runnable));
+    }
+}
diff --git a/vespaclient-java/src/main/java/com/yahoo/vespasummarybenchmark/VespaSummaryBenchmark.java b/vespaclient-java/src/main/java/com/yahoo/vespasummarybenchmark/VespaSummaryBenchmark.java
new file mode 100644
index 00000000000..803445d16f5
--- /dev/null
+++ b/vespaclient-java/src/main/java/com/yahoo/vespasummarybenchmark/VespaSummaryBenchmark.java
@@ -0,0 +1,162 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespasummarybenchmark;
+
+import com.yahoo.compress.CompressionType;
+import com.yahoo.document.GlobalId;
+import com.yahoo.document.idstring.IdString;
+import com.yahoo.document.serialization.DeserializationException;
+import com.yahoo.jrt.*;
+import com.yahoo.log.LogSetup;
+import com.yahoo.slime.*;
+import net.jpountz.lz4.LZ4Factory;
+import net.jpountz.lz4.LZ4FastDecompressor;
+
+import java.io.*;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ *
+ * This is used for testing and benchmarking rpc docsum interface.
+ * time vespa-summary-benchmark file-containing-docids connectionspec summary-class repetitions threads
+ * fx ' time vespa-summary-benchmark feed.xml tcp/localhost:19115 keyvaluesummary 10000 32'
+ *
+ * @author baldersheim
+ */
+public class VespaSummaryBenchmark {
+
+    private final Supervisor supervisor = new Supervisor(new Transport());
+
+    private VespaSummaryBenchmark() { }
+
+    private static List<String> getDocIds(String fileName) {
+        try {
+            FileInputStream fstream = new FileInputStream(fileName);
+            DataInputStream in = new DataInputStream(fstream);
+            BufferedReader br = new BufferedReader(new InputStreamReader(in));
+            String strLine;
+
+            List<String> docIds = new ArrayList<>();
+            while ((strLine = br.readLine()) != null) {
+                docIds.add(strLine);
+            }
+            in.close();
+            return docIds;
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    private List<Target> getTargets(String connectionSpec, int numTargets) {
+        List<Target> targets = new ArrayList<>(numTargets);
+        for ( int i=0; i < numTargets; i++) {
+            targets.add(supervisor.connect(new Spec(connectionSpec)));
+        }
+        return targets;
+    }
+
+    private static Slime createDocsumRequest(String summaryClass, List<GlobalId> gids) {
+        Slime docsumRequest = new Slime();
+        Cursor root = docsumRequest.setObject();
+        root.setString("class", summaryClass);
+        Cursor gidCursor = root.setArray("gids");
+        for (GlobalId gid : gids) {
+            gidCursor.addData(gid.getRawId());
+        }
+        return docsumRequest;
+    }
+
+    private static class Waiter implements RequestWaiter {
+
+        int waitingFor;
+        boolean dump;
+
+        Waiter(int expect, boolean dump) {
+            waitingFor = expect;
+            this.dump = dump;
+        }
+
+        private void print(Request request) {
+            Values ret = request.returnValues();
+            CompressionType type = CompressionType.valueOf(ret.get(0).asInt8());
+            int uncompressedSize = ret.get(1).asInt32();
+            byte [] blob = ret.get(2).asData();
+            if (type == CompressionType.LZ4) {
+                LZ4Factory factory = LZ4Factory.fastestInstance();
+                LZ4FastDecompressor decompressor = factory.fastDecompressor();
+                byte [] uncompressed = new byte [uncompressedSize];
+                int compressedLength = decompressor.decompress(blob, 0, uncompressed, 0, uncompressedSize);
+                if (compressedLength != blob.length) {
+                    throw new DeserializationException("LZ4 decompression failed. compressed size does not match. Expected " + blob.length + ". Got " + compressedLength);
+                }
+                blob = uncompressed;
+            }
+            Slime slime = BinaryFormat.decode(blob);
+            try {
+                new JsonFormat(true).encode(System.out, slime);
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+        }
+
+        @Override
+        public void handleRequestDone(Request request) {
+            synchronized (this) {
+                if (dump) {
+                    print(request);
+                    dump = false;
+                }
+                waitingFor--;
+                if (waitingFor == 0) {
+                    this.notifyAll();
+                }
+            }
+        }
+        void waitForReplies() throws InterruptedException {
+            synchronized (this) {
+                while (waitingFor > 0) {
+                    this.wait();
+                }
+            }
+        }
+    }
+
+    private static void fetchDocIds(String summaryClass, List<Target> targets, List<GlobalId> gids, boolean dump) {
+        Slime docsumRequest = createDocsumRequest(summaryClass, gids);
+        byte [] blob = BinaryFormat.encode(docsumRequest);
+        Waiter waiter = new Waiter(targets.size(), dump);
+        for (Target target : targets) {
+            Request r = new Request("proton.getDocsums");
+            r.parameters().add(new Int8Value(CompressionType.NONE.getCode()));
+            r.parameters().add(new Int32Value(blob.length));
+            r.parameters().add(new DataValue(blob));
+            target.invokeAsync(r, 100.0, waiter);
+        }
+        try {
+            waiter.waitForReplies();
+        } catch (InterruptedException e) {
+            throw new RuntimeException(e);
+        }
+
+    }
+
+    public static void main(String[] args) {
+        LogSetup.initVespaLogging("vespasummarybenchmark");
+        String docidFileName = args[0];
+        String connectionSpec = args[1];
+        String summaryClass = args[2];
+        int numRuns = Integer.parseInt(args[3]);
+        int numTargets = Integer.parseInt(args[4]);
+        VespaSummaryBenchmark benchmark = new VespaSummaryBenchmark();
+        List<String> docIds = getDocIds(docidFileName);
+        List<GlobalId> gids = new ArrayList<>(docIds.size());
+        for (String docid : docIds) {
+            GlobalId gid = new GlobalId(IdString.createIdString(docid));
+            gids.add(gid);
+        }
+        List<Target> targets = benchmark.getTargets(connectionSpec, numTargets);
+        for (int i = 0; i < numRuns; i++) {
+            fetchDocIds(summaryClass, targets, gids, i==0);
+        }
+    }
+}
diff --git a/vespaclient-java/src/main/java/com/yahoo/vespavisit/StdOutVisitorHandler.java b/vespaclient-java/src/main/java/com/yahoo/vespavisit/StdOutVisitorHandler.java
new file mode 100644
index 00000000000..c849bc1741c
--- /dev/null
+++ b/vespaclient-java/src/main/java/com/yahoo/vespavisit/StdOutVisitorHandler.java
@@ -0,0 +1,292 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespavisit;
+
+import com.yahoo.document.BucketId;
+import com.yahoo.document.Document;
+import com.yahoo.document.DocumentId;
+import com.yahoo.document.json.JsonWriter;
+import com.yahoo.document.serialization.XmlStream;
+import com.yahoo.documentapi.AckToken;
+import com.yahoo.documentapi.DumpVisitorDataHandler;
+import com.yahoo.documentapi.ProgressToken;
+import com.yahoo.documentapi.VisitorControlHandler;
+import com.yahoo.documentapi.VisitorDataHandler;
+import com.yahoo.documentapi.messagebus.protocol.DocumentListEntry;
+import com.yahoo.documentapi.messagebus.protocol.DocumentListMessage;
+import com.yahoo.documentapi.messagebus.protocol.EmptyBucketsMessage;
+import com.yahoo.documentapi.messagebus.protocol.MapVisitorMessage;
+import com.yahoo.log.LogLevel;
+import com.yahoo.messagebus.Message;
+
+import java.io.IOException;
+import java.io.PrintStream;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.logging.Logger;
+
+/**
+ * A visitor data and progress handler that writes to STDOUT.
+ *
+ * Due to java not being able to inherit two classes, and neither being an
+ * interface this had to be implemented by creating a wrapper class.
+ *
+ * @author <a href="mailto:thomasg@yahoo-inc.com">Thomas Gundersen</a>
+ */
+public class StdOutVisitorHandler extends VdsVisitHandler {
+    private static final Logger log = Logger.getLogger(
+                                        StdOutVisitorHandler.class.getName());
+    private boolean printIds;
+    private boolean indentXml;
+    private int processTimeMilliSecs;
+    private PrintStream out;
+    private final boolean jsonOutput;
+
+    private VisitorDataHandler dataHandler;
+
+    public StdOutVisitorHandler(boolean printIds, boolean indentXml,
+                                boolean showProgress, boolean showStatistics, boolean doStatistics,
+                                boolean abortOnClusterDown, int processtime, boolean jsonOutput)
+    {
+        super(showProgress, showStatistics, abortOnClusterDown);
+
+        this.printIds = printIds;
+        this.indentXml = indentXml;
+        this.processTimeMilliSecs = processtime;
+        this.jsonOutput = jsonOutput;
+        String charset = "UTF-8";
+        try {
+            out = new PrintStream(System.out, true, charset);
+        } catch (java.io.UnsupportedEncodingException e) {
+            System.out.println(charset + " is an unsupported encoding, " +
+                               "using default instead.");
+            out = System.out;
+        }
+
+        dataHandler = new DataHandler(doStatistics);
+    }
+
+    @Override
+    public void onDone() {
+    }
+
+    public VisitorDataHandler getDataHandler() { return dataHandler; }
+
+    class StatisticsMap extends LinkedHashMap<String, Integer> {
+        int maxSize;
+
+        StatisticsMap(int maxSize) {
+            super(100, (float)0.75, true);
+            this.maxSize = maxSize;
+        }
+
+        protected boolean removeEldestEntry(Map.Entry<String, Integer> eldest) {
+            if (size() > maxSize) {
+                dump(eldest);
+                return true;
+            }
+
+            return false;
+        }
+
+        private void dump(Map.Entry<String, Integer> e) {
+            out.println(e.getKey() + ":" + e.getValue());
+        }
+
+        public void dumpAll() {
+            for (Map.Entry<String, Integer> e : entrySet()) {
+                dump(e);
+            }
+            clear();
+        }
+    }
+
+    class DataHandler extends DumpVisitorDataHandler {
+        boolean doStatistics;
+        StatisticsMap statisticsMap = new StatisticsMap(10000);
+        private volatile boolean first = true;
+
+        public DataHandler(boolean doStatistics) {
+            this.doStatistics = doStatistics;
+        }
+
+        @Override
+        public void onMessage(Message m, AckToken token) {
+            if (processTimeMilliSecs > 0) {
+                try {
+                    Thread.sleep(processTimeMilliSecs);
+                } catch (InterruptedException e) {}
+            }
+
+            synchronized (printLock) {
+                if (m instanceof MapVisitorMessage) {
+                    onMapVisitorData(((MapVisitorMessage)m).getData());
+                    ack(token);
+                } else if (m instanceof DocumentListMessage) {
+                    DocumentListMessage dlm = (DocumentListMessage)m;
+                    onDocumentList(dlm.getBucketId(), dlm.getDocuments());
+                    ack(token);
+                } else if (m instanceof EmptyBucketsMessage) {
+                    onEmptyBuckets(((EmptyBucketsMessage)m).getBucketIds());
+                    ack(token);
+                } else {
+                    super.onMessage(m, token);
+                }
+            }
+        }
+
+        @Override
+        public void onDocument(Document doc, long timestamp) {
+            try {
+                if (lastLineIsProgress) {
+                    System.err.print('\r');
+                }
+
+                if (printIds) {
+                    out.print(doc.getId());
+                    out.print(" (Last modified at ");
+                    out.println(timestamp + ")");
+                } else {
+                    if (jsonOutput) {
+                        writeJsonDocument(doc);
+                    } else {
+                        out.print(doc.toXML(
+                                indentXml ? "  " : ""));
+                    }
+                }
+            } catch (Exception e) {
+                System.err.println("Failed to output document: "
+                        + e.getMessage());
+                getControlHandler().abort();
+            }
+        }
+
+        private void writeJsonDocument(Document doc) throws IOException {
+            writeFeedStartOrRecordSeparator();
+            out.write(JsonWriter.toByteArray(doc));
+        }
+
+        @Override
+        public void onRemove(DocumentId docId) {
+            try {
+                if (lastLineIsProgress) {
+                    System.err.print('\r');
+                }
+
+                if (printIds) {
+                    out.println(docId + " (Removed)");
+                } else {
+                    if (jsonOutput) {
+                        writeJsonDocumentRemove(docId);
+                    } else {
+                        XmlStream stream = new XmlStream();
+                        stream.beginTag("remove");
+                        stream.addAttribute("documentid", docId);
+                        stream.endTag();
+                        assert(stream.isFinalized());
+                        out.print(stream);
+                    }
+                }
+            } catch (Exception e) {
+                System.err.println("Failed to output document: "
+                        + e.getMessage());
+                getControlHandler().abort();
+            }
+        }
+
+        private void writeJsonDocumentRemove(DocumentId docId)
+                throws IOException {
+            writeFeedStartOrRecordSeparator();
+            out.write(JsonWriter.documentRemove(docId));
+        }
+
+        private void writeFeedStartOrRecordSeparator() {
+            if (first) {
+                out.println("[");
+                first = false;
+            } else {
+                out.println(",");
+            }
+        }
+
+        private void writeFeedEnd() {
+            out.println("]");
+        }
+
+        public void onMapVisitorData(Map<String, String> data) {
+            for (String key : data.keySet()) {
+                if (doStatistics) {
+                    Integer i = statisticsMap.get(key);
+                    if (i != null) {
+                        statisticsMap.put(key, Integer.parseInt(data.get(key)) + i);
+                    } else {
+                        statisticsMap.put(key, Integer.parseInt(data.get(key)));
+                    }
+                } else {
+                    out.println(key + ":" + data.get(key));
+                }
+            }
+        }
+
+        public void onDocumentList(BucketId bucketId, List<DocumentListEntry> documents) {
+            out.println("Got document list of bucket " + bucketId.toString());
+            for (DocumentListEntry entry : documents) {
+                entry.getDocument().setLastModified(entry.getTimestamp());
+                onDocument(entry.getDocument(), entry.getTimestamp());
+            }
+        }
+
+        public void onEmptyBuckets(List<BucketId> bucketIds) {
+            StringBuilder buckets = new StringBuilder();
+            for(BucketId bid : bucketIds) {
+                buckets.append(" ");
+                buckets.append(bid.toString());
+            }
+            log.log(LogLevel.INFO, "Got EmptyBuckets: " + buckets);
+        }
+
+        public synchronized void onDone() {
+            if (jsonOutput) {
+                writeFeedEnd();
+            }
+            statisticsMap.dumpAll();
+            super.onDone();
+        }
+    }
+
+    class ControlHandler extends VisitorControlHandler {
+        public void onProgress(ProgressToken token) {
+            if (showProgress) {
+                synchronized (printLock) {
+                    if (lastLineIsProgress) {
+                        System.err.print('\r');
+                    }
+                    System.err.format("%.1f %% finished.",
+                                      token.percentFinished());
+                    lastLineIsProgress = true;
+                }
+            }
+            super.onProgress(token);
+        }
+
+        public void onDone(CompletionCode code, String message) {
+            if (lastLineIsProgress) {
+                System.err.print('\n');
+                lastLineIsProgress = false;
+            }
+            if (code != CompletionCode.SUCCESS) {
+                if (code == CompletionCode.ABORTED) {
+                    System.err.println("Visitor aborted: " + message);
+                } else if (code == CompletionCode.TIMEOUT) {
+                    System.err.println("Visitor timed out: " + message);
+                } else {
+                    System.err.println("Visitor aborted due to unknown issue "
+                                     + code + ": " + message);
+                }
+            } else if (showProgress) {
+                System.err.println("Completed visiting.");
+            }
+            super.onDone(code, message);
+        }
+    }
+}
diff --git a/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisit.java b/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisit.java
new file mode 100644
index 00000000000..ff072b845de
--- /dev/null
+++ b/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisit.java
@@ -0,0 +1,789 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespavisit;
+
+import com.yahoo.document.select.parser.ParseException;
+import com.yahoo.documentapi.ProgressToken;
+import com.yahoo.documentapi.VisitorControlHandler;
+import com.yahoo.documentapi.VisitorParameters;
+import com.yahoo.documentapi.VisitorSession;
+import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess;
+import com.yahoo.documentapi.messagebus.MessageBusParams;
+import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet;
+import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol;
+import com.yahoo.log.LogSetup;
+import com.yahoo.document.select.OrderingSpecification;
+import com.yahoo.messagebus.StaticThrottlePolicy;
+import com.yahoo.vespaclient.ClusterDef;
+import com.yahoo.vespaclient.ClusterList;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.DefaultParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+
+import java.io.*;
+import java.nio.charset.Charset;
+import java.util.Map;
+
+/**
+ * Example client using visiting
+ *
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>, based on work by <a href="mailto:humbe@yahoo-inc.com">H&aring;kon Humberset</a>
+ */
+public class VdsVisit {
+    private VdsVisitParameters params;
+    private MessageBusParams mbparams = new MessageBusParams(new LoadTypeSet());
+    private VisitorSession session;
+
+    private final VisitorSessionAccessorFactory sessionAccessorFactory;
+    private VisitorSessionAccessor sessionAccessor;
+    private ShutdownHookRegistrar shutdownHookRegistrar;
+
+    public interface ShutdownHookRegistrar {
+        public void registerShutdownHook(Thread thread);
+    }
+
+    public interface VisitorSessionAccessor {
+        public VisitorSession createVisitorSession(VisitorParameters params) throws ParseException;
+        public void shutdown();
+    }
+
+    public interface VisitorSessionAccessorFactory {
+        public VisitorSessionAccessor createVisitorSessionAccessor();
+    }
+
+    private static class MessageBusVisitorSessionAccessor implements VisitorSessionAccessor {
+        private MessageBusDocumentAccess access;
+
+        private MessageBusVisitorSessionAccessor(MessageBusParams mbparams) {
+            access = new MessageBusDocumentAccess(mbparams);
+        }
+        @Override
+        public VisitorSession createVisitorSession(VisitorParameters params) throws ParseException {
+            return access.createVisitorSession(params);
+        }
+
+        @Override
+        public void shutdown() {
+            access.shutdown();
+        }
+    }
+
+    private static class MessageBusVisitorSessionAccessorFactory implements VisitorSessionAccessorFactory {
+        MessageBusParams mbparams;
+
+        private MessageBusVisitorSessionAccessorFactory(MessageBusParams mbparams) {
+            this.mbparams = mbparams;
+        }
+
+        @Override
+        public VisitorSessionAccessor createVisitorSessionAccessor() {
+            return new MessageBusVisitorSessionAccessor(mbparams);
+        }
+    }
+
+    private static class JvmRuntimeShutdownHookRegistrar implements ShutdownHookRegistrar {
+        @Override
+        public void registerShutdownHook(Thread thread) {
+            Runtime.getRuntime().addShutdownHook(thread);
+        }
+    }
+
+    public VdsVisit() {
+        this.sessionAccessorFactory = new MessageBusVisitorSessionAccessorFactory(mbparams);
+        this.shutdownHookRegistrar = new JvmRuntimeShutdownHookRegistrar();
+    }
+
+    public VdsVisit(VisitorSessionAccessorFactory sessionAccessorFactory,
+                    ShutdownHookRegistrar shutdownHookRegistrar)
+    {
+        this.sessionAccessorFactory = sessionAccessorFactory;
+        this.shutdownHookRegistrar = shutdownHookRegistrar;
+    }
+
+    public static void main(String args[]) {
+        LogSetup.initVespaLogging("vespavisit");
+        VdsVisit vdsVisit = new VdsVisit();
+
+        Options options = createOptions();
+
+        try {
+            ArgumentParser parser = new ArgumentParser(options);
+            vdsVisit.params = parser.parse(args);
+            if (vdsVisit.params == null) {
+                vdsVisit.printSyntax(options);
+                System.exit(0);
+            }
+            ClusterList clusterList = new ClusterList("client");
+            vdsVisit.params.getVisitorParameters().setRoute(
+                    resolveClusterRoute(clusterList, vdsVisit.params.getCluster()));
+        } catch (org.apache.commons.cli.ParseException e) {
+            System.err.println("Failed to parse arguments. Try --help for syntax. " + e.getMessage());
+            System.exit(1);
+        } catch (IllegalArgumentException e) {
+            System.err.println(e.getMessage());
+            System.exit(1);
+        }
+
+        if (vdsVisit.params.isVerbose()) {
+            verbosePrintParameters(vdsVisit.params, System.err);
+        }
+
+        try {
+            vdsVisit.run();
+        } catch (Exception e) {
+            e.printStackTrace();
+            System.exit(1);
+        }
+    }
+
+    private void printSyntax(Options options) {
+        HelpFormatter formatter = new HelpFormatter();
+        formatter.printHelp("vespavisit <options>", "Visit documents from VDS", options , "");
+    }
+
+    @SuppressWarnings("AccessStaticViaInstance")
+    protected static Options createOptions() {
+        Options options = new Options();
+        options.addOption("h", "help", false, "Show this syntax page.");
+
+        options.addOption(Option.builder("d")
+                .longOpt("datahandler")
+                .hasArg(true)
+                .argName("target")
+                .desc("Send results to the given target.")
+                .build());
+
+        options.addOption(Option.builder("s")
+                .longOpt("selection")
+                .hasArg(true)
+                .argName("selection")
+                .desc("What documents to visit.")
+                .build());
+
+        options.addOption(Option.builder("f")
+                .longOpt("from")
+                .hasArg(true)
+                .argName("timestamp")
+                .desc("Only visit from the given timestamp (microseconds).")
+                .type(Number.class)
+                .build());
+
+        options.addOption(Option.builder("t")
+                .longOpt("to")
+                .hasArg(true)
+                .argName("timestamp")
+                .desc("Only visit up to the given timestamp (microseconds).")
+                .type(Number.class).build());
+
+        options.addOption("e", "headersonly", false, "Only visit headers of documents.[Deprecated]");
+
+        options.addOption(Option.builder("l")
+                .longOpt("fieldset")
+                .hasArg(true)
+                .argName("fieldset")
+                .desc("Retrieve the specified fields only (see http://vespa.corp.yahoo.com/5/documentation/reference/fieldsets.html). Default is [all].")
+                .build());
+
+        options.addOption(Option.builder()
+                .longOpt("visitinconsistentbuckets")
+                .hasArg(false)
+                .desc("Don't wait for inconsistent buckets to become consistent.")
+                .build());
+
+        options.addOption(Option.builder("m")
+                .longOpt("maxpending")
+                .hasArg(true)
+                .argName("num")
+                .desc("Maximum pending messages to data handlers per storage visitor.")
+                .type(Number.class)
+                .build());
+
+        options.addOption(Option.builder()
+                .longOpt("maxpendingsuperbuckets")
+                .hasArg(true)
+                .argName("num")
+                .desc("Maximum pending visitor messages from the vespavisit client. If set, dynamic throttling of visitors will be disabled!")
+                .type(Number.class)
+                .build());
+
+        options.addOption(Option.builder("b")
+                .longOpt("maxbuckets")
+                .hasArg(true)
+                .argName("num")
+                .desc("Maximum buckets per visitor.")
+                .type(Number.class)
+                .build());
+
+        options.addOption("i", "printids", false, "Display only document identifiers.");
+
+        options.addOption(Option.builder("p")
+                .longOpt("progress")
+                .hasArg(true)
+                .argName("file")
+                .desc("Use given file to track progress.")
+                .build());
+
+        options.addOption(Option.builder("o")
+                .longOpt("timeout")
+                .hasArg(true)
+                .argName("milliseconds")
+                .desc("Time out visitor after given time.")
+                .type(Number.class)
+                .build());
+
+        options.addOption(Option.builder("u")
+                .longOpt("buckettimeout")
+                .hasArg(true)
+                .argName("milliseconds")
+                .desc("Fail visitor if visiting a single bucket takes longer than this (default same as timeout)")
+                .type(Number.class)
+                .build());
+
+        options.addOption(Option.builder()
+                .longOpt("visitlibrary")
+                .hasArg(true)
+                .argName("string")
+                .desc("Use the given visitor library.")
+                .build());
+
+        options.addOption(Option.builder()
+                .longOpt("libraryparam")
+                .numberOfArgs(2)
+                .argName("key> <val")
+                .desc("Give the following parameter to the visitor.")
+                .build());
+
+        options.addOption("r", "visitremoves", false, "Include information of removed documents.");
+
+        options.addOption(Option.builder("c")
+                .longOpt("cluster")
+                .hasArg(true)
+                .argName("cluster")
+                .desc("Visit the given VDS cluster.")
+                .build());
+
+        options.addOption("v", "verbose", false, "Indent XML, show progress and info on STDERR.");
+
+        options.addOption(Option.builder()
+                .longOpt("statistics")
+                .hasArg(true)
+                .argName("args")
+                .desc("Use CountVisitor for document statistics. Use comma-separated arguments.")
+                .build());
+
+        options.addOption(Option.builder()
+                .longOpt("abortonclusterdown")
+                .hasArg(false)
+                .desc("Abort if cluster is down.")
+                .build());
+
+        options.addOption(Option.builder()
+                .longOpt("maxhits")
+                .hasArg(true)
+                .argName("num")
+                .desc("Abort visiting when we have received this many \"first pass\" documents. Only appropriate for visiting involving id.order. This is only an approximate number, all pending work will be completed and those documents will also be returned.")
+                .type(Number.class)
+                .build());
+
+        options.addOption(Option.builder()
+                .longOpt("maxtotalhits")
+                .hasArg(true)
+                .argName("num")
+                .desc("Abort visiting when we have received this many total documents. This is only an approximate number, all pending work will be completed and those documents will also be returned.")
+                .type(Number.class)
+                .build());
+
+        options.addOption(Option.builder()
+                .longOpt("processtime")
+                .hasArg(true)
+                .argName("num")
+                .desc("Sleep this amount of millisecs before processing message. (Debug option for pretending to be slow client)")
+                .type(Number.class)
+                .build());
+        options.addOption(Option.builder()
+                .longOpt("priority")
+                .hasArg(true)
+                .argName("name")
+                .desc("Priority used for each visitor. Defaults to NORMAL_3. " +
+                        "Use with care to avoid starving lower prioritized traffic in the cluster")
+                .build());
+
+        options.addOption(Option.builder()
+                .longOpt("ordering")
+                .hasArg(true)
+                .argName("order")
+                .desc("Order to visit documents in. Only makes sense in conjunction with a document selection involving id.order. Legal values are \"ascending\" and \"descending\"")
+                .build());
+
+        options.addOption(Option.builder()
+                .longOpt("tracelevel")
+                .hasArg(true)
+                .argName("level")
+                .desc("Tracelevel ([0-9]) to use for debugging purposes")
+                .type(Number.class)
+                .build());
+
+        options.addOption(Option.builder()
+                .longOpt("skipbucketsonfatalerrors")
+                .hasArg(false)
+                .desc("Skip visiting super buckets with fatal error codes.")
+                .build());
+
+        options.addOption(Option.builder()
+                .longOpt("jsonoutput")
+                .desc("Output documents as JSON")
+                .hasArg(false)
+                .build());
+        return options;
+    }
+
+    public static class VdsVisitParameters {
+        private VisitorParameters visitorParameters;
+        /** If not specified in options, will get form cluster list */
+        private String cluster = null;
+        private boolean verbose = false;
+        private boolean printIdsOnly = false;
+        private String statisticsParts = null;
+        private boolean abortOnClusterDown = false;
+        private int processTime = 0;
+        private int fullTimeout = 7 * 24 * 60 * 60 * 1000;
+        private boolean jsonOutput = false;
+
+        public VisitorParameters getVisitorParameters() {
+            return visitorParameters;
+        }
+
+        public void setVisitorParameters(VisitorParameters visitorParameters) {
+            this.visitorParameters = visitorParameters;
+        }
+
+        public String getCluster() {
+            return cluster;
+        }
+
+        public void setCluster(String cluster) {
+            this.cluster = cluster;
+        }
+
+        public boolean isVerbose() {
+            return verbose;
+        }
+
+        public void setVerbose(boolean verbose) {
+            this.verbose = verbose;
+        }
+
+        public boolean isPrintIdsOnly() {
+            return printIdsOnly;
+        }
+
+        public void setPrintIdsOnly(boolean printIdsOnly) {
+            this.printIdsOnly = printIdsOnly;
+        }
+
+        public String getStatisticsParts() {
+            return statisticsParts;
+        }
+
+        public void setStatisticsParts(String statisticsParts) {
+            this.statisticsParts = statisticsParts;
+        }
+
+        public boolean getAbortOnClusterDown() {
+            return abortOnClusterDown;
+        }
+
+        public void setAbortOnClusterDown(boolean abortOnClusterDown) {
+            this.abortOnClusterDown = abortOnClusterDown;
+        }
+
+        public int getFullTimeout() {
+            return fullTimeout;
+        }
+
+        public void setFullTimeout(int fullTimeout) {
+            this.fullTimeout = fullTimeout;
+        }
+
+        public int getProcessTime() {
+            return processTime;
+        }
+
+        public void setProcessTime(int processTime) {
+            this.processTime = processTime;
+        }
+
+        public void setJsonOutput(boolean jsonOutput) {
+            this.jsonOutput = jsonOutput;
+        }
+    }
+
+    protected static class ArgumentParser {
+        private Options options;
+
+        public ArgumentParser(Options options) {
+            this.options = options;
+        }
+
+        public VdsVisitParameters parse(String args[]) throws org.apache.commons.cli.ParseException {
+            VdsVisitParameters allParams = new VdsVisitParameters();
+            VisitorParameters params = new VisitorParameters("");
+            CommandLineParser parser = new DefaultParser();
+            CommandLine line = parser.parse(options, args);
+
+            if (line.hasOption("h")) {
+                return null;
+            }
+            if (line.hasOption("d")) {
+                params.setRemoteDataHandler(line.getOptionValue("d"));
+            }
+            if (line.hasOption("s")) {
+                params.setDocumentSelection(line.getOptionValue("s"));
+            }
+            if (line.hasOption("f")) {
+                params.setFromTimestamp(((Number) line.getParsedOptionValue("f")).longValue());
+            }
+            if (line.hasOption("t")) {
+                params.setToTimestamp(((Number) line.getParsedOptionValue("t")).longValue());
+            }
+            if (line.hasOption("e")) {
+                params.fieldSet("[header]");
+            }
+            if (line.hasOption("l")) {
+                params.fieldSet(line.getOptionValue("l"));
+            }
+            if (line.hasOption("visitinconsistentbuckets")) {
+                params.visitInconsistentBuckets(true);
+            }
+            if (line.hasOption("m")) {
+                params.setMaxPending(((Number) line.getParsedOptionValue("m")).intValue());
+            }
+            if (line.hasOption("b")) {
+                params.setMaxBucketsPerVisitor(((Number) line.getParsedOptionValue("b")).intValue());
+            }
+            if (line.hasOption("i")) {
+                allParams.setPrintIdsOnly(true);
+                params.fieldSet("[id]");
+            }
+            if (line.hasOption("p")) {
+                params.setResumeFileName(line.getOptionValue("p"));
+            }
+            if (line.hasOption("o")) {
+                allParams.setFullTimeout(((Number) line.getParsedOptionValue("o")).intValue());
+                params.setTimeoutMs(allParams.getFullTimeout());
+            }
+            if (line.hasOption("u")) {
+                params.setTimeoutMs(((Number) line.getParsedOptionValue("u")).intValue());
+            }
+            if (line.hasOption("visitlibrary")) {
+                params.setVisitorLibrary(line.getOptionValue("visitlibrary"));
+            }
+            if (line.hasOption("libraryparam")) {
+                String key = line.getOptionValues("libraryparam")[0];
+                String value = line.getOptionValues("libraryparam")[1];
+                params.setLibraryParameter(key, value);
+            }
+            if (line.hasOption("r")) {
+                params.visitRemoves(true);
+            }
+            if (line.hasOption("c")) {
+                allParams.setCluster(line.getOptionValue("c"));
+            }
+
+            if (line.hasOption("v")) {
+                allParams.setVerbose(true);
+            }
+
+            if (line.hasOption("statistics")) {
+                allParams.setStatisticsParts(line.getOptionValue("statistics"));
+                params.fieldSet("[id]");
+                params.setVisitorLibrary("CountVisitor");
+            }
+
+            if (line.hasOption("abortonclusterdown")) {
+                allParams.setAbortOnClusterDown(true);
+            }
+            if (line.hasOption("processtime")) {
+                allParams.setProcessTime(((Number) line.getParsedOptionValue("processtime")).intValue());
+            }
+            if (line.hasOption("maxhits")) {
+                params.setMaxFirstPassHits(((Number)line.getParsedOptionValue("maxhits")).intValue());
+            }
+            if (line.hasOption("maxtotalhits")) {
+                params.setMaxTotalHits(((Number)line.getParsedOptionValue("maxtotalhits")).intValue());
+            }
+            if (line.hasOption("tracelevel")) {
+                params.setTraceLevel(((Number)line.getParsedOptionValue("tracelevel")).intValue());
+            }
+            if (line.hasOption("priority")) {
+                try {
+                    DocumentProtocol.Priority priority = DocumentProtocol.getPriorityByName(
+                            line.getOptionValue("priority"));
+                    params.setPriority(priority);
+                } catch (IllegalArgumentException e) {
+                    throw new IllegalArgumentException("Unknown priority name");
+                }
+            } else {
+                // Let bulk visitor jobs have a low priority by default to avoid stalling concurrent
+                // (real time) write and read operations.
+                params.setPriority(DocumentProtocol.Priority.LOW_1);
+            }
+            if (line.hasOption("ordering")) {
+                String opt = line.getOptionValue("ordering");
+                if (opt.equalsIgnoreCase("ascending")) {
+                    params.setVisitorOrdering(OrderingSpecification.ASCENDING);
+                } else if (opt.equalsIgnoreCase("descending")) {
+                    params.setVisitorOrdering(OrderingSpecification.DESCENDING);
+                } else {
+                    throw new IllegalArgumentException("Unknown ordering. Legal values are \"ascending\", \"descending\"");
+                }
+            }
+            if (line.hasOption("skipbucketsonfatalerrors")) {
+                params.skipBucketsOnFatalErrors(true);
+            }
+            if (line.hasOption("maxpendingsuperbuckets")) {
+                StaticThrottlePolicy throttlePolicy = new StaticThrottlePolicy();
+                throttlePolicy.setMaxPendingCount(((Number)line.getParsedOptionValue("maxpendingsuperbuckets")).intValue());
+                params.setThrottlePolicy(throttlePolicy);
+            }
+            if (line.hasOption("jsonoutput")) {
+                allParams.setJsonOutput(true);
+            }
+
+            allParams.setVisitorParameters(params);
+            return allParams;
+        }
+    }
+
+    // For unit testing only
+    protected void setVdsVisitParameters(VdsVisitParameters vdsVisitParameters) {
+        this.params = vdsVisitParameters;
+    }
+
+    protected static String resolveClusterRoute(ClusterList clusters, String wantedCluster) {
+        if (clusters.getStorageClusters().size() == 0) {
+            throw new IllegalArgumentException("Your Vespa cluster does not have any content clusters " +
+                    "declared. Visiting feature is not available.");
+        }
+
+        ClusterDef found = null;
+
+        String names = "";
+        for (ClusterDef c : clusters.getStorageClusters()) {
+            if (!names.isEmpty()) {
+                names += ", ";
+            }
+            names += c.getName();
+        }
+        if (wantedCluster != null) {
+            for (ClusterDef c : clusters.getStorageClusters()) {
+                if (c.getName().equals(wantedCluster)) {
+                    found = c;
+                }
+            }
+            if (found == null) {
+                throw new IllegalArgumentException("Your vespa cluster contains the content clusters " +
+                        names + ", not " + wantedCluster + ". Please select a valid vespa cluster.");
+            }
+        } else if (clusters.getStorageClusters().size() == 1) {
+            found = clusters.getStorageClusters().get(0);
+        } else {
+            throw new IllegalArgumentException("Your vespa cluster contains the content clusters " +
+                    names + ". Please use the -c option to select one of them as a target for visiting.");
+        }
+
+        return "[Storage:cluster=" + found.getName() + ";clusterconfigid=" + found.getConfigId() + "]";
+    }
+
+    protected static void verbosePrintParameters(VdsVisitParameters vdsParams, PrintStream out) {
+        VisitorParameters params = vdsParams.getVisitorParameters();
+        if (params.getTimeoutMs() != -1) {
+            out.println("Time out visitor after " + params.getTimeoutMs() + " ms.");
+        }
+        if (params.getDocumentSelection() == null || params.getDocumentSelection().equals("")) {
+            out.println("Visiting all documents");
+        } else {
+            out.println("Visiting documents matching: " + params.getDocumentSelection());
+        }
+        if (params.getFromTimestamp() != 0 && params.getToTimestamp() != 0) {
+            out.println("Visiting in the inclusive timestamp range "
+                               + params.getFromTimestamp() + " - " + params.getToTimestamp() + ".");
+        } else if (params.getFromTimestamp() != 0) {
+            out.println("Visiting from and including timestamp " + params.getFromTimestamp() + ".");
+        } else if (params.getToTimestamp() != 0) {
+            out.println("Visiting to and including timestamp " + params.getToTimestamp() + ".");
+        }
+        out.println("Visiting field set " + params.fieldSet() + ".");
+        if (params.visitInconsistentBuckets()) {
+            out.println("Visiting inconsistent buckets.");
+        }
+        if (params.visitRemoves()) {
+            out.println("Including remove entries.");
+        }
+        if (params.getResumeFileName() != null && !"".equals(params.getResumeFileName())) {
+            out.println("Tracking progress in file: " + params.getResumeFileName());
+        }
+        if (vdsParams.isPrintIdsOnly()) {
+            out.println("Only showing document identifiers.");
+        }
+        out.println("Let visitor have maximum " + params.getMaxPending() + " replies pending on data handlers per storage node visitor.");
+        out.println("Visit maximum " + params.getMaxBucketsPerVisitor() + " buckets per visitor.");
+        if (params.getRemoteDataHandler() != null) {
+            out.println("Sending data to data handler at: " + params.getRemoteDataHandler());
+        }
+        if (params.getRoute() != null) {
+            out.println("Visiting cluster '" + params.getRoute() + "'.");
+        }
+        if (params.getVisitorLibrary() != null) {
+            out.println("Using visitor library '" + params.getVisitorLibrary() + "'.");
+        }
+        if (params.getLibraryParameters().size() > 0) {
+            out.println("Adding the following library specific parameters:");
+            for (Map.Entry<String, byte[]> entry : params.getLibraryParameters().entrySet()) {
+                out.println("  " + entry.getKey() + " = " +
+                        new String(entry.getValue(), Charset.forName("utf-8")));
+            }
+        }
+        if (params.getPriority() != DocumentProtocol.Priority.NORMAL_3) {
+            out.println("Visitor priority " + params.getPriority().name());
+        }
+        if (params.skipBucketsOnFatalErrors()) {
+            out.println("Skip visiting super buckets with fatal errors.");
+        }
+    }
+
+    private void onDocumentSelectionException(Exception e) {
+        System.err.println("Illegal document selection string '" +
+                params.getVisitorParameters().getDocumentSelection() + "'.\n");
+        System.exit(1);
+    }
+
+    private void onIllegalArgumentException(Exception e) {
+        System.err.println("Illegal arguments : \n");
+        System.err.println(e.getMessage());
+        System.exit(1);
+    }
+
+    public void run() {
+        System.exit(doRun());
+    }
+
+    protected int doRun() {
+        VisitorParameters visitorParameters = params.getVisitorParameters();
+        // If progress file already exists, create resume token from it
+        if (visitorParameters.getResumeFileName() != null &&
+            !"".equals(visitorParameters.getResumeFileName()))
+        {
+            try {
+                File file = new File(visitorParameters.getResumeFileName());
+                FileInputStream fos = new FileInputStream(file);
+
+                StringBuilder builder = new StringBuilder();
+                byte[] b = new byte[100000];
+                int length;
+
+                while ((length = fos.read(b)) > 0) {
+                    builder.append(new String(b, 0, length));
+                }
+                fos.close();
+                visitorParameters.setResumeToken(new ProgressToken(builder.toString()));
+
+                if (params.isVerbose()) {
+                    System.err.format("Resuming visitor already %.1f %% finished.\n",
+                            visitorParameters.getResumeToken().percentFinished());
+                }
+            } catch (FileNotFoundException e) {
+                // Ignore; file has not been created yet but will be shortly.
+            } catch (IOException e) {
+                System.err.println("Could not open progress file: " + visitorParameters.getResumeFileName());
+                e.printStackTrace(System.err);
+                return 1;
+            }
+        }
+
+        initShutdownHook();
+        sessionAccessor = sessionAccessorFactory.createVisitorSessionAccessor();
+
+        VdsVisitHandler handler;
+
+        handler = new StdOutVisitorHandler(
+                params.isPrintIdsOnly(),
+                params.isVerbose(),
+                params.isVerbose(),
+                params.isVerbose(),
+                params.getStatisticsParts() != null,
+                params.getAbortOnClusterDown(),
+                params.getProcessTime(),
+                params.jsonOutput);
+
+        if (visitorParameters.getResumeFileName() != null) {
+            handler.setProgressFileName(visitorParameters.getResumeFileName());
+        }
+
+        visitorParameters.setControlHandler(handler.getControlHandler());
+        if (visitorParameters.getRemoteDataHandler() == null) {
+            visitorParameters.setLocalDataHandler(handler.getDataHandler());
+        }
+
+        if (params.getStatisticsParts() != null) {
+            String[] parts = params.getStatisticsParts().split(",");
+            for (String s : parts) {
+                visitorParameters.setLibraryParameter(s, "true");
+            }
+        }
+
+        try {
+            session = sessionAccessor.createVisitorSession(visitorParameters);
+            while (true) {
+                try {
+                    if (session.waitUntilDone(params.getFullTimeout())) break;
+                } catch (InterruptedException e) {}
+            }
+
+            if (visitorParameters.getTraceLevel() > 0) {
+                System.out.println(session.getTrace().toString());
+            }
+        } catch (ParseException e) {
+            onDocumentSelectionException(e);
+        } catch (IllegalArgumentException e) {
+            onIllegalArgumentException(e);
+        } catch (Exception e) {
+            System.err.println("Document selection string was: " + visitorParameters.getDocumentSelection());
+            System.err.println("Caught unexpected exception: ");
+            e.printStackTrace(System.err);
+            return 1;
+        }
+        if (visitorParameters.getControlHandler().getResult().code
+                == VisitorControlHandler.CompletionCode.SUCCESS)
+        {
+            return 0;
+        } else {
+            return 1;
+        }
+    }
+
+    private void initShutdownHook() {
+        shutdownHookRegistrar.registerShutdownHook(new CleanUpThread());
+    }
+
+    class CleanUpThread extends Thread {
+        public void run() {
+            try {
+                if (session != null) {
+                    session.destroy();
+                }
+            } catch (IllegalStateException ise) {
+                //ignore this
+            }
+            try {
+                if (sessionAccessor != null) {
+                    sessionAccessor.shutdown();
+                }
+            } catch (IllegalStateException ise) {
+                //ignore this too
+            }
+        }
+    }
+}
diff --git a/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisitHandler.java b/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisitHandler.java
new file mode 100644
index 00000000000..ad3e0a0b359
--- /dev/null
+++ b/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisitHandler.java
@@ -0,0 +1,181 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespavisit;
+
+import com.yahoo.documentapi.ProgressToken;
+import com.yahoo.documentapi.VisitorControlHandler;
+import com.yahoo.documentapi.VisitorDataHandler;
+import com.yahoo.vdslib.VisitorStatistics;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+
+import java.util.Date;
+import java.util.TimeZone;
+import java.text.DateFormat;
+import java.text.DecimalFormat;
+import java.text.SimpleDateFormat;
+
+/**
+ * An abstract class that can be subclassed by different visitor handlers.
+ *
+ * @author <a href="mailto:thomasg@yahoo-inc.com">Thomas Gundersen</a>
+ */
+public abstract class VdsVisitHandler {
+    boolean showProgress;
+    boolean showStatistics;
+    boolean abortOnClusterDown;
+    boolean lastLineIsProgress = false;
+    String lastPercentage;
+    final Object printLock = new Object();
+
+    protected String progressFileName = "";
+
+    final VisitorControlHandler controlHandler = new ControlHandler();
+
+    public VdsVisitHandler(boolean showProgress, boolean showStatistics, boolean abortOnClusterDown)
+    {
+        this.showProgress = showProgress;
+        this.showStatistics = showStatistics;
+        this.abortOnClusterDown = abortOnClusterDown;
+    }
+
+    public boolean getShowProgress() {
+        return showProgress;
+    }
+
+    public boolean getShowStatistics() {
+        return showStatistics;
+    }
+
+    public boolean getAbortOnClusterDown() {
+        return abortOnClusterDown;
+    }
+
+    public boolean getLastLineIsProgress() {
+        return lastLineIsProgress;
+    }
+
+    public void setLastLineIsProgress(boolean isProgress) {
+        lastLineIsProgress = isProgress;
+    }
+
+    public String getLastPercentage() {
+        return lastPercentage;
+    }
+
+    public void setLastPercentage(String lastPercentage) {
+        this.lastPercentage = lastPercentage;
+    }
+
+    public Object getPrintLock() {
+        return printLock;
+    }
+
+    public void onDone() {
+    }
+
+    public String getProgressFileName() {
+        return progressFileName;
+    }
+
+    public void setProgressFileName(String progressFileName) {
+        this.progressFileName = progressFileName;
+    }
+
+    public VisitorControlHandler getControlHandler() { return controlHandler; }
+    public abstract VisitorDataHandler getDataHandler();
+
+    class ControlHandler extends VisitorControlHandler {
+        VisitorStatistics statistics;
+
+        public void onProgress(ProgressToken token) {
+            if (progressFileName.length() > 0) {
+                 try {
+                     synchronized (token) {
+                         File file = new File(progressFileName + ".tmp");
+                         FileOutputStream fos = new FileOutputStream(file);
+                         fos.write(token.toString().getBytes());
+                         fos.close();
+                         file.renameTo(new File(progressFileName));
+                     }
+                 }
+                 catch (IOException e) {
+                     e.printStackTrace();
+                 }
+            }
+            if (showProgress) {
+                synchronized (printLock) {
+                    DecimalFormat df = new DecimalFormat("#.#");
+                    String percentage = df.format(token.percentFinished());
+                    if (!percentage.equals(lastPercentage)) {
+                        if (lastLineIsProgress) {
+                            System.err.print('\r');
+                        }
+                        System.err.print(percentage + " % finished.");
+                        lastLineIsProgress = true;
+                        lastPercentage = percentage;
+                    }
+                }
+            }
+            super.onProgress(token);
+        }
+
+        @Override
+        public void onVisitorStatistics(VisitorStatistics visitorStatistics) {
+            statistics = visitorStatistics;
+        }
+
+        private String getDateTime() {
+            DateFormat dateFormat =
+                new SimpleDateFormat("yyyy-MM-dd HH:mm:ss zzz");
+            dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+            Date date = new Date();
+            return dateFormat.format(date);
+        }
+
+        public void onVisitorError(String message) {
+            synchronized (printLock) {
+                if (lastLineIsProgress) {
+                    System.err.print('\r');
+                    lastLineIsProgress = false;
+                }
+                System.err.println("Visitor error (" + getDateTime() + "): " +
+                                   message);
+                if (abortOnClusterDown &&
+                    !isDone() &&
+                    (message.lastIndexOf("Could not resolve")>=0 ||
+                     message.lastIndexOf("don't allow external load")>=0)) {
+                    System.err.println("Aborting visitor as " +
+                                       "--abortonclusterdown flag is set.");
+                    abort();
+                }
+            }
+        }
+        public void onDone(CompletionCode code, String message) {
+            if (lastLineIsProgress) {
+                System.err.print('\n');
+                lastLineIsProgress = false;
+            }
+            if (code != CompletionCode.SUCCESS) {
+                if (code == CompletionCode.ABORTED) {
+                    System.err.println("Visitor aborted: " + message);
+                } else if (code == CompletionCode.TIMEOUT) {
+                    System.err.println("Visitor timed out: " + message);
+                } else {
+                    System.err.println("Visitor aborted due to unknown issue "
+                                     + code + ": " + message);
+                }
+            } else {
+                if (showProgress) {
+                    System.err.println("Completed visiting.");
+                }
+                if (showStatistics) {
+                    System.err.println("*** Visitor statistics");
+                    System.err.println(statistics == null ? "Nothing visited" : statistics.toString());
+                }
+            }
+            super.onDone(code, message);
+        }
+    }
+}
diff --git a/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisitTarget.java b/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisitTarget.java
new file mode 100644
index 00000000000..3ef0619cfd8
--- /dev/null
+++ b/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisitTarget.java
@@ -0,0 +1,286 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespavisit;
+
+import com.yahoo.documentapi.DocumentAccess;
+import com.yahoo.documentapi.VisitorControlHandler;
+import com.yahoo.documentapi.VisitorDataHandler;
+import com.yahoo.documentapi.VisitorDestinationParameters;
+import com.yahoo.documentapi.VisitorDestinationSession;
+import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess;
+import com.yahoo.documentapi.messagebus.MessageBusParams;
+import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet;
+import com.yahoo.log.LogLevel;
+import com.yahoo.log.LogSetup;
+import com.yahoo.messagebus.network.Identity;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.DefaultParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+
+import java.lang.reflect.Constructor;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.logging.Logger;
+
+/**
+ * Example client using visiting
+ *
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>, based on work by <a href="mailto:humbe@yahoo-inc.com">H&aring;kon Humberset</a>
+ */
+public class VdsVisitTarget {
+    private static final Logger log = Logger.getLogger(VdsVisitTarget.class.getName());
+
+    private boolean printIds = false;
+    DocumentAccess access;
+    VisitorDestinationSession session;
+    String slobrokAddress = null;
+    int port = -1;
+    private boolean verbose = false;
+    private int processTime = 0;
+    private String handlerClassName = StdOutVisitorHandler.class.getName();
+    private String[] handlerArgs = null;
+
+    public boolean isPrintIds() {
+        return printIds;
+    }
+
+    public String getSlobrokAddress() {
+        return slobrokAddress;
+    }
+
+    public boolean isVerbose() {
+        return verbose;
+    }
+
+    public int getPort() {
+        return port;
+    }
+
+    public int getProcessTime() {
+        return processTime;
+    }
+
+    public String getHandlerClassName() {
+        return handlerClassName;
+    }
+
+    public String[] getHandlerArgs() {
+        return handlerArgs;
+    }
+
+    public static void main(String args[]) {
+        LogSetup.initVespaLogging("vespavisittarget");
+        VdsVisitTarget visitTarget = new VdsVisitTarget();
+
+
+        try {
+            visitTarget.parseArguments(args);
+            visitTarget.initShutdownHook();
+            visitTarget.run();
+            System.exit(0);
+        } catch (HelpShownException e) {
+            System.exit(0);
+        } catch (IllegalArgumentException e) {
+            System.err.println(e.getMessage());
+            System.exit(1);
+        } catch (org.apache.commons.cli.ParseException e) {
+            System.err.println("Failed to parse arguments. Try --help for syntax. " + e.getMessage());
+            System.exit(1);
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+
+    private static Options createOptions() {
+        Options options = new Options();
+
+        options.addOption("h", "help", false, "Show this syntax page.");
+
+        options.addOption(Option.builder("s")
+                .longOpt("bindtoslobrok")
+                .hasArg(true)
+                .argName("address")
+                .desc("Bind to the given slobrok address.")
+                .build());
+
+        options.addOption(Option.builder("t")
+                .longOpt("bindtosocket")
+                .hasArg(true)
+                .argName("port")
+                .desc("Bind to the given TCP port")
+                .type(Number.class)
+                .build());
+
+        options.addOption(Option.builder("p")
+                .longOpt("processtime")
+                .hasArg(true)
+                .argName("msecs")
+                .desc("Sleep this amount of millisecs before processing message. (Debug option for pretending to be slow client).")
+                .type(Number.class)
+                .build());
+
+        options.addOption(Option.builder("c")
+                .longOpt("visithandler")
+                .hasArg(true)
+                .argName("classname")
+                .desc("Use the given class as a visit handler (defaults to StdOutVisitorHandler)")
+                .build());
+
+        options.addOption(Option.builder("o")
+                .longOpt("visitoptions")
+                .hasArg(true)
+                .argName("args")
+                .desc("Option arguments to pass through to the visitor handler instance")
+                .build());
+
+        options.addOption("i", "printids", false, "Display only document identifiers.");
+        options.addOption("v", "verbose", false, "Indent XML, show progress and info on STDERR.");
+
+        return options;
+    }
+
+    private void printSyntax(Options options) {
+        HelpFormatter formatter = new HelpFormatter();
+        formatter.printHelp("vespavisittarget <options>", "Retrieve results from a visitor", options ,
+                            "One, and only one, of the binding options must be present.\n" +
+                            "\n" +
+                            "For more detailed information, such as defaults and format of\n" +
+                            "arguments, refer to 'man vespavisittarget'.\n");
+    }
+
+    class HelpShownException extends Exception {}
+
+    void parseArguments(String args[]) throws ParseException, HelpShownException {
+        Options options = createOptions();
+
+        CommandLineParser parser = new DefaultParser();
+        CommandLine line = parser.parse(options, args);
+
+        if (line.hasOption("h")) {
+            printSyntax(options);
+            throw new HelpShownException();
+        }
+        if (line.hasOption("s")) {
+            slobrokAddress = line.getOptionValue("s");
+        }
+        if (line.hasOption("t")) {
+            port = ((Number) line.getParsedOptionValue("t")).intValue();
+        }
+        if (line.hasOption("i")) {
+            printIds = true;
+        }
+        if (line.hasOption("p")) {
+            processTime = ((Number) line.getParsedOptionValue("p")).intValue();
+        }
+        if (line.hasOption("v")) {
+            verbose = true;
+        }
+        if (line.hasOption("c")) {
+            handlerClassName = line.getOptionValue("c");
+        }
+        if (line.hasOption("o")) {
+            handlerArgs = line.getOptionValue("o").split(" ");
+        }
+
+        if (!(slobrokAddress == null ^ port == -1)) {
+            throw new IllegalArgumentException("You must specify one, and only one, binding option");
+        }
+        if (port != -1 && port < 0 || port > 65535) {
+            throw new IllegalArgumentException("The port must be in the range 0-65535");
+        }
+        if (verbose) {
+            if (port != -1) {
+                System.err.println("Binding to socket " + getTcpAddress());
+            } else {
+                System.err.println("Binding to slobrok address: " + slobrokAddress + "/visit-destination");
+            }
+        }
+    }
+
+    private String getTcpAddress() {
+        try {
+            InetAddress addr = InetAddress.getLocalHost();
+            String hostname = addr.getHostName();
+            return "tcp/" + hostname + ":" + port + "/visit-destination";
+        } catch (UnknownHostException e) {
+            System.err.println("Failed to detect hostname.");
+            System.exit(1);
+        }
+        return "";
+    }
+
+    @SuppressWarnings("unchecked")
+    public void run() throws Exception {
+        initShutdownHook();
+        log.log(LogLevel.DEBUG, "Starting VdsVisitTarget");
+        MessageBusParams mbusParams = new MessageBusParams(new LoadTypeSet());
+        mbusParams.getRPCNetworkParams().setIdentity(new Identity(slobrokAddress));
+
+        if (port > 0) {
+            mbusParams.getRPCNetworkParams().setListenPort(port);
+        }
+
+        access = new MessageBusDocumentAccess(mbusParams);
+
+        VdsVisitHandler handler;
+
+        Class<?> cls = Thread.currentThread().getContextClassLoader()
+                .loadClass(handlerClassName);
+        try {
+            // Any custom data handlers may have a constructor that takes in args,
+            // so that the user can pass cmd line options to them
+            Class<?>[] consTypes = new Class<?>[] { boolean.class, boolean.class,
+                    boolean.class, boolean.class, boolean.class,
+                    boolean.class, int.class, String[].class };
+            Constructor<?> cons = cls.getConstructor(consTypes);
+            handler = (VdsVisitHandler)cons.newInstance(
+                    printIds, verbose, verbose, verbose, false, false,
+                    processTime, handlerArgs);
+        } catch (NoSuchMethodException e) {
+            // Retry, this time matching the StdOutVisitorHandler constructor
+            // arg list
+            Class<?>[] consTypes = new Class<?>[] { boolean.class, boolean.class,
+                    boolean.class, boolean.class, boolean.class,
+                    boolean.class, int.class, boolean.class };
+            Constructor<?> cons = cls.getConstructor(consTypes);
+            handler = (VdsVisitHandler)cons.newInstance(
+                    printIds, verbose, verbose, verbose, false, false, processTime, false);
+        }
+
+        VisitorDataHandler dataHandler = handler.getDataHandler();
+        VisitorControlHandler controlHandler = handler.getControlHandler();
+
+        VisitorDestinationParameters params = new VisitorDestinationParameters(
+                "visit-destination", dataHandler);
+        session = access.createVisitorDestinationSession(params);
+        while (!controlHandler.isDone()) {
+            Thread.sleep(1000);
+        }
+    }
+
+    private void initShutdownHook() {
+        Runtime.getRuntime().addShutdownHook(new CleanUpThread());
+    }
+
+    class CleanUpThread extends Thread {
+        public void run() {
+            try {
+                if (session != null) {
+                    session.destroy();
+                }
+            } catch (IllegalStateException ise) {
+                //ignore this
+            }
+            try {
+                if (access != null) {
+                    access.shutdown();
+                }
+            } catch (IllegalStateException ise) {
+                //ignore this too
+            }
+        }
+    }
+}
diff --git a/vespaclient-java/src/main/sh/vds-document-statistics.sh b/vespaclient-java/src/main/sh/vds-document-statistics.sh
new file mode 100755
index 00000000000..3677137dbd4
--- /dev/null
+++ b/vespaclient-java/src/main/sh/vds-document-statistics.sh
@@ -0,0 +1,20 @@
+#!/bin/sh
+test -z "$VESPA_HOME" && VESPA_HOME=/home/y
+
+. $VESPA_HOME/libexec/vespa/common-env.sh
+
+function help {
+    echo "Usage: vds-document-statistics [ category, ... ]"
+    echo "  Where category is one or more of: user, group, scheme, namespace"
+    echo ""
+    echo "vds-document-statistics generates documents counts based on one or more categories."
+    exit 0
+}
+if [ "$1" == "-h" ]; then
+  help
+fi
+if [ "$1" == "" ]; then
+  help
+fi
+export MALLOC_ARENA_MAX=1 #Does not need fast allocation
+exec java -Xms32m -Xmx128m $(getJavaOptionsIPV46) -cp ${VESPA_HOME}/lib/jars/vespaclient-java-jar-with-dependencies.jar com.yahoo.vespavisit.Main --statistics "$1"
diff --git a/vespaclient-java/src/main/sh/vdsstat.sh b/vespaclient-java/src/main/sh/vdsstat.sh
new file mode 100644
index 00000000000..ef3e2cdbe20
--- /dev/null
+++ b/vespaclient-java/src/main/sh/vdsstat.sh
@@ -0,0 +1,13 @@
+#!/bin/sh
+test -z "$VESPA_HOME" && VESPA_HOME=/home/y
+
+. $VESPA_HOME/libexec/vespa/common-env.sh
+
+export MALLOC_ARENA_MAX=1 #Does not need fast allocation
+exec java \
+-server -enableassertions \
+-XX:ThreadStackSize=512 \
+-XX:MaxJavaStackTraceDepth=-1 \
+-Djava.awt.headless=true \
+-Xms128m -Xmx1024m $(getJavaOptionsIPV46) \
+-cp ${VESPA_HOME}/lib/jars/vespaclient-java-jar-with-dependencies.jar com.yahoo.vespastat.Main "$@"
diff --git a/vespaclient-java/src/main/sh/vespa-query-profile-dump-tool.sh b/vespaclient-java/src/main/sh/vespa-query-profile-dump-tool.sh
new file mode 100755
index 00000000000..1a70fcb006d
--- /dev/null
+++ b/vespaclient-java/src/main/sh/vespa-query-profile-dump-tool.sh
@@ -0,0 +1,6 @@
+#!/bin/sh
+test -z "$VESPA_HOME" && VESPA_HOME=/home/y
+
+. $VESPA_HOME/libexec/vespa/common-env.sh
+
+java $(getJavaOptionsIPV46) -cp ${VESPA_HOME}/lib/jars/vespaclient-java-jar-with-dependencies.jar com.yahoo.search.query.profile.DumpTool $@
diff --git a/vespaclient-java/src/main/sh/vespa-summary-benchmark.sh b/vespaclient-java/src/main/sh/vespa-summary-benchmark.sh
new file mode 100755
index 00000000000..7534639d07b
--- /dev/null
+++ b/vespaclient-java/src/main/sh/vespa-summary-benchmark.sh
@@ -0,0 +1,15 @@
+#!/bin/sh
+test -z "$VESPA_HOME" && VESPA_HOME=/home/y
+
+. $VESPA_HOME/libexec/vespa/common-env.sh
+
+export VESPA_LOG_TARGET=file:/dev/null
+export MALLOC_ARENA_MAX=1 # Does not need fast allocation
+java \
+-server -enableassertions \
+-XX:ThreadStackSize=512 \
+-XX:MaxJavaStackTraceDepth=-1 \
+-Djava.library.path=${VESPA_HOME}/libexec64/native:${VESPA_HOME}/lib64 \
+-XX:MaxDirectMemorySize=32m -Djava.awt.headless=true \
+-Xms128m -Xmx1024m $(getJavaOptionsIPV46) \
+-cp ${VESPA_HOME}/lib/jars/vespaclient-java-jar-with-dependencies.jar com.yahoo.vespasummarybenchmark.VespaSummaryBenchmark "$@"
diff --git a/vespaclient-java/src/main/sh/vespadestination.sh b/vespaclient-java/src/main/sh/vespadestination.sh
new file mode 100755
index 00000000000..f2168cb6db2
--- /dev/null
+++ b/vespaclient-java/src/main/sh/vespadestination.sh
@@ -0,0 +1,12 @@
+#!/bin/sh
+test -z "$VESPA_HOME" && VESPA_HOME=/home/y
+
+. $VESPA_HOME/libexec/vespa/common-env.sh
+
+export MALLOC_ARENA_MAX=1 #Does not need fast allocation
+exec java \
+-server -enableassertions \
+-XX:ThreadStackSize=512 \
+-Djava.library.path=${VESPA_HOME}/libexec64/native:${VESPA_HOME}/lib64 \
+-XX:MaxDirectMemorySize=32m -Djava.awt.headless=true $(getJavaOptionsIPV46) \
+-cp ${VESPA_HOME}/lib/jars/vespaclient-java-jar-with-dependencies.jar:$CLASSPATH com.yahoo.dummyreceiver.DummyReceiver "$@"
diff --git a/vespaclient-java/src/main/sh/vespafeeder.sh b/vespaclient-java/src/main/sh/vespafeeder.sh
new file mode 100755
index 00000000000..f74bc794ed7
--- /dev/null
+++ b/vespaclient-java/src/main/sh/vespafeeder.sh
@@ -0,0 +1,15 @@
+#!/bin/sh
+test -z "$VESPA_HOME" && VESPA_HOME=/home/y
+
+. $VESPA_HOME/libexec/vespa/common-env.sh
+
+export VESPA_LOG_TARGET=file:/dev/null
+export MALLOC_ARENA_MAX=1 # Does not need fast allocation
+java \
+-server -enableassertions \
+-XX:ThreadStackSize=512 \
+-XX:MaxJavaStackTraceDepth=-1 \
+-Djava.library.path=${VESPA_HOME}/libexec64/native:${VESPA_HOME}/lib64 \
+-XX:MaxDirectMemorySize=32m -Djava.awt.headless=true \
+-Xms128m -Xmx1024m $(getJavaOptionsIPV46) \
+-cp ${VESPA_HOME}/lib/jars/vespaclient-java-jar-with-dependencies.jar com.yahoo.vespafeeder.VespaFeeder "$@"
diff --git a/vespaclient-java/src/main/sh/vespaget.sh b/vespaclient-java/src/main/sh/vespaget.sh
new file mode 100644
index 00000000000..514ff170742
--- /dev/null
+++ b/vespaclient-java/src/main/sh/vespaget.sh
@@ -0,0 +1,14 @@
+#!/bin/sh
+test -z "$VESPA_HOME" && VESPA_HOME=/home/y
+
+. $VESPA_HOME/libexec/vespa/common-env.sh
+
+export MALLOC_ARENA_MAX=1 #Does not need fast allocation
+exec java \
+-server -enableassertions \
+-XX:ThreadStackSize=512 \
+-XX:MaxJavaStackTraceDepth=-1 \
+-Djava.awt.headless=true \
+-DVESPA_LOG_LEVEL="all -debug -spam -config -info -event" \
+-Xms128m -Xmx1024m $(getJavaOptionsIPV46) \
+-cp ${VESPA_HOME}/lib/jars/vespaclient-java-jar-with-dependencies.jar com.yahoo.vespaget.Main "$@"
diff --git a/vespaclient-java/src/main/sh/vespavisit.1 b/vespaclient-java/src/main/sh/vespavisit.1
new file mode 100644
index 00000000000..b9a6c488bf9
--- /dev/null
+++ b/vespaclient-java/src/main/sh/vespavisit.1
@@ -0,0 +1,159 @@
+.TH VESPAVISIT 1 2008-03-07 "Vespa" "Vespa Documentation"
+.SH NAME
+vespavisit \- Visit documents from a Vespa installation
+.SH SYNPOSIS
+.B vespavisit
+[\fIOPTION\fR]...
+.SH DESCRIPTION
+.PP
+In the regular case, retrieve documents stored in VESPA, and either print
+them to STDOUT or send them to a given MessageBus route.
+.PP
+A Vespa visit operation processes a set of stored documents, in undefined
+order, locally on the storage nodes where they are stored. A visitor library
+available on all storage nodes will receive the documents stored locally, and
+can process these and send messages to the visitor data handler. The regular
+case is to use the DumpVisitor library to merely send the documents themselves
+in blocks back to the data handler, which by default is this client that will
+write the documents to STDOUT.
+.PP
+Mandatory arguments to long options are mandatory for short options too.
+Short options can not currently be concatenated together.
+.TP
+\fB\-s\fR, \fB\-\-selection\fR \fISELECTION\fR
+A document selection string, specifying what documents to visit. Documentation
+on the language itself can be found in the documentation. Note that this argument
+should probably be quoted to prevent your shell from invalidating your
+selection.
+.TP
+\fB\-f\fR, \fB\-\-from\fR \fITIME\fR
+If this option is given, only documents from given timestamp or newer will be
+visited. The time is given in microseconds since 1970.
+.TP
+\fB\-t\fR, \fB\-\-to\fR \fITIME\fR
+If this option is given, only documents up to and including the given timestamp
+will be visited. The time is given in microseconds since 1970.
+.TP
+\fB\-e\fR, \fB\-\-headersonly\fR
+By default, the whole documents stored are processed. If this option is given
+only the header parts of documents will be processed. By defining the big
+document fields as body fields, you can efficiently visit all the header fields
+using this option.
+.TP
+\fB\-i\fR, \fB\-\-printids\fR
+Using this option, only the document identifiers will be printed to STDOUT.
+In addition, if visiting removes, an additional tag will be added so you can
+see whether document has been removed or not. This option implies headers only
+visiting, and can only be used if no datahandler is specified.
+.TP
+\fB\-d\fR, \fB\-\-datahandler\fR \fIVISITTARGET\fR
+The data handler is the destination of messages sent from the visitor library.
+By default, the data handler is the vespavisit process you start, which will
+merely print all returned data to STDOUT. A visit target can be specified
+instead. See the chapter below on visit targets.
+.TP
+\fB\-p\fR, \fB\-\-progress\fR \fIFILE\fR
+By setting a progress file, current visitor progress will be saved to this
+file at regular intervals. If this file exists on startup, the visitor will
+continue from this point.
+.TP
+\fB\-o\fR, \fB\-\-timeout\fR \fITIMEOUT\fR
+Time out the visitor after given number of milliseconds.
+.TP
+\fB\-r\fR, \fB\-\-visitremoves\fR
+By default, only documents existing in Vespa will be processed. By giving
+this option, also entries identifying documents previously existing will
+be returned. This is useful for secondary copies of data that wants to know
+whether documents it has stored has been removed. Note that documents deleted
+a long time ago will no longer be tracked. Vespa keeps remove entries for
+a configurable amount of time.
+.TP
+\fB\-m\fR, \fB\-\-maxpending\fR \fINUM\fR
+Maximum pending docblock messages to data handlers. This may be used to
+increase or reduce visiting speed, but should not be set too high so that data
+handlers run out of memory. To get an estimate of memory consumption on each
+data handler, multiply maxpending with defaultdocblocksize in stor-visitor
+config and divide by number of data handlers. Default value for maxpending is
+16.
+.TP
+\fB\-c\fR, \fB\-\-cluster\fR \fICLUSTER\fR
+Visit the given VDS cluster.
+.TP
+\fB\-v\fR, \fB\-\-verbose\fR
+More verbose output. Indent XML and add progress and info to STDERR.
+.TP
+\fB\-h\fR, \fB\-\-help\fR
+Shows a short syntax reminder.
+.PP
+Advanced options:
+.PP
+The below options are used for advanced usage or for testing.
+.TP
+\fB\-\-visitlibrary\fR \fILIBRARY\fR
+By default, the DumpVisitor library, sending documents back to the data handler,
+is used when visiting. Another library can be specified using this option. The
+library filename should be the name given here, with lib prepended and .so
+appended.
+.TP
+\fB\-\-libraryparam\fR \fIKEY\fR \fIVALUE\fR
+The default DumpVisitor library has no options to set, but custom libraries
+may need user specifiable options. Here such options can be specified. Look
+at visitor library documentation for legal parameters.
+.TP
+\fB\-\-polling\fR \fIarg\fR
+The document API implements both a polling and a callback visitor API. The
+callback API is most efficient and used by default. The polling API might be
+simpler for users used to such APIs. Some VESPA system tests use this option
+to test that the polling API works.
+.TP
+\fB\-\-visitinconsistentbuckets\fR
+In some cases Vespa may temporarily be in an inconsistent state, that is,
+different nodes contain different copies of the data. Collections of documents
+are grouped into so-called buckets. The normal behavior of visiting is to wait
+for the inconsistencies to resolve before actually visiting the data. This
+might be a problem for time critical applications. Setting this option will
+result in the bucket copy with most documents to be visited in case of
+inconsistencies, which means that the data returned by the visitor are not
+guaranteed to be correct.
+.SH VISIT TARGET
+Results from visiting can be sent to many different kind of targets.
+.TP
+\fBMessage bus routes\fR
+You can specify a message bus route name directly, and this route will be used
+to send the results. This is typically used when doing reprocessing within
+Vespa. Message bus routes are set up in the application package. In addition
+some routes may have been autogenerated in simple setups, for instance a
+route called \fIdefault\fR is generated if your setup is so simple that Vespa
+can guess where you want to send your data.
+.TP
+\fBSlobrok address\fR
+You can also specify a slobrok address for data to be sent to. A slobrok address
+is a slash separated path where you can use asterisk to mean any element within
+this path. For instance, if you have a docproc cluster called \fImydpcluster\fR
+it will have registered its nodes with slobrok names like
+\fIdocproc/cluster.mydpcluster/docproc/0/feed_processor\fR, where the 0 here
+indicates the first node in the cluster. You can thus specify to send visit data
+to this docproc cluster by stating a slobrok address of
+\fIdocproc/cluster.mydpcluster/docproc/*/feed_processor\fR. Note that this will
+not send all the data to one or all the nodes. The data sent from the visitor
+will be distributed among the matching nodes, but each message will just be sent
+to one node.
+
+Slobrok names may also be used if you use the \fBvespavisittarget\fR tool to
+retrieve the data at some location. If you start vespavisittarget on two nodes,
+listening to slobrok names \fImynode/0/visit-destination\fR and
+\fImynode/1/visit-destination\fR you can send the results to these nodes by
+specifying \fImynode/*/visit-destination\fR as the data handler. See
+\fBman vespavisittarget\fR for naming conventions used for such targets.
+.TP
+\fBTCP socket\fR
+TCP sockets can also be specified directly. This requires that the endpoint
+speaks FNET RPC though. This is typically done, either by using the
+\fBvespavisittarget\fR tool, or by using a visitor destination programmatically
+by using utility class in the document API. A socket address looks like the
+following: tcp/\fIhostname\fR:\fIport\fR/\fIservicename\fR. For instance, an
+address generated by the \fBvespavisittarget\fR tool might look like the
+following: \fItcp/myhost.com:12345/visit-destination\fR.
+
+.SH AUTHOR
+Written by Haakon Humberset.
diff --git a/vespaclient-java/src/main/sh/vespavisit.sh b/vespaclient-java/src/main/sh/vespavisit.sh
new file mode 100755
index 00000000000..eb6c9487b88
--- /dev/null
+++ b/vespaclient-java/src/main/sh/vespavisit.sh
@@ -0,0 +1,14 @@
+#!/bin/sh
+test -z "$VESPA_HOME" && VESPA_HOME=/home/y
+
+. $VESPA_HOME/libexec/vespa/common-env.sh
+
+export MALLOC_ARENA_MAX=1 #Does not need fast allocation
+exec java \
+-server -enableassertions \
+-XX:ThreadStackSize=512 \
+-XX:MaxJavaStackTraceDepth=-1 \
+-Djava.library.path=${VESPA_HOME}/libexec64/native:${VESPA_HOME}/lib64 \
+-XX:MaxDirectMemorySize=32m -Djava.awt.headless=true \
+-Xms128m -Xmx1024m $(getJavaOptionsIPV46) \
+-cp ${VESPA_HOME}/lib/jars/vespaclient-java-jar-with-dependencies.jar com.yahoo.vespavisit.VdsVisit "$@"
diff --git a/vespaclient-java/src/main/sh/vespavisittarget.1 b/vespaclient-java/src/main/sh/vespavisittarget.1
new file mode 100644
index 00000000000..7f02215d558
--- /dev/null
+++ b/vespaclient-java/src/main/sh/vespavisittarget.1
@@ -0,0 +1,40 @@
+.TH VESPAVISITTARGET 1 2008-03-07 "Vespa" "Vespa Documentation"
+.SH NAME
+vespavisittarget \- An endpoint for documents visited from a Vespa installation
+.SH SYNPOSIS
+.B vespavisittarget
+[\fIOPTION\fR]...
+.SH DESCRIPTION
+.PP
+When visiting data from Vespa, you might not want to send the data back to the
+controlling process. By using separate visitor targets you can divide load
+between multiple nodes and have the controlling process run at another location.
+The document API has utility classes to set up end points for visitor data from
+Vespa. This application is a small tool that uses these utilities and merely
+writes the data retrieved to STDOUT in XML format.
+.PP
+Mandatory arguments to long options are mandatory for short options too.
+Short options can not currently be concatenated together.
+.TP
+\fB\-s\fR, \fB\-\-bindtoslobrok\fR \fISLOBROKADDRESS\fR
+Bind to the given slobrok address. Note that the value \fI/visit-destination\fR
+will be appended to the given address.
+.TP
+\fB\-t\fR, \fB\-\-bindtosocket\fR \fIPORT\fR
+Bind to the given TCP socket. This will make sure we listen to the given port.
+No slobrok registration is done using this option, so you need to specify
+TCP socket address in visitors to get data sent to this destination.
+.TP
+\fB\-i\fR, \fB\-\-printids\fR
+Using this option, only the document identifiers will be printed to STDOUT.
+In addition, if visiting removes, an additional tag will be added so you can
+see whether document has been removed or not. This option implies headers only
+visiting, and can only be used if no datahandler is specified.
+.TP
+\fB\-v\fR, \fB\-\-verbose\fR
+More verbose output. Indent XML.
+.TP
+\fB\-h\fR, \fB\-\-help\fR
+Shows a short syntax reminder.
+.SH AUTHOR
+Written by Haakon Humberset.
diff --git a/vespaclient-java/src/main/sh/vespavisittarget.sh b/vespaclient-java/src/main/sh/vespavisittarget.sh
new file mode 100755
index 00000000000..7eb9fe17e04
--- /dev/null
+++ b/vespaclient-java/src/main/sh/vespavisittarget.sh
@@ -0,0 +1,13 @@
+#!/bin/sh
+test -z "$VESPA_HOME" && VESPA_HOME=/home/y
+
+. $VESPA_HOME/libexec/vespa/common-env.sh
+
+export MALLOC_ARENA_MAX=1 #Does not need fast allocation
+exec java \
+-server -enableassertions \
+-XX:ThreadStackSize=512 \
+-XX:MaxJavaStackTraceDepth=-1 \
+-Djava.library.path=${VESPA_HOME}/libexec64/native:${VESPA_HOME}/lib64 \
+-XX:MaxDirectMemorySize=32m -Djava.awt.headless=true $(getJavaOptionsIPV46) \
+-cp ${VESPA_HOME}/lib/jars/vespaclient-java-jar-with-dependencies.jar:$CLASSPATH com.yahoo.vespavisit.VdsVisitTarget "$@"
diff --git a/vespaclient-java/src/test/files/documentmanager.cfg b/vespaclient-java/src/test/files/documentmanager.cfg
new file mode 100644
index 00000000000..966361d1fe3
--- /dev/null
+++ b/vespaclient-java/src/test/files/documentmanager.cfg
@@ -0,0 +1,113 @@
+enablecompression false
+datatype[10]
+datatype[0].id 1002
+datatype[0].arraytype[1]
+datatype[0].arraytype[0].datatype 2
+datatype[0].weightedsettype[0]
+datatype[0].structtype[0]
+datatype[0].documenttype[0]
+datatype[1].id 1000
+datatype[1].arraytype[1]
+datatype[1].arraytype[0].datatype 0
+datatype[1].weightedsettype[0]
+datatype[1].structtype[0]
+datatype[1].documenttype[0]
+datatype[2].id 1004
+datatype[2].arraytype[1]
+datatype[2].arraytype[0].datatype 4
+datatype[2].weightedsettype[0]
+datatype[2].structtype[0]
+datatype[2].documenttype[0]
+datatype[3].id 1016
+datatype[3].arraytype[1]
+datatype[3].arraytype[0].datatype 16
+datatype[3].weightedsettype[0]
+datatype[3].structtype[0]
+datatype[3].documenttype[0]
+datatype[4].id 1001
+datatype[4].arraytype[1]
+datatype[4].arraytype[0].datatype 1
+datatype[4].weightedsettype[0]
+datatype[4].structtype[0]
+datatype[4].documenttype[0]
+datatype[5].id 2001
+datatype[5].arraytype[0]
+datatype[5].weightedsettype[1]
+datatype[5].weightedsettype[0].datatype 0
+datatype[5].weightedsettype[0].createifnonexistant false
+datatype[5].weightedsettype[0].removeifzero false
+datatype[5].structtype[0]
+datatype[5].documenttype[0]
+datatype[6].id 2002
+datatype[6].arraytype[0]
+datatype[6].weightedsettype[1]
+datatype[6].weightedsettype[0].datatype 2
+datatype[6].weightedsettype[0].createifnonexistant false
+datatype[6].weightedsettype[0].removeifzero false
+datatype[6].structtype[0]
+datatype[6].documenttype[0]
+datatype[7].id -628990518
+datatype[7].arraytype[0]
+datatype[7].weightedsettype[0]
+datatype[7].structtype[1]
+datatype[7].structtype[0].name news.header
+datatype[7].structtype[0].version 0
+datatype[7].structtype[0].field[6]
+datatype[7].structtype[0].field[0].name url
+datatype[7].structtype[0].field[0].id[0]
+datatype[7].structtype[0].field[0].datatype 10
+datatype[7].structtype[0].field[1].name title
+datatype[7].structtype[0].field[1].id[0]
+datatype[7].structtype[0].field[1].datatype 2
+datatype[7].structtype[0].field[2].name last_downloaded
+datatype[7].structtype[0].field[2].id[0]
+datatype[7].structtype[0].field[2].datatype 0
+datatype[7].structtype[0].field[3].name value_long
+datatype[7].structtype[0].field[3].id[0]
+datatype[7].structtype[0].field[3].datatype 4
+datatype[7].structtype[0].field[4].name value_content
+datatype[7].structtype[0].field[4].id[0]
+datatype[7].structtype[0].field[4].datatype 2
+datatype[7].structtype[0].field[5].name stringarr
+datatype[7].structtype[0].field[5].id[0]
+datatype[7].structtype[0].field[5].datatype 1002
+datatype[7].documenttype[0]
+datatype[8].id 538588767
+datatype[8].arraytype[0]
+datatype[8].weightedsettype[0]
+datatype[8].structtype[1]
+datatype[8].structtype[0].name news.body
+datatype[8].structtype[0].version 0
+datatype[8].structtype[0].field[7]
+datatype[8].structtype[0].field[0].name intarr
+datatype[8].structtype[0].field[0].id[0]
+datatype[8].structtype[0].field[0].datatype 1000
+datatype[8].structtype[0].field[1].name longarr
+datatype[8].structtype[0].field[1].id[0]
+datatype[8].structtype[0].field[1].datatype 1004
+datatype[8].structtype[0].field[2].name bytearr
+datatype[8].structtype[0].field[2].id[0]
+datatype[8].structtype[0].field[2].datatype 1016
+datatype[8].structtype[0].field[3].name floatarr
+datatype[8].structtype[0].field[3].id[0]
+datatype[8].structtype[0].field[3].datatype 1001
+datatype[8].structtype[0].field[4].name weightedsetint
+datatype[8].structtype[0].field[4].id[0]
+datatype[8].structtype[0].field[4].datatype 2001
+datatype[8].structtype[0].field[5].name weightedsetstring
+datatype[8].structtype[0].field[5].id[0]
+datatype[8].structtype[0].field[5].datatype 2002
+datatype[8].structtype[0].field[6].name content
+datatype[8].structtype[0].field[6].id[0]
+datatype[8].structtype[0].field[6].datatype 2
+datatype[8].documenttype[0]
+datatype[9].id -1048827947
+datatype[9].arraytype[0]
+datatype[9].weightedsettype[0]
+datatype[9].structtype[0]
+datatype[9].documenttype[1]
+datatype[9].documenttype[0].name news
+datatype[9].documenttype[0].version 0
+datatype[9].documenttype[0].inherits[0]
+datatype[9].documenttype[0].headerstruct -628990518
+datatype[9].documenttype[0].bodystruct 538588767
diff --git a/vespaclient-java/src/test/files/malformedfeed.json b/vespaclient-java/src/test/files/malformedfeed.json
new file mode 100644
index 00000000000..26691ada676
--- /dev/null
+++ b/vespaclient-java/src/test/files/malformedfeed.json
@@ -0,0 +1,13 @@
+[
+{
+    "put": "id:test:news::foo",
+    "fields": {}
+},
+{
+    "update": "id:test:news::foo",
+    "fields": {}
+},
+{
+    "remove": "id:test:news::foo"
+},
+]
diff --git a/vespaclient-java/src/test/files/myfeed.json b/vespaclient-java/src/test/files/myfeed.json
new file mode 100644
index 00000000000..544f370e62a
--- /dev/null
+++ b/vespaclient-java/src/test/files/myfeed.json
@@ -0,0 +1,13 @@
+[
+{
+    "put": "id:test:news::foo",
+    "fields": {}
+},
+{
+    "update": "id:test:news::foo",
+    "fields": {}
+},
+{
+    "remove": "id:test:news::foo"
+}
+]
diff --git a/vespaclient-java/src/test/files/myfeed.xml b/vespaclient-java/src/test/files/myfeed.xml
new file mode 100644
index 00000000000..008d4fcba13
--- /dev/null
+++ b/vespaclient-java/src/test/files/myfeed.xml
@@ -0,0 +1,5 @@
+<vespafeed>
+    <document documentid="doc:test:foo" documenttype="news"/>
+    <update documentid="doc:test:foo" documenttype="news"/>
+    <remove documentid="doc:test:foo"/>
+</vespafeed>
+\ No newline at end of file
diff --git a/vespaclient-java/src/test/files/progress.txt b/vespaclient-java/src/test/files/progress.txt
new file mode 100644
index 00000000000..77323926140
--- /dev/null
+++ b/vespaclient-java/src/test/files/progress.txt
@@ -0,0 +1,8 @@
+VDS bucket progress file
+14
+4704
+4701
+16384
+3800000000000004:8c00000600000004
+3800000000000002:8c00000000000002
+3800000000003e92:0
diff --git a/vespaclient-java/src/test/java/com/yahoo/vespafeeder/BenchmarkProgressPrinterTest.java b/vespaclient-java/src/test/java/com/yahoo/vespafeeder/BenchmarkProgressPrinterTest.java
new file mode 100644
index 00000000000..2b0b4cc9048
--- /dev/null
+++ b/vespaclient-java/src/test/java/com/yahoo/vespafeeder/BenchmarkProgressPrinterTest.java
@@ -0,0 +1,77 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespafeeder;
+
+import com.yahoo.clientmetrics.RouteMetricSet;
+import com.yahoo.concurrent.Timer;
+import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage;
+import com.yahoo.documentapi.messagebus.protocol.UpdateDocumentMessage;
+import com.yahoo.messagebus.EmptyReply;
+import junit.framework.TestCase;
+
+import java.io.ByteArrayOutputStream;
+import java.io.PrintStream;
+
+/**
+ */
+public class BenchmarkProgressPrinterTest extends TestCase {
+
+    class DummyTimer implements Timer {
+        long ms;
+
+        public long milliTime() { return ms; }
+    }
+
+    public void testSimple() {
+        ByteArrayOutputStream output = new ByteArrayOutputStream();
+        DummyTimer timer = new DummyTimer();
+        timer.ms = 0;
+        BenchmarkProgressPrinter printer = new BenchmarkProgressPrinter(timer, new PrintStream(output));
+        RouteMetricSet metrics = new RouteMetricSet("foobar", printer);
+
+        {
+            EmptyReply reply = new EmptyReply();
+            reply.setMessage(PutDocumentMessage.createEmpty().setTimeReceived(1));
+            metrics.addReply(reply);
+        }
+
+        timer.ms = 1200;
+
+        {
+            EmptyReply reply = new EmptyReply();
+            reply.setMessage(PutDocumentMessage.createEmpty().setTimeReceived(2));
+            metrics.addReply(reply);
+        }
+
+        {
+            EmptyReply reply = new EmptyReply();
+            reply.setMessage(UpdateDocumentMessage.createEmpty().setTimeReceived(3));
+            metrics.addReply(reply);
+        }
+
+        timer.ms = 2400;
+
+        {
+            EmptyReply reply = new EmptyReply();
+            reply.setMessage(UpdateDocumentMessage.createEmpty().setTimeReceived(4));
+            reply.addError(new com.yahoo.messagebus.Error(32, "foo"));
+            metrics.addReply(reply);
+        }
+
+        timer.ms = 62000;
+
+        {
+            EmptyReply reply = new EmptyReply();
+            reply.setMessage(UpdateDocumentMessage.createEmpty().setTimeReceived(5));
+            reply.addError(new com.yahoo.messagebus.Error(64, "bar"));
+            metrics.addReply(reply);
+        }
+
+        metrics.done();
+
+        String val = output.toString().split("\n")[1];
+
+        String correctPattern = "62000,\\s*3,\\s*2,\\s*\\d+,\\s*\\d+,\\s*\\d+$";
+        assertTrue("Value '" + val + "' does not match pattern '" + correctPattern + "'", val.matches(correctPattern));
+    }
+
+}
diff --git a/vespaclient-java/src/test/java/com/yahoo/vespafeeder/ProgressPrinterTest.java b/vespaclient-java/src/test/java/com/yahoo/vespafeeder/ProgressPrinterTest.java
new file mode 100644
index 00000000000..ae49bb1318d
--- /dev/null
+++ b/vespaclient-java/src/test/java/com/yahoo/vespafeeder/ProgressPrinterTest.java
@@ -0,0 +1,90 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespafeeder;
+
+import com.yahoo.clientmetrics.RouteMetricSet;
+import com.yahoo.concurrent.Timer;
+import com.yahoo.documentapi.messagebus.protocol.DocumentIgnoredReply;
+import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage;
+import com.yahoo.documentapi.messagebus.protocol.UpdateDocumentMessage;
+import com.yahoo.messagebus.EmptyReply;
+import junit.framework.TestCase;
+
+import java.io.ByteArrayOutputStream;
+import java.io.PrintStream;
+
+/**
+ */
+public class ProgressPrinterTest extends TestCase {
+
+    class DummyTimer implements Timer {
+        long ms;
+
+        public long milliTime() { return ms; }
+    }
+
+    public void testSimple() {
+        ByteArrayOutputStream output = new ByteArrayOutputStream();
+        DummyTimer timer = new DummyTimer();
+        timer.ms = 0;
+        ProgressPrinter printer = new ProgressPrinter(timer, new PrintStream(output));
+        RouteMetricSet metrics = new RouteMetricSet("foobar", printer);
+
+        {
+            EmptyReply reply = new EmptyReply();
+            reply.setMessage(PutDocumentMessage.createEmpty().setTimeReceived(1));
+            metrics.addReply(reply);
+        }
+
+        timer.ms = 1200;
+
+        {
+            EmptyReply reply = new EmptyReply();
+            reply.setMessage(PutDocumentMessage.createEmpty().setTimeReceived(2));
+            metrics.addReply(reply);
+        }
+
+        {
+            EmptyReply reply = new EmptyReply();
+            reply.setMessage(UpdateDocumentMessage.createEmpty().setTimeReceived(3));
+            metrics.addReply(reply);
+        }
+
+        timer.ms = 2400;
+
+        {
+            DocumentIgnoredReply reply = new DocumentIgnoredReply();
+            reply.setMessage(PutDocumentMessage.createEmpty().setTimeReceived(0));
+            metrics.addReply(reply);
+        }
+
+        {
+            EmptyReply reply = new EmptyReply();
+            reply.setMessage(UpdateDocumentMessage.createEmpty().setTimeReceived(5));
+            reply.addError(new com.yahoo.messagebus.Error(32, "foo"));
+            metrics.addReply(reply);
+        }
+
+        timer.ms = 62000;
+
+        {
+            EmptyReply reply = new EmptyReply();
+            reply.setMessage(UpdateDocumentMessage.createEmpty().setTimeReceived(6));
+            reply.addError(new com.yahoo.messagebus.Error(64, "bar"));
+            metrics.addReply(reply);
+        }
+
+        String val = output.toString().replaceAll("latency\\(min, max, avg\\): .*", "latency(min, max, avg): 0, 0, 0");
+
+        String correct =
+                "\rSuccessfully sent 2 messages so far" +
+                "\rSuccessfully sent 3 messages so far" +
+                "\n\n" +
+                "Messages sent to vespa (route foobar) :\n" +
+                "---------------------------------------\n" +
+                "PutDocument:\tok: 2 msgs/sec: 0.03 failed: 0 ignored: 1 latency(min, max, avg): 0, 0, 0\n" +
+                "UpdateDocument:\tok: 1 msgs/sec: 0.02 failed: 2 ignored: 0 latency(min, max, avg): 0, 0, 0\n";
+
+        assertEquals(correct, val);
+    }
+
+}
diff --git a/vespaclient-java/src/test/java/com/yahoo/vespafeeder/VespaFeederTestCase.java b/vespaclient-java/src/test/java/com/yahoo/vespafeeder/VespaFeederTestCase.java
new file mode 100644
index 00000000000..42d4b082ff3
--- /dev/null
+++ b/vespaclient-java/src/test/java/com/yahoo/vespafeeder/VespaFeederTestCase.java
@@ -0,0 +1,208 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespafeeder;
+
+import static org.junit.Assert.*;
+
+import java.io.BufferedInputStream;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.util.ArrayList;
+import java.util.Arrays;
+
+import com.yahoo.clientmetrics.RouteMetricSet;
+import com.yahoo.document.DocumentTypeManager;
+import com.yahoo.document.DocumentTypeManagerConfigurer;
+import com.yahoo.document.DocumentUpdate;
+import com.yahoo.documentapi.messagebus.protocol.*;
+import com.yahoo.feedapi.DummySessionFactory;
+import com.yahoo.feedhandler.VespaFeedHandler;
+import com.yahoo.text.Utf8;
+import com.yahoo.vespaclient.config.FeederConfig;
+import com.yahoo.vespafeeder.Arguments.HelpShownException;
+
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+public class VespaFeederTestCase {
+
+    @Rule
+    public ExpectedException exception = ExpectedException.none();
+
+    @Test
+    public void testParseArgs() throws Exception {
+        String argsS="--abortondataerror false --abortonsenderror false --file foo.xml --maxpending 10" +
+                " --maxpendingsize 11 --maxfeedrate 29 --mode benchmark --noretry --retrydelay 12 --route e6 --timeout 13 --trace 4" +
+                " --validate -v bar.xml --priority LOW_1";
+
+        Arguments arguments = new Arguments(argsS.split(" "), DummySessionFactory.createWithAutoReply());
+
+        FeederConfig config = arguments.getFeederConfig();
+        assertEquals(false, config.abortondocumenterror());
+        assertEquals(13.0, config.timeout(), 0.00001);
+        assertEquals(false, config.retryenabled());
+        assertEquals(12.0, config.retrydelay(), 0.0001);
+        assertEquals("e6", config.route());
+        assertEquals(4, config.tracelevel());
+        assertEquals(false, config.abortonsenderror());
+        assertEquals(10, config.maxpendingdocs());
+        assertEquals(11, config.maxpendingbytes());
+        assertEquals(29.0, config.maxfeedrate(), 0.0001);
+        assertTrue(arguments.isVerbose());
+        assertFalse(config.createifnonexistent());
+
+        assertEquals("LOW_1", arguments.getPriority());
+        assertEquals("benchmark", arguments.getMode());
+        assertEquals("foo.xml", arguments.getFiles().get(0));
+        assertEquals("bar.xml", arguments.getFiles().get(1));
+    }
+
+    @Test
+    public void requireThatCreateIfNonExistentArgumentCanBeParsed() throws Exception {
+        String argsS="--create-if-non-existent --file foo.xml";
+        Arguments arguments = new Arguments(argsS.split(" "), DummySessionFactory.createWithAutoReply());
+        assertTrue(arguments.getFeederConfig().createifnonexistent());
+    }
+
+    @Test
+    public void testHelp() throws Exception {
+        String argsS="-h";
+
+        try {
+            new Arguments(argsS.split(" "), null);
+            assertTrue(false);
+        } catch (Arguments.HelpShownException e) {
+
+        }
+    }
+
+    @Test
+    public void requireCorrectInputTypeDetection() throws IOException {
+        {
+            BufferedInputStream b = new BufferedInputStream(
+                    new ByteArrayInputStream(Utf8.toBytes("[]")));
+            InputStreamRequest r = new InputStreamRequest(b);
+            VespaFeeder.setJsonInput(r, b);
+            assertEquals("true", r.getProperty(VespaFeedHandler.JSON_INPUT));
+        }
+        {
+            BufferedInputStream b = new BufferedInputStream(
+                    new ByteArrayInputStream(Utf8.toBytes("<document />")));
+            InputStreamRequest r = new InputStreamRequest(b);
+            VespaFeeder.setJsonInput(r, b);
+            assertEquals("false", r.getProperty(VespaFeedHandler.JSON_INPUT));
+        }
+    }
+
+    public void assertRenderErrorOutput(String expected, String[] errors) {
+        ArrayList<String> l = new ArrayList<String>();
+        l.addAll(Arrays.asList(errors));
+        assertEquals(expected, VespaFeeder.renderErrors(l).getMessage());
+    }
+
+    @Test
+    public void testRenderErrors() {
+        {
+            String[] errors = { "foo" };
+            assertRenderErrorOutput("Errors:\n" +
+                                    "-------\n" +
+                                    "    foo\n", errors);
+        }
+
+        {
+            String[] errors = { "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11"};
+            assertRenderErrorOutput("First 10 errors (of 11):\n" +
+                    "------------------------\n" +
+                    "    1\n    2\n    3\n    4\n    5\n    6\n    7\n    8\n    9\n    10\n", errors);
+        }
+    }
+
+    public RouteMetricSet.ProgressCallback getProgressPrinter(String args) throws Exception {
+        Arguments arguments = new Arguments(args.split(" "), DummySessionFactory.createWithAutoReply());
+        return new VespaFeeder(arguments, null).createProgressCallback(System.out);
+    }
+
+    @Test
+    public void testCreateProgressPrinter() throws Exception {
+        assert(getProgressPrinter("--mode benchmark") instanceof BenchmarkProgressPrinter);
+        assert(getProgressPrinter("") instanceof ProgressPrinter);
+    }
+
+    private static class FeedFixture {
+        DummySessionFactory sessionFactory = DummySessionFactory.createWithAutoReply();
+        ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
+        PrintStream printStream = new PrintStream(outputStream);
+        DocumentTypeManager typeManager = new DocumentTypeManager();
+        FeedFixture() {
+            DocumentTypeManagerConfigurer.configure(typeManager, "file:src/test/files/documentmanager.cfg");
+        }
+    }
+
+    @Test
+    public void feedFile() throws Exception {
+        FeedFixture f = new FeedFixture();
+        Arguments arguments = new Arguments("--file src/test/files/myfeed.xml --priority LOW_1".split(" "), f.sessionFactory);
+        new VespaFeeder(arguments, f.typeManager).parseFiles(System.in, f.printStream);
+
+        assertEquals(3, f.sessionFactory.messages.size());
+        assertEquals(DocumentProtocol.Priority.LOW_1, ((PutDocumentMessage)f.sessionFactory.messages.get(0)).getPriority());
+        assertEquals("doc:test:foo", ((PutDocumentMessage) f.sessionFactory.messages.get(0)).getDocumentPut().getDocument().getId().toString());
+        DocumentUpdate update = ((UpdateDocumentMessage) f.sessionFactory.messages.get(1)).getDocumentUpdate();
+        assertEquals("doc:test:foo", update.getId().toString());
+        assertFalse(update.getCreateIfNonExistent());
+        assertEquals("doc:test:foo", ((RemoveDocumentMessage) f.sessionFactory.messages.get(2)).getDocumentId().toString());
+
+        assertTrue(f.outputStream.toString().contains("Messages sent to vespa"));
+    }
+
+    @Test
+    public void feedJson() throws Exception {
+        FeedFixture feedFixture = feed("src/test/files/myfeed.json", true);
+
+        assertJsonFeedState(feedFixture);
+    }
+
+    protected void assertJsonFeedState(FeedFixture feedFixture) {
+        assertEquals(3, feedFixture.sessionFactory.messages.size());
+        assertEquals(DocumentProtocol.Priority.LOW_1, ((PutDocumentMessage)feedFixture.sessionFactory.messages.get(0)).getPriority());
+        assertEquals("id:test:news::foo", ((PutDocumentMessage) feedFixture.sessionFactory.messages.get(0)).getDocumentPut().getDocument().getId().toString());
+        DocumentUpdate update = ((UpdateDocumentMessage) feedFixture.sessionFactory.messages.get(1)).getDocumentUpdate();
+        assertEquals("id:test:news::foo", update.getId().toString());
+        assertFalse(update.getCreateIfNonExistent());
+        assertEquals("id:test:news::foo", ((RemoveDocumentMessage) feedFixture.sessionFactory.messages.get(2)).getDocumentId().toString());
+
+        assertTrue(feedFixture.outputStream.toString().contains("Messages sent to vespa"));
+    }
+
+    @Test
+    public void requireThatCreateIfNonExistentArgumentIsUsed() throws Exception {
+        FeedFixture f = new FeedFixture();
+        Arguments arguments = new Arguments("--file src/test/files/myfeed.xml --create-if-non-existent".split(" "), f.sessionFactory);
+        new VespaFeeder(arguments, f.typeManager).parseFiles(System.in, f.printStream);
+
+        assertEquals(3, f.sessionFactory.messages.size());
+        DocumentUpdate update = ((UpdateDocumentMessage) f.sessionFactory.messages.get(1)).getDocumentUpdate();
+        assertTrue(update.getCreateIfNonExistent());
+    }
+
+    @Test
+    public void feedMalformedJson() throws Exception {
+        exception.expect(VespaFeeder.FeedErrorException.class);
+        exception.expectMessage("JsonParseException");
+        feed("src/test/files/malformedfeed.json", false);
+    }
+
+    protected FeedFixture feed(String feed, boolean abortOnDataError) throws HelpShownException,
+            FileNotFoundException, Exception {
+        String abortOnDataErrorArgument = abortOnDataError ? "" : " --abortondataerror no";
+        FeedFixture feedFixture = new FeedFixture();
+        Arguments arguments = new Arguments(("--file "
+                + feed
+                + " --priority LOW_1" + abortOnDataErrorArgument).split(" "), feedFixture.sessionFactory);
+        new VespaFeeder(arguments, feedFixture.typeManager).parseFiles(System.in, feedFixture.printStream);
+        return feedFixture;
+    }
+}
diff --git a/vespaclient-java/src/test/java/com/yahoo/vespaget/CommandLineOptionsTest.java b/vespaclient-java/src/test/java/com/yahoo/vespaget/CommandLineOptionsTest.java
new file mode 100644
index 00000000000..3e707b04256
--- /dev/null
+++ b/vespaclient-java/src/test/java/com/yahoo/vespaget/CommandLineOptionsTest.java
@@ -0,0 +1,195 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespaget;
+
+import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+import java.io.*;
+import java.util.Iterator;
+
+import static org.junit.Assert.*;
+
+/**
+ * Test class for {@link CommandLineOptions}
+ *
+ * @author bjorncs
+ * @since 5.26
+ */
+public class CommandLineOptionsTest {
+
+    private final InputStream emptyStream = new InputStream() {
+
+        @Override
+        public int read() throws IOException {
+            return -1;
+        }
+    };
+
+    @Rule
+    public final ExpectedException exception = ExpectedException.none();
+
+    private ClientParameters getParsedOptions(InputStream in, String... args) {
+        CommandLineOptions options = new CommandLineOptions(in);
+        return options.parseCommandLineArguments(args);
+    }
+
+    private ClientParameters getParsedOptions(String... args) {
+        return getParsedOptions(emptyStream, args);
+    }
+
+    @Test
+    public void testDefaultOptions() {
+        ClientParameters params = getParsedOptions();
+        assertFalse(params.help);
+        assertFalse(params.documentIds.hasNext());
+        assertFalse(params.printIdsOnly);
+        assertEquals("[all]", params.fieldSet);
+        assertEquals("default", params.route);
+        assertTrue(params.cluster.isEmpty());
+        assertEquals("client", params.configId);
+        assertFalse(params.showDocSize);
+        assertEquals(0, params.timeout, 0);
+        assertFalse(params.noRetry);
+        assertEquals(0, params.traceLevel);
+        assertEquals(DocumentProtocol.Priority.NORMAL_2, params.priority);
+        assertTrue(params.loadTypeName.isEmpty());
+    }
+
+    @Test
+    public void testValidOptions() {
+        ClientParameters params = getParsedOptions(
+                "--fieldset", "[fieldset]",
+                "--route", "dummyroute",
+                "--configid", "dummyconfig",
+                "--showdocsize",
+                "--timeout", "0.25",
+                "--noretry",
+                "--trace", "1",
+                "--priority", Integer.toString(DocumentProtocol.Priority.HIGH_3.getValue()),
+                "--loadtype", "dummyloadtype",
+                "id:1", "id:2"
+        );
+
+        assertEquals("[fieldset]", params.fieldSet);
+        assertEquals("dummyroute", params.route);
+        assertEquals("dummyconfig", params.configId);
+        assertTrue(params.showDocSize);
+        assertEquals(0.25, params.timeout, 0.0001);
+        assertTrue(params.noRetry);
+        assertEquals(1, params.traceLevel);
+        assertEquals(DocumentProtocol.Priority.HIGH_3, params.priority);
+        assertEquals("dummyloadtype", params.loadTypeName);
+
+        Iterator<String> documentsIds = params.documentIds;
+        assertEquals("id:1", documentsIds.next());
+        assertEquals("id:2", documentsIds.next());
+        assertFalse(documentsIds.hasNext());
+    }
+
+    @Test
+    public void testInvalidCombination1() {
+        exception.expect(IllegalArgumentException.class);
+        exception.expectMessage("Print ids and headers only options are mutually exclusive.");
+        getParsedOptions("--headersonly", "--printids");
+    }
+
+    @Test
+    public void testInvalidCombination2() {
+        exception.expect(IllegalArgumentException.class);
+        exception.expectMessage("Field set option can not be used in combination with print ids or headers only options.");
+        getParsedOptions("--headersonly", "--fieldset", "[header]");
+    }
+
+    @Test
+    public void testInvalidCombination3() {
+        exception.expect(IllegalArgumentException.class);
+        exception.expectMessage("Field set option can not be used in combination with print ids or headers only options.");
+        getParsedOptions("--printids", "--fieldset", "[header]");
+    }
+
+    @Test
+    public void testInvalidCombination4() {
+        exception.expect(IllegalArgumentException.class);
+        exception.expectMessage("Cluster and route options are mutually exclusive.");
+        getParsedOptions("--route", "dummyroute", "--cluster", "dummycluster");
+    }
+
+    @Test
+    public void testInvalidPriority() {
+        exception.expect(IllegalArgumentException.class);
+        exception.expectMessage("Invalid priority: 16");
+        getParsedOptions("--priority", "16");
+    }
+
+    @Test
+    public void testInvalidTraceLevel1() {
+        exception.expect(IllegalArgumentException.class);
+        exception.expectMessage("Invalid tracelevel: -1");
+        getParsedOptions("--trace", "-1");
+    }
+
+    @Test
+    public void testInvalidTraceLevel2() {
+        exception.expect(IllegalArgumentException.class);
+        exception.expectMessage("Invalid tracelevel: 10");
+        getParsedOptions("--trace", "10");
+    }
+
+    @Test
+    public void testPrintids() {
+        ClientParameters params = getParsedOptions("--printids");
+        assertEquals("[id]", params.fieldSet);
+    }
+
+    @Test
+    public void testHeadersOnly() {
+        ClientParameters params = getParsedOptions("--headersonly");
+        assertEquals("[header]", params.fieldSet);
+    }
+
+    @Test
+    public void testCluster() {
+        ClientParameters params = getParsedOptions("--cluster", "dummycluster");
+        assertEquals("dummycluster", params.cluster);
+        assertTrue(params.route.isEmpty());
+    }
+
+    @Test
+    public void testHelp() {
+        ClientParameters params = getParsedOptions("--help");
+        assertTrue(params.help);
+    }
+
+    @Test
+    public void testDocumentIdsFromInputStream() throws UnsupportedEncodingException {
+        InputStream in = new ByteArrayInputStream("id:1 id:2 id:3".getBytes("UTF-8"));
+        ClientParameters params = getParsedOptions(in, "");
+
+        Iterator<String> documentsIds = params.documentIds;
+        assertEquals("id:1", documentsIds.next());
+        assertEquals("id:2", documentsIds.next());
+        assertEquals("id:3", documentsIds.next());
+        assertFalse(documentsIds.hasNext());
+    }
+
+    @Test
+    public void testPrintHelp() {
+        ByteArrayOutputStream outContent = new ByteArrayOutputStream();
+        PrintStream oldOut = System.out;
+        System.setOut(new PrintStream(outContent));
+        try {
+            CommandLineOptions options = new CommandLineOptions(emptyStream);
+            options.printHelp();
+
+            String output = outContent.toString();
+            assertTrue(output.contains("vespaget <options> [documentid...]"));
+            assertTrue(output.contains("Fetch a document from a Vespa Content cluster."));
+        } finally {
+            System.setOut(oldOut);
+            outContent.reset();
+
+        }
+    }
+}
diff --git a/vespaclient-java/src/test/java/com/yahoo/vespaget/DocumentRetrieverTest.java b/vespaclient-java/src/test/java/com/yahoo/vespaget/DocumentRetrieverTest.java
new file mode 100644
index 00000000000..c3d3fcc71e9
--- /dev/null
+++ b/vespaclient-java/src/test/java/com/yahoo/vespaget/DocumentRetrieverTest.java
@@ -0,0 +1,376 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespaget;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonParseException;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.yahoo.document.DataType;
+import com.yahoo.document.Document;
+import com.yahoo.document.DocumentId;
+import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess;
+import com.yahoo.documentapi.messagebus.MessageBusParams;
+import com.yahoo.documentapi.messagebus.MessageBusSyncSession;
+import com.yahoo.documentapi.messagebus.loadtypes.LoadType;
+import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet;
+import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol;
+import com.yahoo.documentapi.messagebus.protocol.GetDocumentMessage;
+import com.yahoo.documentapi.messagebus.protocol.GetDocumentReply;
+import com.yahoo.messagebus.Error;
+import com.yahoo.messagebus.Reply;
+import com.yahoo.vespaclient.ClusterDef;
+import com.yahoo.vespaclient.ClusterList;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.mockito.ArgumentMatcher;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.argThat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+/**
+ * Test class for {@link DocumentRetriever}
+ *
+ * @author bjorncs
+ */
+public class DocumentRetrieverTest {
+
+    public static final String DOC_ID_1 = "id:storage_test:document::1";
+    public static final String DOC_ID_2 = "id:storage_test:document::2";
+    public static final String DOC_ID_3 = "id:storage_test:document::3";
+
+    private final ByteArrayOutputStream outContent = new ByteArrayOutputStream();
+    private final ByteArrayOutputStream errContent = new ByteArrayOutputStream();
+
+    private DocumentAccessFactory mockedFactory;
+    private MessageBusDocumentAccess mockedDocumentAccess;
+    private MessageBusSyncSession mockedSession;
+    private PrintStream oldOut;
+    private PrintStream oldErr;
+
+    @Rule
+    public final ExpectedException exception = ExpectedException.none();
+
+    @Before
+    public void setUpStreams() {
+        oldOut = System.out;
+        oldErr = System.err;
+        System.setOut(new PrintStream(outContent));
+        System.setErr(new PrintStream(errContent));
+    }
+
+    @Before
+    public void prepareMessageBusMocks() {
+        this.mockedFactory = mock(DocumentAccessFactory.class);
+        this.mockedDocumentAccess = mock(MessageBusDocumentAccess.class);
+        this.mockedSession = mock(MessageBusSyncSession.class);
+        when(mockedFactory.createDocumentAccess(any())).thenReturn(mockedDocumentAccess);
+        when(mockedDocumentAccess.createSyncSession(any())).thenReturn(mockedSession);
+    }
+
+    @After
+    public void cleanUpStreams() {
+        System.setOut(oldOut);
+        System.setErr(oldErr);
+        outContent.reset();
+        errContent.reset();
+    }
+
+    private static ClientParameters.Builder createParameters() {
+        return new ClientParameters.Builder()
+                .setPriority(DocumentProtocol.Priority.NORMAL_2)
+                .setCluster("")
+                .setRoute("default")
+                .setConfigId("client")
+                .setFieldSet("[all]")
+                .setPrintIdsOnly(false)
+                .setHelp(false)
+                .setShowDocSize(false)
+                .setLoadTypeName("")
+                .setNoRetry(false)
+                .setTraceLevel(0)
+                .setTimeout(0)
+                .setDocumentIds(Collections.emptyIterator());
+    }
+
+    private static Iterator<String> asIterator(String... docIds) {
+        return Arrays.asList(docIds).iterator();
+    }
+
+    private static Reply createDocumentReply(String docId) {
+        return new GetDocumentReply(new Document(DataType.DOCUMENT, new DocumentId(docId)));
+    }
+
+    private void assertContainsDocument(String documentId) {
+        assertTrue(outContent.toString().contains(String.format(
+                "<document documenttype=\"document\" documentid=\"%s\"/>", documentId)));
+    }
+
+    private DocumentRetriever createDocumentRetriever(ClientParameters params) {
+        return createDocumentRetriever(params, new ClusterList());
+    }
+
+    private DocumentRetriever createDocumentRetriever(ClientParameters params, ClusterList clusterList) {
+        return new DocumentRetriever(
+                clusterList,
+                mockedFactory,
+                new LoadTypeSet(),
+                params);
+    }
+
+    @Test
+    public void testSendSingleMessage() throws DocumentRetrieverException {
+        ClientParameters params = createParameters()
+                .setDocumentIds(asIterator(DOC_ID_1))
+                .setPriority(DocumentProtocol.Priority.HIGH_1)
+                .setNoRetry(true)
+                .setLoadTypeName("loadtype")
+                .build();
+
+        when(mockedSession.syncSend(any())).thenReturn(createDocumentReply(DOC_ID_1));
+
+        LoadTypeSet loadTypeSet = new LoadTypeSet();
+        loadTypeSet.addLoadType(1, "loadtype", DocumentProtocol.Priority.HIGH_1);
+        DocumentRetriever documentRetriever = new DocumentRetriever(
+                new ClusterList(),
+                mockedFactory,
+                loadTypeSet,
+                params);
+        documentRetriever.retrieveDocuments();
+
+        verify(mockedSession, times(1)).syncSend(argThat(new ArgumentMatcher<GetDocumentMessage>() {
+            @Override
+            public boolean matches(Object o) {
+                GetDocumentMessage msg = (GetDocumentMessage) o;
+                return msg.getPriority().equals(DocumentProtocol.Priority.HIGH_1) &&
+                        !msg.getRetryEnabled() &&
+                        msg.getLoadType().equals(new LoadType(1, "loadtype", DocumentProtocol.Priority.HIGH_1));
+            }
+        }));
+        assertContainsDocument(DOC_ID_1);
+    }
+
+    @Test
+    public void testMultipleMessages() throws DocumentRetrieverException {
+        ClientParameters params = createParameters()
+                .setDocumentIds(asIterator(DOC_ID_1, DOC_ID_2, DOC_ID_3))
+                .build();
+
+        when(mockedSession.syncSend(any())).thenReturn(
+                createDocumentReply(DOC_ID_1),
+                createDocumentReply(DOC_ID_2),
+                createDocumentReply(DOC_ID_3));
+
+        DocumentRetriever documentRetriever = createDocumentRetriever(params);
+        documentRetriever.retrieveDocuments();
+
+        verify(mockedSession, times(3)).syncSend(any());
+        assertContainsDocument(DOC_ID_1);
+        assertContainsDocument(DOC_ID_2);
+        assertContainsDocument(DOC_ID_3);
+    }
+
+    @Test
+    public void testJsonOutput() throws DocumentRetrieverException, JsonParseException, IOException {
+        ClientParameters params = createParameters()
+                .setDocumentIds(asIterator(DOC_ID_1, DOC_ID_2, DOC_ID_3))
+                .setJsonOutput(true)
+                .build();
+
+        when(mockedSession.syncSend(any())).thenReturn(
+                createDocumentReply(DOC_ID_1),
+                createDocumentReply(DOC_ID_2),
+                createDocumentReply(DOC_ID_3));
+
+        DocumentRetriever documentRetriever = createDocumentRetriever(params);
+        documentRetriever.retrieveDocuments();
+
+        verify(mockedSession, times(3)).syncSend(any());
+        ObjectMapper m = new ObjectMapper();
+        @SuppressWarnings("unchecked")
+        List<Map<String, Object>> feed = m.readValue(outContent.toByteArray(), List.class);
+        assertEquals(DOC_ID_1, feed.get(0).get("id"));
+        assertEquals(DOC_ID_2, feed.get(1).get("id"));
+        assertEquals(DOC_ID_3, feed.get(2).get("id"));
+    }
+
+    @Test
+    public void testShutdownHook() throws DocumentRetrieverException {
+        ClientParameters params = createParameters()
+                .setDocumentIds(asIterator(DOC_ID_1))
+                .build();
+
+        when(mockedSession.syncSend(any())).thenReturn(createDocumentReply(DOC_ID_1));
+
+        DocumentRetriever documentRetriever = createDocumentRetriever(params);
+        documentRetriever.retrieveDocuments();
+        documentRetriever.shutdown();
+
+        verify(mockedSession, times(1)).destroy();
+        verify(mockedDocumentAccess, times(1)).shutdown();
+    }
+
+    @Test
+    public void testInvalidLoadType() throws DocumentRetrieverException {
+        exception.expect(DocumentRetrieverException.class);
+        exception.expectMessage("Loadtype with name 'undefinedloadtype' does not exist.\n");
+
+        ClientParameters params = createParameters()
+                .setLoadTypeName("undefinedloadtype")
+                .build();
+
+        DocumentRetriever documentRetriever = createDocumentRetriever(params);
+        documentRetriever.retrieveDocuments();
+    }
+
+    @Test
+    public void testClusterLookup() throws DocumentRetrieverException {
+        final String cluster = "storage", configId = "content/cluster.foo/storage",
+                expectedRoute = "[Storage:cluster=storage;clusterconfigid=content/cluster.foo/storage]";
+
+        ClientParameters params = createParameters()
+                .setCluster(cluster)
+                .build();
+
+        ClusterList clusterList = new ClusterList(Collections.singletonList(new ClusterDef(cluster, configId)));
+
+        DocumentRetriever documentRetriever = createDocumentRetriever(params, clusterList);
+        documentRetriever.retrieveDocuments();
+
+        verify(mockedFactory).createDocumentAccess(argThat(new ArgumentMatcher<MessageBusParams>() {
+            @Override
+            public boolean matches(Object o) {
+                return ((MessageBusParams) o).getRoute().equals(expectedRoute);
+            }
+        }));
+    }
+
+    @Test
+    public void testInvalidClusterName() throws DocumentRetrieverException {
+        exception.expect(DocumentRetrieverException.class);
+        exception.expectMessage("The Vespa cluster contains the content clusters storage, not invalidclustername. Please select a valid vespa cluster.");
+
+        ClientParameters params = createParameters()
+                .setCluster("invalidclustername")
+                .build();
+
+        ClusterList clusterList = new ClusterList(Collections.singletonList(new ClusterDef("storage", "content/cluster.foo/storage")));
+
+        DocumentRetriever documentRetriever = createDocumentRetriever(params, clusterList);
+        documentRetriever.retrieveDocuments();
+    }
+
+    @Test
+    public void testEmtpyClusterList() throws DocumentRetrieverException {
+        exception.expect(DocumentRetrieverException.class);
+        exception.expectMessage("The Vespa cluster does not have any content clusters declared.");
+
+        ClientParameters params = createParameters()
+                .setCluster("invalidclustername")
+                .build();
+
+        DocumentRetriever documentRetriever = createDocumentRetriever(params);
+        documentRetriever.retrieveDocuments();
+    }
+
+    @Test
+    public void testHandlingErrorFromMessageBus() throws DocumentRetrieverException {
+        ClientParameters params = createParameters()
+                .setDocumentIds(asIterator(DOC_ID_1))
+                .build();
+
+        Reply r = new GetDocumentReply(null);
+        r.addError(new Error(0, "Error message"));
+        when(mockedSession.syncSend(any())).thenReturn(r);
+
+        DocumentRetriever documentRetriever = createDocumentRetriever(params);
+        documentRetriever.retrieveDocuments();
+
+        assertTrue(errContent.toString().contains("Request failed"));
+    }
+
+    @Test
+    public void testShowDocSize() throws DocumentRetrieverException {
+        ClientParameters params = createParameters()
+                .setDocumentIds(asIterator(DOC_ID_1))
+                .setShowDocSize(true)
+                .build();
+
+        Document document = new Document(DataType.DOCUMENT, new DocumentId(DOC_ID_1));
+        when(mockedSession.syncSend(any())).thenReturn(new GetDocumentReply(document));
+
+        DocumentRetriever documentRetriever = createDocumentRetriever(params);
+        documentRetriever.retrieveDocuments();
+
+        assertTrue(outContent.toString().contains(String.format("Document size: %d bytes", document.getSerializedSize())));
+    }
+
+    @Test
+    public void testPrintIdOnly() throws DocumentRetrieverException {
+        ClientParameters params = createParameters()
+                .setDocumentIds(asIterator(DOC_ID_1))
+                .setPrintIdsOnly(true)
+                .build();
+
+        when(mockedSession.syncSend(any())).thenReturn(createDocumentReply(DOC_ID_1));
+
+        DocumentRetriever documentRetriever = createDocumentRetriever(params);
+        documentRetriever.retrieveDocuments();
+
+        assertEquals(DOC_ID_1 + "\n", outContent.toString());
+    }
+
+    @Test
+    public void testDocumentNotFound() throws DocumentRetrieverException {
+        ClientParameters params = createParameters()
+                .setDocumentIds(asIterator(DOC_ID_1))
+                .setPrintIdsOnly(true)
+                .build();
+
+        when(mockedSession.syncSend(any())).thenReturn(new GetDocumentReply(null));
+
+        DocumentRetriever documentRetriever = createDocumentRetriever(params);
+        documentRetriever.retrieveDocuments();
+
+        verify(mockedSession, times(1)).syncSend(any());
+        assertEquals(outContent.toString(), "Document not found.\n");
+    }
+
+    @Test
+    public void testTrace() throws DocumentRetrieverException {
+        final int traceLevel = 9;
+        ClientParameters params = createParameters()
+                .setDocumentIds(asIterator(DOC_ID_1))
+                .setTraceLevel(traceLevel)
+                .build();
+
+        GetDocumentReply reply = new GetDocumentReply(new Document(DataType.DOCUMENT, new DocumentId(DOC_ID_1)));
+        reply.getTrace().getRoot().addChild("childnode");
+        when(mockedSession.syncSend(any())).thenReturn(reply);
+
+        DocumentRetriever documentRetriever = createDocumentRetriever(params);
+        documentRetriever.retrieveDocuments();
+
+        verify(mockedSession, times(1)).setTraceLevel(traceLevel);
+        assertTrue(outContent.toString().contains("<trace>"));
+    }
+
+}
diff --git a/vespaclient-java/src/test/java/com/yahoo/vespastat/BucketStatsPrinterTest.java b/vespaclient-java/src/test/java/com/yahoo/vespastat/BucketStatsPrinterTest.java
new file mode 100644
index 00000000000..2df8b2c4751
--- /dev/null
+++ b/vespaclient-java/src/test/java/com/yahoo/vespastat/BucketStatsPrinterTest.java
@@ -0,0 +1,87 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespastat;
+
+import com.yahoo.document.BucketId;
+import com.yahoo.documentapi.messagebus.protocol.GetBucketListReply;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.io.PrintStream;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class BucketStatsPrinterTest {
+
+    private BucketStatsRetriever retriever;
+    private final ByteArrayOutputStream out = new ByteArrayOutputStream();
+
+    @Before
+    public void mockBucketStatsRetriever() throws BucketStatsException {
+        retriever = mock(BucketStatsRetriever.class);
+        when(retriever.getBucketIdForType(any(), any())).thenReturn(new BucketId(0x42));
+        when(retriever.retrieveBucketList(any())).thenReturn(Collections.emptyList());
+        when(retriever.retrieveBucketStats(any(), any(), any())).thenReturn("");
+    }
+
+    @After
+    public void resetOutputMock() {
+        out.reset();
+    }
+
+    private String getOutputString() {
+        String content = out.toString();
+        out.reset();
+        return content;
+    }
+
+    private String retreiveAndPrintBucketStats(ClientParameters.SelectionType type, String id, boolean dumpData) throws BucketStatsException {
+        BucketStatsPrinter printer = new BucketStatsPrinter(retriever, new PrintStream(out));
+        printer.retrieveAndPrintBucketStats(type, id, dumpData);
+        return getOutputString();
+    }
+
+    @Test
+    public void testShouldPrintBucketIdForUserAndGroup() throws BucketStatsException {
+        String output = retreiveAndPrintBucketStats(ClientParameters.SelectionType.USER, "1234", false);
+        assertTrue(output.contains("Generated 32-bit bucket id"));
+
+        output = retreiveAndPrintBucketStats(ClientParameters.SelectionType.GROUP, "mygroup", false);
+        assertTrue(output.contains("Generated 32-bit bucket id"));
+    }
+
+    @Test
+    public void testShouldPrintWarningIfBucketListEmpty() throws BucketStatsException {
+        String output = retreiveAndPrintBucketStats(ClientParameters.SelectionType.USER, "1234", false);
+        assertTrue(output.contains("No actual files were stored for this bucket"));
+    }
+
+    @Test
+    public void testShouldPrintBucketList() throws BucketStatsException {
+        List<GetBucketListReply.BucketInfo> bucketList = new ArrayList<>();
+        String dummyInfoString = "dummyinformation";
+        bucketList.add(new GetBucketListReply.BucketInfo(new BucketId(0), dummyInfoString));
+        when(retriever.retrieveBucketList(any())).thenReturn(bucketList);
+
+        String output = retreiveAndPrintBucketStats(ClientParameters.SelectionType.USER, "1234", false);
+        assertTrue(output.contains(dummyInfoString));
+    }
+
+    @Test
+    public void testShouldPrintBucketStats() throws BucketStatsException {
+        String dummyBucketStats = "dummystats";
+        GetBucketListReply.BucketInfo bucketInfo = new GetBucketListReply.BucketInfo(new BucketId(0), "dummy");
+        when(retriever.retrieveBucketList(any())).thenReturn(Collections.singletonList(bucketInfo));
+        when(retriever.retrieveBucketStats(any(), any(), any())).thenReturn(dummyBucketStats);
+
+        String output = retreiveAndPrintBucketStats(ClientParameters.SelectionType.USER, "1234", true);
+        assertTrue(output.contains(dummyBucketStats));
+    }
+}
diff --git a/vespaclient-java/src/test/java/com/yahoo/vespastat/BucketStatsRetrieverTest.java b/vespaclient-java/src/test/java/com/yahoo/vespastat/BucketStatsRetrieverTest.java
new file mode 100644
index 00000000000..38a79aa7b5d
--- /dev/null
+++ b/vespaclient-java/src/test/java/com/yahoo/vespastat/BucketStatsRetrieverTest.java
@@ -0,0 +1,141 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespastat;
+
+import com.yahoo.document.BucketId;
+import com.yahoo.document.BucketIdFactory;
+import com.yahoo.document.DocumentId;
+import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess;
+import com.yahoo.documentapi.messagebus.MessageBusSyncSession;
+import com.yahoo.documentapi.messagebus.protocol.GetBucketListReply;
+import com.yahoo.documentapi.messagebus.protocol.StatBucketReply;
+import com.yahoo.messagebus.Error;
+import com.yahoo.messagebus.Message;
+import com.yahoo.messagebus.routing.Route;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.ArgumentMatcher;
+
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.argThat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+public class BucketStatsRetrieverTest {
+    private final BucketIdFactory bucketIdFactory = new BucketIdFactory();
+
+    private DocumentAccessFactory mockedFactory;
+    private MessageBusDocumentAccess mockedDocumentAccess;
+    private MessageBusSyncSession mockedSession;
+
+
+    @Before
+    public void prepareMessageBusMocks() {
+        this.mockedFactory = mock(DocumentAccessFactory.class);
+        this.mockedDocumentAccess = mock(MessageBusDocumentAccess.class);
+        this.mockedSession = mock(MessageBusSyncSession.class);
+        when(mockedFactory.createDocumentAccess()).thenReturn(mockedDocumentAccess);
+        when(mockedDocumentAccess.createSyncSession(any())).thenReturn(mockedSession);
+    }
+
+    @Test
+    public void testGetBucketId() throws BucketStatsException {
+        BucketStatsRetriever retriever = createRetriever();
+
+        assertEquals("BucketId(0x80000000000004d2)",
+                retriever.getBucketIdForType(ClientParameters.SelectionType.USER, "1234").toString());
+        assertEquals("BucketId(0x800000003a7455d7)",
+                retriever.getBucketIdForType(ClientParameters.SelectionType.GROUP, "mygroup").toString());
+        assertEquals("BucketId(0x800000003a7455d7)",
+                retriever.getBucketIdForType(ClientParameters.SelectionType.BUCKET, "0x800000003a7455d7").toString());
+        assertEquals("BucketId(0xeb018ac5e5732db3)",
+                retriever.getBucketIdForType(ClientParameters.SelectionType.DOCUMENT, "id:ns:type::another").toString());
+        assertEquals("BucketId(0xeadd5fe811a2012c)",
+                retriever.getBucketIdForType(ClientParameters.SelectionType.GID, "0x2c01a21163cb7d0ce85fddd6").toString());
+    }
+
+    @Test
+    public void testRetrieveBucketList() throws BucketStatsException {
+        String bucketInfo = "I like turtles!";
+        BucketId bucketId = bucketIdFactory.getBucketId(new DocumentId("id:ns:type::another"));
+
+        GetBucketListReply reply = new GetBucketListReply();
+        reply.getBuckets().add(new GetBucketListReply.BucketInfo(bucketId, bucketInfo));
+        when(mockedSession.syncSend(any())).thenReturn(reply);
+
+        List<GetBucketListReply.BucketInfo> bucketList = createRetriever().retrieveBucketList(bucketId);
+
+        verify(mockedSession, times(1)).syncSend(any());
+        assertEquals(1, bucketList.size());
+        assertEquals(bucketInfo, bucketList.get(0).getBucketInformation());
+    }
+
+    @Test
+    public void testRetrieveBucketStats() throws BucketStatsException {
+        String docId = "id:ns:type::another";
+        String bucketInfo = "I like turtles!";
+        BucketId bucketId = bucketIdFactory.getBucketId(new DocumentId(docId));
+
+        StatBucketReply reply = new StatBucketReply();
+        reply.setResults(bucketInfo);
+        when(mockedSession.syncSend(any())).thenReturn(reply);
+        String result = createRetriever().retrieveBucketStats(ClientParameters.SelectionType.DOCUMENT, docId, bucketId);
+
+        verify(mockedSession, times(1)).syncSend(any());
+        assertEquals(bucketInfo, result);
+    }
+
+    @Test
+    public void testShutdownHook() {
+        class MockShutdownRegistrar implements BucketStatsRetriever.ShutdownHookRegistrar {
+            public Runnable shutdownRunnable;
+            @Override
+            public void registerShutdownHook(Runnable runnable) {
+                shutdownRunnable = runnable;
+            }
+        }
+        MockShutdownRegistrar registrar = new MockShutdownRegistrar();
+        new BucketStatsRetriever(mockedFactory, "default", registrar);
+        registrar.shutdownRunnable.run();
+
+        verify(mockedSession, times(1)).destroy();
+        verify(mockedDocumentAccess, times(1)).shutdown();
+    }
+
+    @Test(expected = BucketStatsException.class)
+    public void testShouldFailOnReplyError() throws BucketStatsException {
+        GetBucketListReply reply = new GetBucketListReply();
+        reply.addError(new Error(0, "errormsg"));
+        when(mockedSession.syncSend(any())).thenReturn(reply);
+
+        createRetriever().retrieveBucketList(new BucketId(1));
+    }
+
+    @Test
+    public void testRoute() throws BucketStatsException {
+        String route = "default";
+        BucketId bucketId = bucketIdFactory.getBucketId(new DocumentId("id:ns:type::another"));
+        GetBucketListReply reply = new GetBucketListReply();
+        reply.getBuckets().add(new GetBucketListReply.BucketInfo(bucketId, "I like turtles!"));
+        when(mockedSession.syncSend(any())).thenReturn(reply);
+
+        BucketStatsRetriever retriever = new BucketStatsRetriever(mockedFactory, route, t -> {});
+        retriever.retrieveBucketList(new BucketId(0));
+
+        verify(mockedSession).syncSend(argThat(new ArgumentMatcher<Message>() {
+            @Override
+            public boolean matches(Object o) {
+                return ((Message) o).getRoute().equals(Route.parse(route));
+            }
+        }));
+    }
+
+    private BucketStatsRetriever createRetriever() {
+        return new BucketStatsRetriever(mockedFactory, "default", t -> {});
+    }
+
+}
diff --git a/vespaclient-java/src/test/java/com/yahoo/vespastat/CommandLineOptionsTest.java b/vespaclient-java/src/test/java/com/yahoo/vespastat/CommandLineOptionsTest.java
new file mode 100644
index 00000000000..e90c47e3150
--- /dev/null
+++ b/vespaclient-java/src/test/java/com/yahoo/vespastat/CommandLineOptionsTest.java
@@ -0,0 +1,78 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespastat;
+
+import org.junit.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.io.PrintStream;
+
+import static org.junit.Assert.*;
+
+public class CommandLineOptionsTest {
+
+    private ClientParameters getParsedOptions(String... args) {
+        CommandLineOptions parser = new CommandLineOptions();
+        return parser.parseCommandLineArguments(args);
+    }
+
+    @Test
+    public void testHelp() {
+        assertTrue(getParsedOptions("--help").help);
+    }
+
+    @Test
+    public void testMultipleOptions() {
+        ClientParameters params = getParsedOptions("--dump", "--route", "dummyroute", "--user", "userid");
+        assertTrue(params.dumpData);
+        assertEquals("dummyroute", params.route);
+        assertEquals(ClientParameters.SelectionType.USER, params.selectionType);
+        assertEquals("userid", params.id);
+    }
+
+    @Test
+    public void testSelectionTypes() {
+        assertEquals(ClientParameters.SelectionType.USER, getParsedOptions("--user", "id").selectionType);
+        assertEquals(ClientParameters.SelectionType.DOCUMENT, getParsedOptions("--document", "id").selectionType);
+        assertEquals(ClientParameters.SelectionType.BUCKET, getParsedOptions("--bucket", "id").selectionType);
+        assertEquals(ClientParameters.SelectionType.GROUP, getParsedOptions("--group", "id").selectionType);
+        assertEquals(ClientParameters.SelectionType.GID, getParsedOptions("--gid", "id").selectionType);
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testMissingSelectionType() {
+       getParsedOptions();
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testFailOnMultipleDumpTypes() {
+        getParsedOptions("--user", "id", "--document", "id", "--group", "id", "--gid", "id");
+    }
+
+    @Test
+    public void testForceDumpOnDocumentOrGid() {
+        assertTrue(getParsedOptions("--document", "docid").dumpData);
+        assertTrue(getParsedOptions("--gid", "gid").dumpData);
+    }
+
+    @Test
+    public void testPrintHelp() {
+        ByteArrayOutputStream outContent = new ByteArrayOutputStream();
+        PrintStream oldOut = System.out;
+        System.setOut(new PrintStream(outContent));
+        try {
+            CommandLineOptions options = new CommandLineOptions();
+            options.printHelp();
+            String output = outContent.toString();
+            assertTrue(output.contains("vdsstat [options]"));
+        } finally {
+            System.setOut(oldOut);
+            outContent.reset();
+        }
+    }
+
+    @Test
+    public void testDefaultRoute() {
+        assertEquals("default", getParsedOptions("--user", "dummyuser").route);
+    }
+
+}
diff --git a/vespaclient-java/src/test/java/com/yahoo/vespavisit/VdsVisitTargetTestCase.java b/vespaclient-java/src/test/java/com/yahoo/vespavisit/VdsVisitTargetTestCase.java
new file mode 100644
index 00000000000..4eec05f7bc7
--- /dev/null
+++ b/vespaclient-java/src/test/java/com/yahoo/vespavisit/VdsVisitTargetTestCase.java
@@ -0,0 +1,56 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespavisit;
+
+import org.junit.Test;
+import static org.junit.Assert.*;
+
+public class VdsVisitTargetTestCase {
+
+    @Test
+    public void testParametersSlobrok() throws Exception {
+        VdsVisitTarget target = new VdsVisitTarget();
+        target.parseArguments(new String[]{
+                "--bindtoslobrok", "myname",
+                "--processtime", "34",
+                "--visithandler", "Foo",
+                "--visitoptions", "foo bar zoo",
+                "-i",
+                "-v"
+        });
+
+        assertEquals("myname", target.getSlobrokAddress());
+        assertEquals(34, target.getProcessTime());
+        assertEquals("Foo", target.getHandlerClassName());
+        assertEquals(3, target.getHandlerArgs().length);
+        assertEquals("foo", target.getHandlerArgs()[0]);
+        assertEquals("bar", target.getHandlerArgs()[1]);
+        assertEquals("zoo", target.getHandlerArgs()[2]);
+        assertTrue(target.isVerbose());
+        assertTrue(target.isPrintIds());
+    }
+
+    @Test
+    public void testParametersPort() throws Exception {
+        VdsVisitTarget target = new VdsVisitTarget();
+        target.parseArguments("--bindtosocket 1234".split(" "));
+        assertEquals(1234, target.getPort());
+        assertEquals(null, target.getSlobrokAddress());
+    }
+
+    public void assertException(String params) {
+        try {
+            VdsVisitTarget target = new VdsVisitTarget();
+            target.parseArguments(params.split(" "));
+            assertTrue(false);
+        } catch (Exception e) {
+
+        }
+    }
+
+    @Test
+    public void testPortAndSlobrok() {
+        assertException("--bindtoslobrok foo --bindtosocket 1234");
+        assertException("--bindtoport foo");
+    }
+
+}
diff --git a/vespaclient-java/src/test/java/com/yahoo/vespavisit/VdsVisitTestCase.java b/vespaclient-java/src/test/java/com/yahoo/vespavisit/VdsVisitTestCase.java
new file mode 100644
index 00000000000..49060a5715f
--- /dev/null
+++ b/vespaclient-java/src/test/java/com/yahoo/vespavisit/VdsVisitTestCase.java
@@ -0,0 +1,475 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespavisit;
+
+import com.yahoo.document.select.OrderingSpecification;
+import com.yahoo.document.select.parser.ParseException;
+import com.yahoo.documentapi.*;
+import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol;
+import com.yahoo.messagebus.StaticThrottlePolicy;
+import com.yahoo.messagebus.Trace;
+import com.yahoo.vespaclient.ClusterDef;
+import com.yahoo.vespaclient.ClusterList;
+import org.apache.commons.cli.Options;
+import org.junit.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.io.PrintStream;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import static org.junit.Assert.*;
+
+public class VdsVisitTestCase {
+
+    private VdsVisit.ArgumentParser createMockArgumentParser() {
+        Options opts = VdsVisit.createOptions();
+        return new VdsVisit.ArgumentParser(opts);
+    }
+
+    @Test
+    public void testCommandLineShortOptions() throws Exception {
+        // short options testing (for options that do not collide with each other)
+        String[] args = new String[] {
+                "-d", "foo.remote",
+                "-s", "'id.user=1234'",
+                "-f", "5678",
+                "-t", "9012",
+                "-l", "foodoc.bar,foodoc.baz",
+                "-m", "6000",
+                "-b", "5",
+                "-p", "foo-progress.txt",
+                "-u", "123456789",
+                "-c", "kittens",
+                "-r",
+                "-v"
+        };
+        VdsVisit.ArgumentParser parser = createMockArgumentParser();
+        VdsVisit.VdsVisitParameters allParams = parser.parse(args);
+        assertNotNull(allParams);
+
+        VisitorParameters params = allParams.getVisitorParameters();
+        assertNotNull(params);
+        assertEquals("foo.remote", params.getRemoteDataHandler());
+        assertEquals("'id.user=1234'", params.getDocumentSelection());
+        assertEquals(5678, params.getFromTimestamp());
+        assertEquals(9012, params.getToTimestamp());
+        assertEquals("foodoc.bar,foodoc.baz", params.getFieldSet());
+        assertEquals(6000, params.getMaxPending());
+        assertEquals(5, params.getMaxBucketsPerVisitor());
+        assertEquals("foo-progress.txt", params.getResumeFileName());
+        assertEquals(123456789, params.getTimeoutMs());
+        assertEquals(7 * 24 * 60 * 60 * 1000, allParams.getFullTimeout());
+        assertEquals("kittens", allParams.getCluster());
+        assertTrue(allParams.isVerbose());
+    }
+
+    /**
+     * Test the parameters that could not be used in conjunction with
+     * those in the first parameter test.
+     * @throws Exception
+     */
+    @Test
+    public void testCommandLineShortOptions2() throws Exception {
+        // Short options testing (for options that do not collide with each other)
+        String[] args = new String[] {
+                "-o", "654321",
+                "-e"
+        };
+        VdsVisit.ArgumentParser parser = createMockArgumentParser();
+        VdsVisit.VdsVisitParameters allParams = parser.parse(args);
+        assertNotNull(allParams);
+
+        VisitorParameters params = allParams.getVisitorParameters();
+        assertNotNull(params);
+        assertEquals(654321, allParams.getFullTimeout());
+        assertEquals(654321, params.getTimeoutMs());
+        assertEquals("[header]", params.getFieldSet());
+    }
+
+    @Test
+    public void testCommandLineShortOptionsPrintIdsOnly() throws Exception {
+        // Short options testing (for options that do not collide with each other)
+        String[] args = new String[] {
+                "-i"
+        };
+        VdsVisit.ArgumentParser parser = createMockArgumentParser();
+        VdsVisit.VdsVisitParameters allParams = parser.parse(args);
+        assertNotNull(allParams);
+
+        VisitorParameters params = allParams.getVisitorParameters();
+        assertNotNull(params);
+        assertEquals("[id]", params.getFieldSet());
+        assertTrue(allParams.isPrintIdsOnly());
+    }
+
+    @Test
+    public void testCommandLineLongOptions() throws Exception {
+        // short options testing (for options that do not collide with each other)
+        String[] args = new String[] {
+                "--datahandler", "foo.remote",
+                "--selection", "'id.user=1234'",
+                "--from", "5678",
+                "--to", "9012",
+                "--fieldset", "foodoc.bar,foodoc.baz",
+                "--maxpending", "6000",
+                "--maxbuckets", "5",
+                "--progress", "foo-progress.txt",
+                "--maxpendingsuperbuckets", "3",
+                "--buckettimeout", "123456789",
+                "--cluster", "kittens",
+                "--visitinconsistentbuckets",
+                "--visitlibrary", "fnord",
+                "--libraryparam", "asdf", "rargh",
+                "--libraryparam", "pinkie", "pie",
+                "--processtime", "555",
+                "--maxhits", "1001",
+                "--maxtotalhits", "2002",
+                "--tracelevel", "8",
+                "--priority", "NORMAL_1",
+                "--ordering", "ascending",
+                "--skipbucketsonfatalerrors",
+                "--abortonclusterdown",
+                "--visitremoves"
+        };
+        VdsVisit.ArgumentParser parser = createMockArgumentParser();
+        VdsVisit.VdsVisitParameters allParams = parser.parse(args);
+        assertNotNull(allParams);
+
+        VisitorParameters params = allParams.getVisitorParameters();
+        assertNotNull(params);
+
+        assertEquals("foo.remote", params.getRemoteDataHandler());
+        assertEquals("'id.user=1234'", params.getDocumentSelection());
+        assertEquals(5678, params.getFromTimestamp());
+        assertEquals(9012, params.getToTimestamp());
+        assertEquals("foodoc.bar,foodoc.baz", params.getFieldSet());
+        assertEquals(6000, params.getMaxPending());
+        assertEquals(5, params.getMaxBucketsPerVisitor());
+        assertEquals("foo-progress.txt", params.getResumeFileName());
+        assertEquals(123456789, params.getTimeoutMs());
+        assertEquals(7 * 24 * 60 * 60 * 1000, allParams.getFullTimeout());
+        assertEquals("kittens", allParams.getCluster());
+
+        assertTrue(params.getThrottlePolicy() instanceof StaticThrottlePolicy);
+        assertEquals(3, ((StaticThrottlePolicy)params.getThrottlePolicy()).getMaxPendingCount());
+
+        assertTrue(params.visitInconsistentBuckets());
+        assertEquals("fnord", params.getVisitorLibrary());
+        // TODO: FIXME? multiple library params doesn't work
+        assertTrue(Arrays.equals("rargh".getBytes(), params.getLibraryParameters().get("asdf")));
+        //assertTrue(Arrays.equals("pie".getBytes(), params.getLibraryParameters().get("pinkie")));
+        assertEquals(555, allParams.getProcessTime());
+        assertEquals(1001, params.getMaxFirstPassHits());
+        assertEquals(2002, params.getMaxTotalHits());
+        assertEquals(8, params.getTraceLevel());
+        assertEquals(DocumentProtocol.Priority.NORMAL_1, params.getPriority());
+        assertEquals(OrderingSpecification.ASCENDING, params.getVisitorOrdering());
+        assertTrue(allParams.getAbortOnClusterDown());
+        assertTrue(params.visitRemoves());
+
+        ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
+        PrintStream printStream = new PrintStream(outputStream);
+        VdsVisit.verbosePrintParameters(allParams, printStream);
+        printStream.flush();
+        String nl = System.getProperty("line.separator"); // the joys of running tests on windows
+        assertEquals(
+                "Time out visitor after 123456789 ms." + nl +
+                        "Visiting documents matching: 'id.user=1234'" + nl +
+                        "Visiting in the inclusive timestamp range 5678 - 9012." + nl +
+                        "Visiting field set foodoc.bar,foodoc.baz." + nl +
+                        "Visiting inconsistent buckets." + nl +
+                        "Including remove entries." + nl +
+                        "Tracking progress in file: foo-progress.txt" + nl +
+                        "Let visitor have maximum 6000 replies pending on data handlers per storage node visitor." + nl +
+                        "Visit maximum 5 buckets per visitor." + nl +
+                        "Sending data to data handler at: foo.remote" + nl +
+                        "Using visitor library 'fnord'." + nl +
+                        "Adding the following library specific parameters:" + nl +
+                        "  asdf = rargh" + nl +
+                        "Visitor priority NORMAL_1" + nl +
+                        "Skip visiting super buckets with fatal errors." + nl,
+                outputStream.toString("utf-8"));
+
+        args = new String[] {
+                "--ordering", "descending"
+        };
+        allParams = parser.parse(args);
+        params = allParams.getVisitorParameters();
+        assertEquals(OrderingSpecification.DESCENDING, params.getVisitorOrdering());
+    }
+
+    private static String[] emptyArgList() { return new String[]{}; }
+
+    @Test
+    public void visitor_priority_is_low1_by_default() throws Exception {
+        VdsVisit.VdsVisitParameters allParams = createMockArgumentParser().parse(emptyArgList());
+
+        VisitorParameters params = allParams.getVisitorParameters();
+        assertEquals(DocumentProtocol.Priority.LOW_1, params.getPriority());
+    }
+
+    @Test
+    public void testBadPriorityValue() throws Exception {
+        String[] args = new String[] {
+                "--priority", "super_hyper_important"
+        };
+        VdsVisit.ArgumentParser parser = createMockArgumentParser();
+        try {
+            parser.parse(args);
+            fail("no exception thrown");
+        } catch (IllegalArgumentException e) {
+            assertTrue(e.getMessage().contains("Unknown priority name"));
+        }
+    }
+
+    @Test
+    public void testBadOrderingValue() throws Exception {
+        String[] args = new String[] {
+                "--ordering", "yonder"
+        };
+        VdsVisit.ArgumentParser parser = createMockArgumentParser();
+        try {
+            parser.parse(args);
+            fail("no exception thrown");
+        } catch (IllegalArgumentException e) {
+            assertTrue(e.getMessage().contains("Unknown ordering"));
+        }
+    }
+
+    @Test
+    public void testCommandLineShortOptionsInvokeHelp() throws Exception {
+        // Short options testing (for options that do not collide with each other)
+        String[] args = new String[] {
+                "-h"
+        };
+        VdsVisit.ArgumentParser parser = createMockArgumentParser();
+        VdsVisit.VdsVisitParameters allParams = parser.parse(args);
+        assertNull(allParams);
+    }
+
+    @Test
+    public void testAutoSelectClusterRoute() throws Exception {
+        List<ClusterDef> clusterDefs = new ArrayList<>();
+        clusterDefs.add(new ClusterDef("storage", "content/cluster.foo/storage"));
+        ClusterList clusterList = new ClusterList(clusterDefs);
+
+        String route = VdsVisit.resolveClusterRoute(clusterList, null);
+        assertEquals("[Storage:cluster=storage;clusterconfigid=content/cluster.foo/storage]", route);
+    }
+
+    @Test
+    public void testBadClusterName() throws Exception {
+        List<ClusterDef> clusterDefs = new ArrayList<>();
+        clusterDefs.add(new ClusterDef("storage", "content/cluster.foo/storage"));
+        ClusterList clusterList = new ClusterList(clusterDefs);
+        try {
+            VdsVisit.resolveClusterRoute(clusterList, "borkbork");
+        } catch (IllegalArgumentException e) {
+            assertTrue(e.getMessage().contains("Your vespa cluster contains the content clusters storage, not borkbork."));
+        }
+    }
+
+    @Test
+    public void testRequireClusterOptionIfMultipleClusters() {
+        List<ClusterDef> clusterDefs = new ArrayList<>();
+        clusterDefs.add(new ClusterDef("storage", "content/cluster.foo/storage"));
+        clusterDefs.add(new ClusterDef("storage2", "content/cluster.bar/storage"));
+        ClusterList clusterList = new ClusterList(clusterDefs);
+        try {
+            VdsVisit.resolveClusterRoute(clusterList, null);
+        } catch (IllegalArgumentException e) {
+            assertTrue(e.getMessage().contains("Please use the -c option to select one of them"));
+        }
+    }
+
+    @Test
+    public void testExplicitClusterOptionWithMultipleClusters() {
+        List<ClusterDef> clusterDefs = new ArrayList<>();
+        clusterDefs.add(new ClusterDef("storage", "content/cluster.foo/storage"));
+        clusterDefs.add(new ClusterDef("storage2", "content/cluster.bar/storage"));
+        ClusterList clusterList = new ClusterList(clusterDefs);
+
+        String route = VdsVisit.resolveClusterRoute(clusterList, "storage2");
+        assertEquals("[Storage:cluster=storage2;clusterconfigid=content/cluster.bar/storage]", route);
+    }
+
+    @Test
+    public void testFailIfNoContentClustersAvailable() {
+        List<ClusterDef> clusterDefs = new ArrayList<>();
+        ClusterList clusterList = new ClusterList(clusterDefs);
+        try {
+            VdsVisit.resolveClusterRoute(clusterList, null);
+        } catch (IllegalArgumentException e) {
+            assertTrue(e.getMessage().contains("Your Vespa cluster does not have any content clusters"));
+        }
+    }
+
+    @Test
+    public void testStatistics() throws Exception {
+        String[] args = new String[] {
+                "--statistics", "foo"
+        };
+        VdsVisit.ArgumentParser parser = createMockArgumentParser();
+        VdsVisit.VdsVisitParameters allParams = parser.parse(args);
+        assertNotNull(allParams);
+
+        VisitorParameters params = allParams.getVisitorParameters();
+        assertNotNull(params);
+        assertEquals("foo", allParams.getStatisticsParts());
+        assertEquals("[id]", params.getFieldSet());
+        assertEquals("CountVisitor", params.getVisitorLibrary());
+    }
+
+    // TODO: use DummyVisitorSession instead?
+    private static class MockVisitorSession implements VisitorSession {
+        private VisitorParameters params;
+
+        public MockVisitorSession(VisitorParameters params) {
+            this.params = params;
+            params.getLocalDataHandler().setSession(this);
+        }
+
+        @Override
+        public boolean isDone() {
+            return true;
+        }
+
+        @Override
+        public ProgressToken getProgress() {
+            return null;
+        }
+
+        @Override
+        public Trace getTrace() {
+            return null;
+        }
+
+        @Override
+        public boolean waitUntilDone(long l) throws InterruptedException {
+            params.getControlHandler().onDone(VisitorControlHandler.CompletionCode.SUCCESS, "woo!");
+            // Return immediately
+            return true;
+        }
+
+        @Override
+        public void ack(AckToken ackToken) {
+        }
+
+        @Override
+        public void abort() {
+        }
+
+        @Override
+        public VisitorResponse getNext() {
+            return null;
+        }
+
+        @Override
+        public VisitorResponse getNext(int i) throws InterruptedException {
+            return null;
+        }
+
+        @Override
+        public void destroy() {
+        }
+    }
+
+    private static class MockVisitorSessionAccessor implements VdsVisit.VisitorSessionAccessor {
+        boolean shutdown = false;
+        @Override
+        public VisitorSession createVisitorSession(VisitorParameters params) throws ParseException {
+            return new MockVisitorSession(params);
+        }
+
+        @Override
+        public void shutdown() {
+            shutdown = true;
+        }
+
+        public boolean isShutdown() {
+            return shutdown;
+        }
+    }
+
+    private static class MockVisitorSessionAccessorFactory implements VdsVisit.VisitorSessionAccessorFactory {
+
+        private MockVisitorSessionAccessor lastCreatedAccessor = null;
+
+        @Override
+        public VdsVisit.VisitorSessionAccessor createVisitorSessionAccessor() {
+            lastCreatedAccessor = new MockVisitorSessionAccessor();
+            return lastCreatedAccessor;
+        }
+
+        public MockVisitorSessionAccessor getLastCreatedAccessor() {
+            return lastCreatedAccessor;
+        }
+    }
+
+    private static class MockShutdownHookRegistrar implements VdsVisit.ShutdownHookRegistrar {
+        Thread cleanUpThread;
+
+        @Override
+        public void registerShutdownHook(Thread thread) {
+            cleanUpThread = thread;
+        }
+
+        public Thread getCleanUpThread() {
+            return cleanUpThread;
+        }
+    }
+
+    @Test
+    public void testVdsVisitRunLogic() {
+        MockVisitorSessionAccessorFactory accessorFactory = new MockVisitorSessionAccessorFactory();
+        MockShutdownHookRegistrar shutdownHookRegistrar = new MockShutdownHookRegistrar();
+        VdsVisit vdsVisit = new VdsVisit(accessorFactory, shutdownHookRegistrar);
+
+        VdsVisit.VdsVisitParameters params = new VdsVisit.VdsVisitParameters();
+        VisitorParameters visitorParameters = new VisitorParameters("");
+        params.setVisitorParameters(visitorParameters);
+
+        visitorParameters.setResumeFileName("src/test/files/progress.txt");
+        vdsVisit.setVdsVisitParameters(params);
+
+        int code = vdsVisit.doRun();
+        assertEquals(0, code);
+
+        assertNotNull(shutdownHookRegistrar.getCleanUpThread());
+        shutdownHookRegistrar.getCleanUpThread().run();
+
+        assertNotNull(accessorFactory.getLastCreatedAccessor());
+        assertTrue(accessorFactory.getLastCreatedAccessor().isShutdown());
+
+        // Ensure progress token stuff was read from file
+        ProgressToken progress = visitorParameters.getResumeToken();
+        assertNotNull(progress);
+        assertEquals(14, progress.getDistributionBitCount());
+        assertEquals(3, progress.getPendingBucketCount());
+    }
+
+    @Test
+    public void testVdsVisitRunLogicProgressFileNotYetCreated() {
+        MockVisitorSessionAccessorFactory accessorFactory = new MockVisitorSessionAccessorFactory();
+        MockShutdownHookRegistrar shutdownHookRegistrar = new MockShutdownHookRegistrar();
+        VdsVisit vdsVisit = new VdsVisit(accessorFactory, shutdownHookRegistrar);
+
+        VdsVisit.VdsVisitParameters params = new VdsVisit.VdsVisitParameters();
+        VisitorParameters visitorParameters = new VisitorParameters("");
+        params.setVisitorParameters(visitorParameters);
+
+        visitorParameters.setResumeFileName("src/test/files/progress-not-existing.txt");
+        vdsVisit.setVdsVisitParameters(params);
+
+        // Should not fail with file not found
+        int code = vdsVisit.doRun();
+        assertEquals(0, code);
+
+        assertNotNull(shutdownHookRegistrar.getCleanUpThread());
+        shutdownHookRegistrar.getCleanUpThread().run();
+
+        assertNotNull(accessorFactory.getLastCreatedAccessor());
+        assertTrue(accessorFactory.getLastCreatedAccessor().isShutdown());
+    }
+}
diff --git a/vespaclient/src/vespa/vespaclient/spoolmaster/application.cpp b/vespaclient/src/vespa/vespaclient/spoolmaster/application.cpp
index fe0880d9ea1..e708e623592 100644
--- a/vespaclient/src/vespa/vespaclient/spoolmaster/application.cpp
+++ b/vespaclient/src/vespa/vespaclient/spoolmaster/application.cpp
@@ -1,10 +1,8 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 #include <vespa/defaults.h>
-#include <vector>
-#include <string>
+#include <thread>
 #include <iostream>
 #include <algorithm>
-#include <cstdio>
 #include <dirent.h>
 #include <unistd.h>
 
@@ -189,7 +187,7 @@ Application::Main()
             if (scanInbox() && findOutboxes()) {
                 moveLinks();
             } else {
-                FastOS_Thread::Sleep(200);
+                std::this_thread::sleep_for(std::chrono::milliseconds(200));
             }
         }
     }
diff --git a/vespaclient/src/vespa/vespaclient/vdsstates/statesapp.cpp b/vespaclient/src/vespa/vespaclient/vdsstates/statesapp.cpp
index 7ea270eb306..b1be394b65f 100644
--- a/vespaclient/src/vespa/vespaclient/vdsstates/statesapp.cpp
+++ b/vespaclient/src/vespa/vespaclient/vdsstates/statesapp.cpp
@@ -1,11 +1,8 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
-
-
 #include <vespa/document/util/stringutil.h>
 #include <vespa/fnet/frt/frt.h>
 #include <vespa/slobrok/sbmirror.h>
-#include <iostream>
 #include <vespa/vdslib/distribution/distribution.h>
 #include <vespa/vdslib/state/clusterstate.h>
 #include <vespa/vespalib/util/programoptions.h>
@@ -14,6 +11,8 @@
 #include <vespa/config-stor-distribution.h>
 #include <vespa/config/helper/configgetter.hpp>
 #include <vespa/fastos/app.h>
+#include <sstream>
+#include <iostream>
 
 #include <vespa/log/log.h>
 LOG_SETUP("vdsstatetool");
diff --git a/vespaclient/src/vespa/vespaclient/vespadoclocator/application.cpp b/vespaclient/src/vespa/vespaclient/vespadoclocator/application.cpp
index 303d929f992..ce23fcd93cb 100644
--- a/vespaclient/src/vespa/vespaclient/vespadoclocator/application.cpp
+++ b/vespaclient/src/vespa/vespaclient/vespadoclocator/application.cpp
@@ -1,9 +1,11 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include "application.h"
 #include <boost/program_options.hpp>
 #include <vespa/vespalib/util/exceptions.h>
 #include <vespa/config/common/exceptions.h>
-#include "application.h"
+#include <vespa/document/base/idstringexception.h>
 #include <iostream>
+
 #include <vespa/log/log.h>
 LOG_SETUP("vespadoclocator");
 
diff --git a/vespaclient/src/vespa/vespaclient/vespadoclocator/locator.cpp b/vespaclient/src/vespa/vespaclient/vespadoclocator/locator.cpp
index 030f284c182..af805d461fc 100644
--- a/vespaclient/src/vespa/vespaclient/vespadoclocator/locator.cpp
+++ b/vespaclient/src/vespa/vespaclient/vespadoclocator/locator.cpp
@@ -6,8 +6,7 @@
 #include <vespa/messagebus/iconfighandler.h>
 #include <vespa/messagebus/routing/routingspec.h>
 #include <vespa/vdslib/bucketdistribution.h>
-#include <vespa/messagebus/config-messagebus.h>
-#include <vespa/config/helper/configgetter.h>
+#include <vespa/vespalib/util/stringfmt.h>
 #include <vespa/config/helper/configgetter.hpp>
 
 
diff --git a/vespaclient/src/vespa/vespaclient/vesparoute/application.cpp b/vespaclient/src/vespa/vespaclient/vesparoute/application.cpp
index e9482c1eddc..baf7c01d631 100644
--- a/vespaclient/src/vespa/vespaclient/vesparoute/application.cpp
+++ b/vespaclient/src/vespa/vespaclient/vesparoute/application.cpp
@@ -12,7 +12,7 @@
 #include <vespa/messagebus/rpcmessagebus.h>
 #include <vespa/slobrok/sbmirror.h>
 #include <vespa/config/helper/configgetter.hpp>
-
+#include <vespa/vespalib/util/stringfmt.h>
 
 using config::ConfigGetter;
 using document::DocumenttypesConfig;
diff --git a/vespalib/src/tests/atomic/atomic_bench.cpp b/vespalib/src/tests/atomic/atomic_bench.cpp
index 003ef25ba45..6294f956507 100644
--- a/vespalib/src/tests/atomic/atomic_bench.cpp
+++ b/vespalib/src/tests/atomic/atomic_bench.cpp
@@ -2,6 +2,7 @@
 
 #include <vespa/vespalib/testkit/testapp.h>
 #include <vespa/vespalib/util/atomic.h>
+#include <vespa/fastos/thread.h>
 #include <vector>
 #include <algorithm>
 #include <sstream>
diff --git a/vespalib/src/tests/atomic/atomic_test.cpp b/vespalib/src/tests/atomic/atomic_test.cpp
index 1133d9b51bd..09fc99edeb9 100644
--- a/vespalib/src/tests/atomic/atomic_test.cpp
+++ b/vespalib/src/tests/atomic/atomic_test.cpp
@@ -1,12 +1,14 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-#include <vespa/log/log.h>
-LOG_SETUP("atomic_test");
+
 #include <vespa/vespalib/testkit/testapp.h>
 #include <vespa/vespalib/util/atomic.h>
+#include <vespa/fastos/thread.h>
 #include <vector>
 #include <algorithm>
 #include <sstream>
 
+#include <vespa/log/log.h>
+LOG_SETUP("atomic_test");
 
 class Test : public vespalib::TestApp
 {
diff --git a/vespalib/src/tests/benchmark_timer/benchmark_timer_test.cpp b/vespalib/src/tests/benchmark_timer/benchmark_timer_test.cpp
index b0cd577cc66..ac6a57358d6 100644
--- a/vespalib/src/tests/benchmark_timer/benchmark_timer_test.cpp
+++ b/vespalib/src/tests/benchmark_timer/benchmark_timer_test.cpp
@@ -8,7 +8,7 @@ TEST("require that the benchmark timer can be used as advertised") {
     BenchmarkTimer timer(1.0);
     while (timer.has_budget()) {
         timer.before();
-        FastOS_Thread::Sleep(5);
+        std::this_thread::sleep_for(std::chrono::milliseconds(5));
         timer.after();
     }
     EXPECT_TRUE(timer.min_time() >= 0.0);
@@ -17,15 +17,15 @@ TEST("require that the benchmark timer can be used as advertised") {
 
 TEST("require that the benchmark timer all-in-one benchmarking works") {
     uint32_t sleep_time = 5;
-    double t = BenchmarkTimer::benchmark([sleep_time](){FastOS_Thread::Sleep(sleep_time);}, 1.0);
+    double t = BenchmarkTimer::benchmark([sleep_time](){std::this_thread::sleep_for(std::chrono::milliseconds(sleep_time));}, 1.0);
     fprintf(stderr, "5 ms sleep takes: %g ms\n", t * 1000.0);
 }
 
 TEST("require that the benchmark timer all-in-one benchmarking with baseline works") {
     uint32_t work_time = 10;
     uint32_t baseline_time = 5;
-    double t = BenchmarkTimer::benchmark([&](){FastOS_Thread::Sleep(work_time);},
-                                         [&](){FastOS_Thread::Sleep(baseline_time);}, 1.0);
+    double t = BenchmarkTimer::benchmark([&](){std::this_thread::sleep_for(std::chrono::milliseconds(work_time));},
+                                         [&](){std::this_thread::sleep_for(std::chrono::milliseconds(baseline_time));}, 1.0);
     fprintf(stderr, "10 ms sleep - 5 ms sleep takes: %g ms\n", t * 1000.0);
 }
 
@@ -33,8 +33,8 @@ TEST("require that the benchmark timer all-in-one benchmarking with baseline and
     uint32_t work_time = 2;
     uint32_t baseline_time = 1;
     uint32_t loop_cnt = 0;
-    double t = BenchmarkTimer::benchmark([&](){FastOS_Thread::Sleep(work_time); ++loop_cnt;},
-                                         [&](){FastOS_Thread::Sleep(baseline_time);}, 7, 0.0);
+    double t = BenchmarkTimer::benchmark([&](){std::this_thread::sleep_for(std::chrono::milliseconds(work_time)); ++loop_cnt;},
+                                         [&](){std::this_thread::sleep_for(std::chrono::milliseconds(baseline_time));}, 7, 0.0);
     EXPECT_EQUAL(loop_cnt, 7u);
     fprintf(stderr, "2 ms sleep - 1 ms sleep takes: %g ms\n", t * 1000.0);
 }
diff --git a/vespalib/src/tests/delegatelist/delegatelist.cpp b/vespalib/src/tests/delegatelist/delegatelist.cpp
index 11dd4d7d9e2..6a9751440a6 100644
--- a/vespalib/src/tests/delegatelist/delegatelist.cpp
+++ b/vespalib/src/tests/delegatelist/delegatelist.cpp
@@ -3,10 +3,7 @@
 #include <vespa/vespalib/testkit/testapp.h>
 #include <vespa/vespalib/util/delegatelist.hpp>
 #include <vespa/vespalib/util/guard.h>
-
-#include <memory>
-#include <algorithm>
-#include <vector>
+#include <vespa/fastos/thread.h>
 #include <queue>
 
 #include <vespa/log/log.h>
diff --git a/vespalib/src/tests/exception_classes/exception_classes_test.cpp b/vespalib/src/tests/exception_classes/exception_classes_test.cpp
index 946c3fa32e4..a418cb421ed 100644
--- a/vespalib/src/tests/exception_classes/exception_classes_test.cpp
+++ b/vespalib/src/tests/exception_classes/exception_classes_test.cpp
@@ -1,6 +1,7 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 #include <vespa/vespalib/testkit/test_kit.h>
 #include <vespa/vespalib/util/exceptions.h>
+#include <vespa/vespalib/util/stringfmt.h>
 
 using namespace vespalib;
 
diff --git a/vespalib/src/tests/random/friendfinder.cpp b/vespalib/src/tests/random/friendfinder.cpp
index 5f6e55815e5..7bf390ef083 100644
--- a/vespalib/src/tests/random/friendfinder.cpp
+++ b/vespalib/src/tests/random/friendfinder.cpp
@@ -1,8 +1,8 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 #include <vespa/vespalib/util/random.h>
-#include <stdio.h>
 #include <vespa/vespalib/stllike/string.h>
 #include <cmath>
+#include <vector>
 
 int main(int argc, char **argv)
 {
diff --git a/vespalib/src/tests/time/time_box_test.cpp b/vespalib/src/tests/time/time_box_test.cpp
index 478d749e15d..32cf1eb66b3 100644
--- a/vespalib/src/tests/time/time_box_test.cpp
+++ b/vespalib/src/tests/time/time_box_test.cpp
@@ -10,13 +10,13 @@ TEST("require that long-lived timebox returns falling time left numbers") {
         double timeLeft = box.timeLeft();
         EXPECT_TRUE(timeLeft <= last_timeLeft);
         last_timeLeft = timeLeft;
-        FastOS_Thread::Sleep(10);
+        std::this_thread::sleep_for(std::chrono::milliseconds(10));
     }
 }
 
 TEST("require that short-lived timebox times out") {
     vespalib::TimeBox box(0.125);
-    FastOS_Thread::Sleep(150);
+    std::this_thread::sleep_for(std::chrono::milliseconds(150));
     EXPECT_FALSE(box.hasTimeLeft());
     EXPECT_EQUAL(box.timeLeft(), 0.0);
 }
@@ -27,7 +27,7 @@ TEST("require that short-lived timebox always returns at least minimum time") {
         double timeLeft = box.timeLeft();
         EXPECT_TRUE(timeLeft <= 0.250);
         EXPECT_TRUE(timeLeft >= 0.125);
-        FastOS_Thread::Sleep(30);
+        std::this_thread::sleep_for(std::chrono::milliseconds(30));
     }
     EXPECT_FALSE(box.hasTimeLeft());
     EXPECT_EQUAL(box.timeLeft(), 0.125);
diff --git a/vespalib/src/vespa/vespalib/io/fileutil.cpp b/vespalib/src/vespa/vespalib/io/fileutil.cpp
index c9778971efd..2e3994d329f 100644
--- a/vespalib/src/vespa/vespalib/io/fileutil.cpp
+++ b/vespalib/src/vespa/vespalib/io/fileutil.cpp
@@ -1,16 +1,12 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "fileutil.h"
-
-#include <errno.h>
-#include <ostream>
-#include <sys/stat.h>
-#include <sys/types.h>
-#include <unistd.h>
 #include <vespa/vespalib/util/exceptions.h>
-#include <vespa/vespalib/util/error.h>
 #include <vespa/vespalib/stllike/asciistream.h>
+#include <vespa/vespalib/util/stringfmt.h>
 #include <vespa/fastos/file.h>
+#include <ostream>
+#include <sys/stat.h>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".vespalib.io.fileutil");
diff --git a/vespalib/src/vespa/vespalib/net/server_socket.cpp b/vespalib/src/vespa/vespalib/net/server_socket.cpp
index 9359f5824c0..0a5310bbeb3 100644
--- a/vespalib/src/vespa/vespalib/net/server_socket.cpp
+++ b/vespalib/src/vespa/vespalib/net/server_socket.cpp
@@ -2,6 +2,8 @@
 
 #include "server_socket.h"
 #include "socket_spec.h"
+#include <sys/stat.h>
+#include <dirent.h>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".vespalib.net.server_socket");
diff --git a/vespalib/src/vespa/vespalib/objects/nbostream.cpp b/vespalib/src/vespa/vespalib/objects/nbostream.cpp
index ec04ffb8b3b..e37399bdfcc 100644
--- a/vespalib/src/vespa/vespalib/objects/nbostream.cpp
+++ b/vespalib/src/vespa/vespalib/objects/nbostream.cpp
@@ -2,7 +2,7 @@
 #include "nbostream.hpp"
 #include "hexdump.h"
 #include <vespa/vespalib/util/exceptions.h>
-
+#include <vespa/vespalib/util/stringfmt.h>
 
 namespace vespalib {
 
diff --git a/vespalib/src/vespa/vespalib/util/buffer.h b/vespalib/src/vespa/vespalib/util/buffer.h
index 88fe6359aae..3a43a2c0dc4 100644
--- a/vespalib/src/vespa/vespalib/util/buffer.h
+++ b/vespalib/src/vespa/vespalib/util/buffer.h
@@ -1,7 +1,7 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 #pragma once
 
-#include <sys/types.h>
+#include <cstddef>
 
 namespace vespalib {
 
diff --git a/vespalib/src/vespa/vespalib/util/exception.h b/vespalib/src/vespa/vespalib/util/exception.h
index 44911ea5555..05f48295bfa 100644
--- a/vespalib/src/vespa/vespalib/util/exception.h
+++ b/vespalib/src/vespa/vespalib/util/exception.h
@@ -5,7 +5,6 @@
 
 #include <vespa/vespalib/util/macro.h>
 #include <vespa/vespalib/util/error.h>
-#include <vespa/vespalib/util/stringfmt.h>
 #include <exception>
 
 #define VESPALIB_EXCEPTION_USEBACKTRACES
diff --git a/vespalib/src/vespa/vespalib/util/exceptions.h b/vespalib/src/vespa/vespalib/util/exceptions.h
index 65d17baf654..ca05dac107f 100644
--- a/vespalib/src/vespa/vespalib/util/exceptions.h
+++ b/vespalib/src/vespa/vespalib/util/exceptions.h
@@ -13,6 +13,7 @@
 #pragma once
 
 #include <vespa/vespalib/util/exception.h>
+#include <vespa/vespalib/util/stringfmt.h>
 
 namespace vespalib {
 
diff --git a/vespalib/src/vespa/vespalib/util/memory.h b/vespalib/src/vespa/vespalib/util/memory.h
index 2d940882320..85c8e46dcc2 100644
--- a/vespalib/src/vespa/vespalib/util/memory.h
+++ b/vespalib/src/vespa/vespalib/util/memory.h
@@ -4,8 +4,8 @@
 #pragma once
 
 #include <memory>
-#include <string.h>
-#include <stdlib.h>
+#include <cstring>
+#include <cstdlib>
 
 /// Macro to give you number of elements in a defined array.
 #define VESPA_NELEMS(a)    (sizeof(a)/sizeof(a[0]))
diff --git a/vespalib/src/vespa/vespalib/util/random.cpp b/vespalib/src/vespa/vespalib/util/random.cpp
index 06bfc1f9c25..2abf67605a8 100644
--- a/vespalib/src/vespa/vespalib/util/random.cpp
+++ b/vespalib/src/vespa/vespalib/util/random.cpp
@@ -2,6 +2,9 @@
 
 #include "random.h"
 #include <cmath>
+#include <cstring>
+#include <ctime>
+#include <unistd.h>
 
 namespace vespalib {
 
diff --git a/vespalib/src/vespa/vespalib/util/random.h b/vespalib/src/vespa/vespalib/util/random.h
index c4cce3b42d8..f9d2587272d 100644
--- a/vespalib/src/vespa/vespalib/util/random.h
+++ b/vespalib/src/vespa/vespalib/util/random.h
@@ -1,14 +1,7 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 #pragma once
 
-#include <stdlib.h>
-#include <stdio.h>
-#include <inttypes.h>
-#include <vespa/fastos/types.h>
-#include <vector>
-#include <sys/types.h>
-#include <unistd.h>
-#include <time.h>
+#include <cstdint>
 
 namespace vespalib {
 
@@ -22,9 +15,9 @@ private:
     /**
      * step the random generator once
      **/
-    void iterate(void) {
-        _state = (UINT64_C(0x5DEECE66D) * _state + 0xb) &
-                 UINT64_C(0xFFFFFFFFFFFF);
+    void iterate() {
+        _state = (0x5DEECE66Dul * _state + 0xb) &
+                 0xFFFFFFFFFFFFul;
     }
 
     /**
@@ -73,7 +66,7 @@ public:
      * @brief reset the seed
      **/
     void setSeed(int64_t seed) {
-        _state = (seed ^ UINT64_C(0x5DEECE66D)) & ((1L << 48) -1);
+        _state = (seed ^ 0x5DEECE66Dul) & ((1L << 48) -1);
     };
 
     /**
diff --git a/vespalib/src/vespa/vespalib/util/sync.h b/vespalib/src/vespa/vespalib/util/sync.h
index a44763ec755..315bb79487e 100644
--- a/vespalib/src/vespa/vespalib/util/sync.h
+++ b/vespalib/src/vespa/vespalib/util/sync.h
@@ -34,7 +34,6 @@ public:
 };
 #endif
 
-
 /**
  * @brief A Lock is a synchronization primitive used to ensure mutual
  * exclusion.
diff --git a/vespalog/CMakeLists.txt b/vespalog/CMakeLists.txt
index 17a91b8bbee..d613759e168 100644
--- a/vespalog/CMakeLists.txt
+++ b/vespalog/CMakeLists.txt
@@ -17,5 +17,5 @@ vespa_define_module(
     src/test/threads
 )
 
-vespa_install_script(src/logfmt/logfmt.pl logfmt bin)
-install(FILES src/logfmt/logfmt.1 DESTINATION man/man1)
+vespa_install_script(src/vespa-logfmt/vespa-logfmt.pl vespa-logfmt bin)
+install(FILES src/vespa-logfmt/vespa-logfmt.1 DESTINATION man/man1)
diff --git a/vespalog/src/logctl/.gitignore b/vespalog/src/logctl/.gitignore
index 6089704ed9a..92957f69d15 100644
--- a/vespalog/src/logctl/.gitignore
+++ b/vespalog/src/logctl/.gitignore
@@ -1,3 +1,3 @@
 .depend
 Makefile
-logctl
+vespa-logctl
diff --git a/vespalog/src/logctl/CMakeLists.txt b/vespalog/src/logctl/CMakeLists.txt
index 666e840fe60..23e7b6a10e8 100644
--- a/vespalog/src/logctl/CMakeLists.txt
+++ b/vespalog/src/logctl/CMakeLists.txt
@@ -1,8 +1,8 @@
 # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-vespa_add_executable(vespalog_logctl_app
+vespa_add_executable(vespalog_vespa-logctl_app
     SOURCES
     logctl.cpp
-    OUTPUT_NAME logctl
+    OUTPUT_NAME vespa-logctl
     INSTALL bin
     DEPENDS
     vespalog
diff --git a/vespalog/src/logctl/logctl.cpp b/vespalog/src/logctl/logctl.cpp
index 8457fb944ce..2bae7c03589 100644
--- a/vespalog/src/logctl/logctl.cpp
+++ b/vespalog/src/logctl/logctl.cpp
@@ -1,20 +1,16 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-#include <sys/types.h>
-#include <cstdio>
-#include <cstdlib>
-#include <unistd.h>
-#include <cstring>
-#include <limits.h>
-
-#include <memory>
-#include <vector>
-#include <string>
 
 #include <vespa/defaults.h>
 #include <vespa/log/control-file.h>
 #include <vespa/log/internal.h>
 #include <vespa/log/component.h>
-LOG_SETUP("logctl", "$Id$");
+
+#include <unistd.h>
+#include <dirent.h>
+#include <sys/stat.h>
+
+LOG_SETUP("vespa-logctl");
+
 
 using namespace ns_log;
 
diff --git a/vespalog/src/logger/.gitignore b/vespalog/src/logger/.gitignore
index 7cd5c615ee3..a7f5e36f079 100644
--- a/vespalog/src/logger/.gitignore
+++ b/vespalog/src/logger/.gitignore
@@ -1,5 +1,5 @@
 .depend
 Makefile
 vespa-runserver
-vlogconv
-vlogreplay
+vespa-log-conv
+vespa-log-replay
diff --git a/vespalog/src/logger/CMakeLists.txt b/vespalog/src/logger/CMakeLists.txt
index 3e055e3b288..dba37539b09 100644
--- a/vespalog/src/logger/CMakeLists.txt
+++ b/vespalog/src/logger/CMakeLists.txt
@@ -3,7 +3,7 @@ vespa_add_executable(vespalog_vlogconv_app
     SOURCES
     logger.cpp
     llreader.cpp
-    OUTPUT_NAME vlogconv
+    OUTPUT_NAME vespa-log-conv
     INSTALL bin
     DEPENDS
     vespalog
@@ -11,7 +11,7 @@ vespa_add_executable(vespalog_vlogconv_app
 vespa_add_executable(vespalog_vlogreplay_app
     SOURCES
     logreplay.c
-    OUTPUT_NAME vlogreplay
+    OUTPUT_NAME vespa-log-replay
     INSTALL bin
     DEPENDS
     m
diff --git a/vespalog/src/test/bufferedlogskiptest.cpp b/vespalog/src/test/bufferedlogskiptest.cpp
index 5b386d18e90..cb16b73fa24 100644
--- a/vespalog/src/test/bufferedlogskiptest.cpp
+++ b/vespalog/src/test/bufferedlogskiptest.cpp
@@ -1,18 +1,12 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
-#include <vespa/log/log.h>
+#include <vespa/log/bufferedlogger.h>
 
 #include <fstream>
 #include <iostream>
-#include <sstream>
-
-#include <sys/types.h>
-#include <stdlib.h>
 #include <unistd.h>
-#include <signal.h>
 
-LOG_SETUP("bufferedlogskiptest",
-          "$Id$");
+LOG_SETUP("bufferedlogskiptest");
 
 std::string readFile(const std::string& file) {
     std::ostringstream ost;
diff --git a/vespalog/src/test/bufferedlogtest.cpp b/vespalog/src/test/bufferedlogtest.cpp
index 0e27bd65988..97f70ebe004 100644
--- a/vespalog/src/test/bufferedlogtest.cpp
+++ b/vespalog/src/test/bufferedlogtest.cpp
@@ -1,21 +1,15 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
-#include <vespa/log/log.h>
+#include <vespa/log/bufferedlogger.h>
 
 #include "bufferedlogtest.logger1.h"
 #include "bufferedlogtest.logger2.h"
 
 #include <fstream>
 #include <iostream>
-#include <sstream>
-
-#include <sys/types.h>
-#include <stdlib.h>
 #include <unistd.h>
-#include <signal.h>
 
-LOG_SETUP("bufferedlogtest",
-          "$Id$");
+LOG_SETUP("bufferedlogtest");
 
 std::string readFile(const std::string& file) {
     std::ostringstream ost;
diff --git a/vespalog/src/test/bufferedlogtest.logger1.cpp b/vespalog/src/test/bufferedlogtest.logger1.cpp
index 992de19fea5..5af4b9489a4 100644
--- a/vespalog/src/test/bufferedlogtest.logger1.cpp
+++ b/vespalog/src/test/bufferedlogtest.logger1.cpp
@@ -1,6 +1,6 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
-#include <vespa/log/log.h>
+#include <vespa/log/bufferedlogger.h>
 #include "bufferedlogtest.logger1.h"
 
 LOG_SETUP(".logger1");
diff --git a/vespalog/src/test/bufferedlogtest.logger2.cpp b/vespalog/src/test/bufferedlogtest.logger2.cpp
index 688ecd153aa..ac6935c2c68 100644
--- a/vespalog/src/test/bufferedlogtest.logger2.cpp
+++ b/vespalog/src/test/bufferedlogtest.logger2.cpp
@@ -1,6 +1,6 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
-#include <vespa/log/log.h>
+#include <vespa/log/bufferedlogger.h>
 #include "bufferedlogtest.logger2.h"
 
 LOG_SETUP(".logger2");
diff --git a/vespalog/src/test/threads/testthreads.cpp b/vespalog/src/test/threads/testthreads.cpp
index a708777f350..90b8c13e497 100644
--- a/vespalog/src/test/threads/testthreads.cpp
+++ b/vespalog/src/test/threads/testthreads.cpp
@@ -1,12 +1,12 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 #include <vespa/fastos/app.h>
 #include <vespa/fastos/time.h>
-#include <vespa/log/log.h>
+#include <vespa/fastos/thread.h>
+#include <vespa/log/bufferedlogger.h>
 #include <iostream>
 
 using std::string;
 
-
 LOG_SETUP(".threadtest");
 
 class FileThread : public FastOS_Runnable
diff --git a/vespalog/src/logfmt/.gitignore b/vespalog/src/vespa-logfmt/.gitignore
index 568d69eee3e..568d69eee3e 100644
--- a/vespalog/src/logfmt/.gitignore
+++ b/vespalog/src/vespa-logfmt/.gitignore
diff --git a/vespalog/src/logfmt/logfilter.c b/vespalog/src/vespa-logfmt/logfilter.c
index 709ce2c016e..709ce2c016e 100644
--- a/vespalog/src/logfmt/logfilter.c
+++ b/vespalog/src/vespa-logfmt/logfilter.c
diff --git a/vespalog/src/logfmt/logfmt.1 b/vespalog/src/vespa-logfmt/vespa-logfmt.1
index a16b1ea2eb3..57c8d46370e 100644
--- a/vespalog/src/logfmt/logfmt.1
+++ b/vespalog/src/vespa-logfmt/vespa-logfmt.1
@@ -1,11 +1,11 @@
 .\" $Id: logfmt.1,v 1.12 2007-06-19 09:37:25 daljord Exp $
 .\"
-.Dd October 29, 2004
-.Dt LOGFMT \&1 "Vespa documentation"
-.Os "Yahoo! Vespa" "2.0"
+.Dd Jun 7, 2017
+.Dt vespa-logfmt \&1 "Vespa documentation"
+.Os "Yahoo! Vespa" "6.0"
 .Os
 .Sh NAME
-.Nm logfmt
+.Nm vespa-logfmt
 .Nd select and format messages from Vespa log files
 .Sh SYNOPSIS
 .Nm
@@ -68,7 +68,7 @@ is shown.
 Print the time in human-readable [YYYY-MM-DD HH:mm:ss] format.
 Note that the time is printed in the local timezone; to get GMT
 output use
-.Nm "\*[q]env TZ=GMT logfmt\*[q]"
+.Nm "\*[q]env TZ=GMT vespa-logfmt\*[q]"
 as your command.
 .It msecs
 Add milliseconds after the seconds in
@@ -155,7 +155,7 @@ truncating if necessary.
 The command:
 .Pp
 .Bd -literal -offset indent
-logfmt -l event -s service,message,fmttime,message
+vespa-logfmt -l event -s service,message,fmttime,message
 .Ed
 .Pp
 will display only messages with log level "event",
@@ -179,7 +179,7 @@ order is.
 The command:
 .Pp
 .Bd -literal -offset indent
-logfmt -l all-info,-debug -s level \e
+vespa-logfmt -l all-info,-debug -s level \e
        -s time,usecs,component,message -t -l -event
 .Ed
 .Pp
diff --git a/vespalog/src/logfmt/logfmt.pl b/vespalog/src/vespa-logfmt/vespa-logfmt.pl
index 8d01feb3a6b..8d01feb3a6b 100755
--- a/vespalog/src/logfmt/logfmt.pl
+++ b/vespalog/src/vespa-logfmt/vespa-logfmt.pl
diff --git a/vespalog/src/vespa/log/bufferedlogger.cpp b/vespalog/src/vespa/log/bufferedlogger.cpp
index 30f3dcb648d..db2021324c3 100644
--- a/vespalog/src/vespa/log/bufferedlogger.cpp
+++ b/vespalog/src/vespa/log/bufferedlogger.cpp
@@ -1,14 +1,7 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
-#include <vespa/log/bufferedlogger.h>
-
-#include <iomanip>
-#include <iostream>
-#include <sstream>
-#include <vector>
-#include <stdarg.h>
-#include <sys/time.h>
-
+#include "bufferedlogger.h"
+#include <vespa/fastos/mutex.h>
 #include <boost/multi_index_container.hpp>
 #include <boost/multi_index/identity.hpp>
 #include <boost/multi_index/member.hpp>
@@ -16,6 +9,13 @@
 #include <boost/multi_index/ordered_index.hpp>
 #include <boost/multi_index/sequenced_index.hpp>
 
+#include <iomanip>
+#include <iostream>
+#include <sstream>
+#include <vector>
+#include <cstdarg>
+#include <ctime>
+
 namespace ns_log {
 
 // implementation details for BufferedLogger
diff --git a/vespalog/src/vespa/log/bufferedlogger.h b/vespalog/src/vespa/log/bufferedlogger.h
index 4f20f46af4c..9ab02f7842f 100644
--- a/vespalog/src/vespa/log/bufferedlogger.h
+++ b/vespalog/src/vespa/log/bufferedlogger.h
@@ -82,7 +82,6 @@
 #define VESPA_LOG_LOGENTRYMAXAGE 300 // Max seconds an entry can be cached
 #define VESPA_LOG_COUNTAGEFACTOR   5 // How many seconds each count counts for
 
-#include <vespa/fastos/mutex.h>
 #include <vespa/log/log.h>
 #include <sstream>
 #include <string>
@@ -199,7 +198,6 @@ public:
 
     /** Trim the buffer. Removing old messages if wanted. */
     void trimCache();
-
 };
 
 } // ns_log
diff --git a/vespalog/src/vespa/log/log-target-file.cpp b/vespalog/src/vespa/log/log-target-file.cpp
index 07b638d7f84..e88f1018675 100644
--- a/vespalog/src/vespa/log/log-target-file.cpp
+++ b/vespalog/src/vespa/log/log-target-file.cpp
@@ -1,16 +1,12 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-#include <sys/types.h>
 #include <unistd.h>
 #include <cstring>
-#include <sys/stat.h>
 #include <fcntl.h>
-#include <cstdlib>
-#include <errno.h>
-#include <cstdio>
+#include <cerrno>
+#include <cassert>
 
 #include "log.h"
 LOG_SETUP(".log");
-LOG_RCSID("$Id$");
 #include "log-target-file.h"
 #include "internal.h"
 
diff --git a/vespalog/src/vespa/log/log-target.cpp b/vespalog/src/vespa/log/log-target.cpp
index d34e0067b04..b6692caa308 100644
--- a/vespalog/src/vespa/log/log-target.cpp
+++ b/vespalog/src/vespa/log/log-target.cpp
@@ -1,11 +1,9 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-#include <sys/types.h>
 #include <cstring>
-#include <cstdlib>
+#include <cassert>
 
 #include "log.h"
 LOG_SETUP(".log");
-LOG_RCSID("$Id$");
 
 #include "log-target-fd.h"
 #include "log-target-file.h"
diff --git a/vespalog/src/vespa/log/log.cpp b/vespalog/src/vespa/log/log.cpp
index 7e92008504e..c31d110d241 100644
--- a/vespalog/src/vespa/log/log.cpp
+++ b/vespalog/src/vespa/log/log.cpp
@@ -10,6 +10,7 @@ LOG_SETUP_INDIRECT(".log", "$Id$");
 #include "log-target.h"
 #include "internal.h"
 #include "control-file.h"
+#include "bufferedlogger.h"
 
 #include <vespa/fastos/thread.h>
 
@@ -222,6 +223,7 @@ Logger::doLog(LogLevel level, const char *file, int line, const char *fmt, ...)
         actualSize = tryLog(sizeofPayload, level, file, line, fmt, args);
         va_end(args);
     } while (sizeofPayload < actualSize);
+    ns_log::BufferedLogger::logger.trimCache();
 }
 
 void Logger::doLogCore(uint64_t timestamp, LogLevel level,
diff --git a/vespalog/src/vespa/log/log.h b/vespalog/src/vespa/log/log.h
index 6ecd69de792..53a0eec140c 100644
--- a/vespalog/src/vespa/log/log.h
+++ b/vespalog/src/vespa/log/log.h
@@ -38,14 +38,12 @@ static int log_dummmy __attribute__((unused)) = logger.setRcsId(x)
 do {                                                                          \
     if (logger.wants(ns_log::Logger::level)) {                                \
         logger.doLog(ns_log::Logger::level, __FILE__, __LINE__, __VA_ARGS__); \
-        ns_log::BufferedLogger::logger.trimCache();                           \
     }                                                                         \
 } while (0)
 #define VLOG(level, ...)                                      \
 do {                                                          \
     if (logger.wants(level)) {                                \
         logger.doLog(level, __FILE__, __LINE__, __VA_ARGS__); \
-        ns_log::BufferedLogger::logger.trimCache();           \
     }                                                         \
 } while (0)
 #endif
@@ -302,6 +300,3 @@ extern void log_abort(const char *message,
 #else
 #define LOG_ASSERT(expr)
 #endif // #ifndef NDEBUG
-
-#include <vespa/log/bufferedlogger.h>
-
diff --git a/vsm/src/tests/docsum/docsum.cpp b/vsm/src/tests/docsum/docsum.cpp
index d854421ecb5..1b0fbf70623 100644
--- a/vsm/src/tests/docsum/docsum.cpp
+++ b/vsm/src/tests/docsum/docsum.cpp
@@ -1,8 +1,7 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 #include <vespa/vespalib/testkit/testapp.h>
-
-#include <vector>
 #include <vespa/document/fieldvalue/fieldvalues.h>
+#include <vespa/document/datatype/structdatatype.h>
 #include <vespa/vsm/common/docsum.h>
 #include <vespa/vsm/vsm/flattendocsumwriter.h>
 #include <vespa/vsm/vsm/slimefieldwriter.h>
diff --git a/vsm/src/tests/document/document.cpp b/vsm/src/tests/document/document.cpp
index 35ca55826ba..629e2954953 100644
--- a/vsm/src/tests/document/document.cpp
+++ b/vsm/src/tests/document/document.cpp
@@ -2,10 +2,10 @@
 #include <vespa/vespalib/testkit/testapp.h>
 
 #include <vespa/document/fieldvalue/fieldvalues.h>
+#include <vespa/document/datatype/documenttype.h>
 #include <vespa/vsm/common/storagedocument.h>
 #include <vespa/vespalib/stllike/asciistream.h>
 
-
 using namespace document;
 
 namespace vsm {
diff --git a/vsm/src/vespa/vsm/common/document.h b/vsm/src/vespa/vsm/common/document.h
index a0341aaf7c0..ea0f9216ab9 100644
--- a/vsm/src/vespa/vsm/common/document.h
+++ b/vsm/src/vespa/vsm/common/document.h
@@ -4,6 +4,7 @@
 #include <vespa/searchlib/query/base.h>
 #include <vespa/document/fieldvalue/fieldvalue.h>
 #include <vespa/vespalib/stllike/hash_map.h>
+#include <map>
 
 namespace vespalib {
     class asciistream;
diff --git a/vsm/src/vespa/vsm/common/documenttypemapping.cpp b/vsm/src/vespa/vsm/common/documenttypemapping.cpp
index 96a7effae57..9a8bbb28f65 100644
--- a/vsm/src/vespa/vsm/common/documenttypemapping.cpp
+++ b/vsm/src/vespa/vsm/common/documenttypemapping.cpp
@@ -2,6 +2,7 @@
 
 #include "documenttypemapping.h"
 #include <vespa/document/repo/documenttyperepo.h>
+#include <vespa/document/datatype/documenttype.h>
 #include <vespa/vespalib/stllike/hash_map.hpp>
 
 #include <vespa/log/log.h>
diff --git a/vsm/src/vespa/vsm/searcher/fieldsearcher.h b/vsm/src/vespa/vsm/searcher/fieldsearcher.h
index 3b82f68072b..e28275536f3 100644
--- a/vsm/src/vespa/vsm/searcher/fieldsearcher.h
+++ b/vsm/src/vespa/vsm/searcher/fieldsearcher.h
@@ -4,6 +4,7 @@
 #include <vespa/searchlib/query/query.h>
 #include <vespa/vsm/common/document.h>
 #include <vespa/vsm/common/storagedocument.h>
+#include <vespa/document/fieldvalue/iteratorhandler.h>
 
 namespace vsm {
 
@@ -79,9 +80,8 @@ public:
     size_t maxFieldLength() const { return _maxFieldLength; }
 
 private:
-    class IteratorHandler : public document::FieldValue::IteratorHandler {
+    class IteratorHandler : public document::fieldvalue::IteratorHandler {
     private:
-        typedef document::FieldValue::IteratorHandler::Content Content;
         FieldSearcher & _searcher;
 
         void onPrimitive(uint32_t fid, const Content & c) override;
diff --git a/vsm/src/vespa/vsm/searcher/utf8stringfieldsearcherbase.cpp b/vsm/src/vespa/vsm/searcher/utf8stringfieldsearcherbase.cpp
index 8e444102930..c713e0284ac 100644
--- a/vsm/src/vespa/vsm/searcher/utf8stringfieldsearcherbase.cpp
+++ b/vsm/src/vespa/vsm/searcher/utf8stringfieldsearcherbase.cpp
@@ -1,6 +1,7 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "utf8stringfieldsearcherbase.h"
+#include <cassert>
 
 using search::QueryTerm;
 using search::QueryTermList;
diff --git a/vsm/src/vespa/vsm/searcher/utf8substringsnippetmodifier.cpp b/vsm/src/vespa/vsm/searcher/utf8substringsnippetmodifier.cpp
index 23e64d0bf5e..b5d49f463ef 100644
--- a/vsm/src/vespa/vsm/searcher/utf8substringsnippetmodifier.cpp
+++ b/vsm/src/vespa/vsm/searcher/utf8substringsnippetmodifier.cpp
@@ -1,5 +1,6 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 #include "utf8substringsnippetmodifier.h"
+#include <cassert>
 
 using search::byte;
 using search::QueryTerm;
diff --git a/vsm/src/vespa/vsm/vsm/docsumfilter.cpp b/vsm/src/vespa/vsm/vsm/docsumfilter.cpp
index 961add6d7d6..b7ed6c0b7da 100644
--- a/vsm/src/vespa/vsm/vsm/docsumfilter.cpp
+++ b/vsm/src/vespa/vsm/vsm/docsumfilter.cpp
@@ -4,6 +4,7 @@
 #include "slimefieldwriter.h"
 #include <vespa/searchsummary/docsummary/summaryfieldconverter.h>
 #include <vespa/document/base/exceptions.h>
+#include <vespa/document/fieldvalue/iteratorhandler.h>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".vsm.docsumfilter");
@@ -13,9 +14,8 @@ using namespace search::docsummary;
 
 namespace {
 
-class Handler : public document::FieldValue::IteratorHandler {
+class Handler : public document::fieldvalue::IteratorHandler {
 public:
-    typedef document::FieldValue::IteratorHandler::Content Content;
 };
 
 struct IntResultHandler : public Handler {
diff --git a/vsm/src/vespa/vsm/vsm/flattendocsumwriter.h b/vsm/src/vespa/vsm/vsm/flattendocsumwriter.h
index 9c6ba8dae9c..aa58a6ae6df 100644
--- a/vsm/src/vespa/vsm/vsm/flattendocsumwriter.h
+++ b/vsm/src/vespa/vsm/vsm/flattendocsumwriter.h
@@ -2,6 +2,7 @@
 #pragma once
 
 #include <vespa/document/fieldvalue/fieldvalue.h>
+#include <vespa/document/fieldvalue/iteratorhandler.h>
 #include <vespa/vsm/common/charbuffer.h>
 
 namespace vsm {
@@ -10,10 +11,8 @@ namespace vsm {
  * This class is used to flatten out and write a complex field value.
  * A separator string is inserted between primitive field values.
  **/
-class FlattenDocsumWriter : public document::FieldValue::IteratorHandler {
+class FlattenDocsumWriter : public document::fieldvalue::IteratorHandler {
 private:
-    typedef document::FieldValue::IteratorHandler::Content Content;
-
     CharBuffer  _output;
     vespalib::string _separator;
     bool        _useSeparator;
diff --git a/vsm/src/vespa/vsm/vsm/snippetmodifier.h b/vsm/src/vespa/vsm/vsm/snippetmodifier.h
index 17d6bb44d4c..182f09d5d55 100644
--- a/vsm/src/vespa/vsm/vsm/snippetmodifier.h
+++ b/vsm/src/vespa/vsm/vsm/snippetmodifier.h
@@ -7,6 +7,7 @@
 #include <vespa/vsm/common/fieldmodifier.h>
 #include <vespa/vsm/searcher/utf8substringsnippetmodifier.h>
 #include <vespa/document/fieldvalue/fieldvalue.h>
+#include <vespa/document/fieldvalue/iteratorhandler.h>
 
 namespace vsm {
 
@@ -19,11 +20,9 @@ namespace vsm {
  * responsible for modifying the field value by inserting unit separators before and after matches.
  * A group separator is inserted between primitive field values the same way as done by FlattenDocsumWriter.
  **/
-class SnippetModifier : public FieldModifier, public document::FieldValue::IteratorHandler
+class SnippetModifier : public FieldModifier, public document::fieldvalue::IteratorHandler
 {
 private:
-    typedef document::FieldValue::IteratorHandler::Content Content;
-
     UTF8SubstringSnippetModifier::SP _searcher;
     CharBuffer::SP                   _valueBuf; // buffer to store the final modified field value
     char                             _groupSep;
author	Jon Bratseth <bratseth@yahoo-inc.com>	2017-06-08 11:14:36 +0200
committer	Jon Bratseth <bratseth@yahoo-inc.com>	2017-06-08 11:14:36 +0200
commit	2b5100db52c300ce7b336da6dae2008baad57358 (patch)
tree	bd56fb339921d458b71f320317b03a2318df272f
parent	739833c51230d2bdb7699906b7acaa6a8bb76044 (diff)
parent	ab131f2a48628dd7ed86e112819860c67e8df75b (diff)